From a37148050408cff22859d5b9fb7c2971745d49de Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sat, 21 Oct 2023 16:23:31 -0400 Subject: [PATCH 001/630] main: Anagami --- cmd/walletd/main.go | 196 +++++++++++++++++++++++---- cmd/walletd/node.go | 9 +- cmd/walletd/testnet.go | 299 +++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 5 files changed, 478 insertions(+), 29 deletions(-) create mode 100644 cmd/walletd/testnet.go diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index da11cc1..24aaecf 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -1,15 +1,19 @@ package main import ( - "flag" "fmt" "log" "net" "os" "os/signal" "runtime/debug" + "strings" + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" "golang.org/x/term" + "lukechampine.com/flagg" + "lukechampine.com/frand" ) var commit = "?" @@ -59,40 +63,176 @@ func getAPIPassword() string { return apiPassword } +var ( + rootUsage = `Usage: + walletd [flags] [action] + +Run 'walletd' with no arguments to start the blockchain node and API server. + +Actions: + version print walletd version + +Testnet Actions: + seed generate a seed + mine run CPU miner + balance view wallet balance + send send a simple transaction + txns view transaction history +` + versionUsage = `Usage: + walletd version + +Prints the version of the walletd binary. +` + seedUsage = `Usage: + walletd seed + +Generates a secure testnet seed. +` + mineUsage = `Usage: + walletd mine + +Runs a testnet CPU miner. +` + balanceUsage = `Usage: + walletd balance + +Displays testnet balance. +` + sendUsage = `Usage: + walletd send [flags] [amount] [address] + +Sends a simple testnet transaction. +` + txnsUsage = `Usage: + walletd txns + +Lists testnet transactions and miner rewards. +` +) + func main() { log.SetFlags(0) - gatewayAddr := flag.String("addr", ":9981", "p2p address to listen on") - apiAddr := flag.String("http", "localhost:9980", "address to serve API on") - dir := flag.String("dir", ".", "directory to store node state in") - network := flag.String("network", "mainnet", "network to connect to") - upnp := flag.Bool("upnp", true, "attempt to forward ports and discover IP with UPnP") - flag.Parse() + + var gatewayAddr, apiAddr, dir, network, seed string + var upnp, v2 bool + + rootCmd := flagg.Root + rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) + rootCmd.StringVar(&gatewayAddr, "addr", ":9981", "p2p address to listen on") + rootCmd.StringVar(&apiAddr, "http", "localhost:9980", "address to serve API on") + rootCmd.StringVar(&dir, "dir", ".", "directory to store node state in") + rootCmd.StringVar(&network, "network", "mainnet", "network to connect to") + rootCmd.BoolVar(&upnp, "upnp", true, "attempt to forward ports and discover IP with UPnP") + rootCmd.StringVar(&seed, "seed", "", "testnet seed") + versionCmd := flagg.New("version", versionUsage) + seedCmd := flagg.New("seed", seedUsage) + mineCmd := flagg.New("mine", mineUsage) + balanceCmd := flagg.New("balance", balanceUsage) + sendCmd := flagg.New("send", sendUsage) + sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") + txnsCmd := flagg.New("txns", txnsUsage) + + cmd := flagg.Parse(flagg.Tree{ + Cmd: rootCmd, + Sub: []flagg.Tree{ + {Cmd: versionCmd}, + {Cmd: seedCmd}, + {Cmd: mineCmd}, + {Cmd: balanceCmd}, + {Cmd: sendCmd}, + {Cmd: txnsCmd}, + }, + }) log.Println("walletd v0.1.0") - if flag.Arg(0) == "version" { + switch cmd { + case rootCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + apiPassword := getAPIPassword() + l, err := net.Listen("tcp", apiAddr) + if err != nil { + log.Fatal(err) + } + n, err := newNode(gatewayAddr, dir, network, upnp) + if err != nil { + log.Fatal(err) + } + log.Println("p2p: Listening on", n.s.Addr()) + stop := n.Start() + log.Println("api: Listening on", l.Addr()) + go startWeb(l, n, apiPassword) + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, os.Interrupt) + <-signalCh + log.Println("Shutting down...") + stop() + + case versionCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } log.Println("Commit Hash:", commit) log.Println("Commit Date:", timestamp) - return - } - apiPassword := getAPIPassword() - l, err := net.Listen("tcp", *apiAddr) - if err != nil { - log.Fatal(err) - } + case seedCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + seed := frand.Bytes(8) + var entropy [32]byte + copy(entropy[:], seed) + addr := types.StandardUnlockHash(wallet.NewSeedFromEntropy(&entropy).PublicKey(0)) + fmt.Printf("Seed: %x\n", seed) + fmt.Printf("Address: %v\n", strings.TrimPrefix(addr.String(), "addr:")) - n, err := newNode(*gatewayAddr, *dir, *network, *upnp) - if err != nil { - log.Fatal(err) + case mineCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + seed := loadTestnetSeed(seed) + c := initTestnetClient(apiAddr, network, seed) + runTestnetMiner(c, seed) + + case balanceCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + seed := loadTestnetSeed(seed) + c := initTestnetClient(apiAddr, network, seed) + b, err := c.Wallet("primary").Balance() + check("Couldn't get balance:", err) + fmt.Println(b.Siacoins) + + case sendCmd: + if len(cmd.Args()) != 2 { + cmd.Usage() + return + } + seed := loadTestnetSeed(seed) + c := initTestnetClient(apiAddr, network, seed) + amount, err := types.ParseCurrency(cmd.Arg(0)) + check("Couldn't parse amount:", err) + dest, err := types.ParseAddress(cmd.Arg(1)) + check("Couldn't parse recipient address:", err) + sendTestnet(c, seed, amount, dest, v2) + + case txnsCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + seed := loadTestnetSeed(seed) + c := initTestnetClient(apiAddr, network, seed) + events, err := c.Wallet("primary").Events(0, -1) + check("Couldn't get events:", err) + printTestnetEvents(seed, events) } - log.Println("p2p: Listening on", n.s.Addr()) - stop := n.Start() - log.Println("api: Listening on", l.Addr()) - go startWeb(l, n, apiPassword) - - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt) - <-signalCh - log.Println("Shutting down...") - stop() } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index ba367ea..97d8229 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -58,6 +58,10 @@ var zenBootstrap = []string{ "51.81.208.10:9881", } +var anagamiBootstrap = []string{ + "147.135.16.182:9781", +} + type boltDB struct { tx *bolt.Tx db *bolt.DB @@ -139,8 +143,11 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) case "zen": network, genesisBlock = chain.TestnetZen() bootstrapPeers = zenBootstrap + case "anagami": + network, genesisBlock = TestnetAnagami() + bootstrapPeers = anagamiBootstrap default: - return nil, errors.New("invalid network: must be one of 'mainnet' or 'zen'") + return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go new file mode 100644 index 0000000..00100c5 --- /dev/null +++ b/cmd/walletd/testnet.go @@ -0,0 +1,299 @@ +package main + +import ( + "encoding/hex" + "fmt" + "log" + "math/big" + "os" + "reflect" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/walletd/api" + "go.sia.tech/walletd/wallet" + "golang.org/x/term" + "lukechampine.com/frand" +) + +// TestnetAnagami returns the chain parameters and genesis block for the "Anagami" +// testnet chain. +func TestnetAnagami() (*consensus.Network, types.Block) { + n := &consensus.Network{ + Name: "anagami", + + InitialCoinbase: types.Siacoins(300000), + MinimumCoinbase: types.Siacoins(300000), + InitialTarget: types.BlockID{3: 1}, + } + + n.HardforkDevAddr.Height = 1 + n.HardforkDevAddr.OldAddress = types.Address{} + n.HardforkDevAddr.NewAddress = types.Address{} + + n.HardforkTax.Height = 2 + + n.HardforkStorageProof.Height = 3 + + n.HardforkOak.Height = 5 + n.HardforkOak.FixHeight = 8 + n.HardforkOak.GenesisTimestamp = time.Unix(1697100000, 0) // Oct 12, 2023 @ 08:40 GMT + + n.HardforkASIC.Height = 13 + n.HardforkASIC.OakTime = 10 * time.Minute + n.HardforkASIC.OakTarget = n.InitialTarget + + n.HardforkFoundation.Height = 21 + n.HardforkFoundation.PrimaryAddress, _ = types.ParseAddress("addr:5949fdf56a7c18ba27f6526f22fd560526ce02a1bd4fa3104938ab744b69cf63b6b734b8341f") + n.HardforkFoundation.FailsafeAddress = n.HardforkFoundation.PrimaryAddress + + n.HardforkV2.AllowHeight = 2016 // ~2 weeks in + n.HardforkV2.RequireHeight = 2016 + 288 // ~2 days later + + b := types.Block{ + Timestamp: n.HardforkOak.GenesisTimestamp, + Transactions: []types.Transaction{{ + SiacoinOutputs: []types.SiacoinOutput{{ + Address: n.HardforkFoundation.PrimaryAddress, + Value: types.Siacoins(1).Mul64(1e12), + }}, + SiafundOutputs: []types.SiafundOutput{{ + Address: n.HardforkFoundation.PrimaryAddress, + Value: 10000, + }}, + }}, + } + + return n, b +} + +func loadTestnetSeed(s string) wallet.Seed { + if s == "" { + fmt.Println("Seed not supplied via -seed flag, falling back to manual entry.") + fmt.Print("Seed: ") + pw, err := term.ReadPassword(int(os.Stdin.Fd())) + fmt.Println() + check("Could not read API password:", err) + if err != nil { + log.Fatal(err) + } + s = string(pw) + } + b, err := hex.DecodeString(s) + if err != nil || len(b) != 8 { + log.Fatal("Seed must be 16 hex characters") + } + var entropy [32]byte + copy(entropy[:], b) + return wallet.NewSeedFromEntropy(&entropy) +} + +func initTestnetClient(addr string, network string, seed wallet.Seed) *api.Client { + if network == "mainnet" { + log.Fatal("Testnet actions cannot be used on mainnet") + } + c := api.NewClient("http://"+addr+"/api", getAPIPassword()) + cs, err := c.ConsensusTipState() + check("Couldn't connect to API:", err) + if cs.Network.Name != network { + log.Fatalf("Testnet %q was specified, but walletd is running %v", network, cs.Network.Name) + } + ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) + wc := c.Wallet("primary") + if addrs, err := wc.Addresses(); err == nil && len(addrs) > 0 { + if _, ok := addrs[ourAddr]; !ok { + log.Fatal("Wallet already initialized with a different testnet address") + } + } + if ws, _ := c.Wallets(); len(ws) == 0 { + fmt.Print("Initializing testnet wallet...") + c.AddWallet("primary", nil) + if err := wc.AddAddress(ourAddr, nil); err != nil { + fmt.Println() + log.Fatal(err) + } else if err := wc.Subscribe(0); err != nil { + fmt.Println() + log.Fatal(err) + } + fmt.Println("done.") + } + return c +} + +func runTestnetMiner(c *api.Client, seed wallet.Seed) { + minerAddr := types.StandardUnlockHash(seed.PublicKey(0)) + log.Println("Started mining into", minerAddr) + start := time.Now() + + var hashes float64 + var blocks uint64 +outer: + for { + elapsed := time.Since(start) + cs, err := c.ConsensusTipState() + check("Couldn't get consensus tip state:", err) + n := big.NewInt(int64(hashes)) + n.Mul(n, big.NewInt(int64(24*time.Hour))) + d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) + d.Mul(d, big.NewInt(int64(elapsed))) + r, _ := new(big.Rat).SetFrac(n, d).Float64() + log.Printf("Mining...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) + + txns, v2txns, err := c.TxpoolTransactions() + check("Couldn't get txpool transactions:", err) + b := types.Block{ + ParentID: cs.Index.ID, + Nonce: cs.NonceFactor() * frand.Uint64n(100000), + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: cs.BlockReward()}}, + Transactions: txns, + } + for _, txn := range txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) + } + for _, txn := range v2txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) + } + if len(v2txns) > 0 || cs.Index.Height+1 >= cs.Network.HardforkV2.RequireHeight { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + Transactions: v2txns, + } + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + } + startBlock := time.Now() + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + // ensure nonce meets factor requirement + for b.Nonce%cs.NonceFactor() != 0 { + b.Nonce++ + } + hashes++ + if time.Since(startBlock) > 30*time.Second { + continue outer + } + } + blocks++ + index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} + tip, err := c.ConsensusTip() + check("Couldn't get consensus tip:", err) + if tip != cs.Index { + log.Printf("Mined %v but tip changed, starting over", index) + } else if err := c.SyncerBroadcastBlock(b); err != nil { + log.Println("Mined invalid block:", err) + } else if b.V2 == nil { + log.Printf("Found v1 block %v", index) + } else { + log.Printf("Found v2 block %v", index) + } + } +} + +func sendTestnet(c *api.Client, seed wallet.Seed, amount types.Currency, dest types.Address, v2 bool) { + ourKey := seed.PrivateKey(0) + ourUC := types.StandardUnlockConditions(seed.PublicKey(0)) + ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) + + cs, _ := c.ConsensusTipState() + utxos, _, err := c.Wallet("primary").Outputs() + if err != nil { + log.Fatal(err) + } + frand.Shuffle(len(utxos), reflect.Swapper(utxos)) + var inputSum types.Currency + rem := utxos[:0] + for _, utxo := range utxos { + if inputSum.Cmp(amount) >= 0 { + break + } else if cs.Index.Height > utxo.MaturityHeight { + rem = append(rem, utxo) + inputSum = inputSum.Add(utxo.SiacoinOutput.Value) + } + } + utxos = rem + if inputSum.Cmp(amount) < 0 { + log.Fatal("Insufficient balance") + } + outputs := []types.SiacoinOutput{ + {Address: dest, Value: amount}, + } + minerFee := inputSum.Sub(amount) + if maxFee := types.Siacoins(1); minerFee.Cmp(maxFee) > 0 { + minerFee = maxFee + } + if change := inputSum.Sub(amount.Add(minerFee)); !change.IsZero() { + outputs = append(outputs, types.SiacoinOutput{ + Address: ourAddr, + Value: change, + }) + } + + if v2 { + txn := types.V2Transaction{ + SiacoinInputs: make([]types.V2SiacoinInput, len(utxos)), + SiacoinOutputs: outputs, + MinerFee: minerFee, + } + for i, sce := range utxos { + txn.SiacoinInputs[i].Parent = sce + txn.SiacoinInputs[i].SatisfiedPolicy.Policy = types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(ourUC), + } + } + sigHash := cs.InputSigHash(txn) + for i := range utxos { + txn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{ourKey.SignHash(sigHash)} + } + if err := c.TxpoolBroadcast(nil, []types.V2Transaction{txn}); err != nil { + log.Fatal(err) + } + log.Println("Broadcast", txn.ID(), "successfully") + } else { + txn := types.Transaction{ + SiacoinInputs: make([]types.SiacoinInput, len(utxos)), + SiacoinOutputs: outputs, + Signatures: make([]types.TransactionSignature, len(utxos)), + } + if !minerFee.IsZero() { + txn.MinerFees = append(txn.MinerFees, minerFee) + } + for i, sce := range utxos { + txn.SiacoinInputs[i] = types.SiacoinInput{ + ParentID: types.SiacoinOutputID(sce.ID), + UnlockConditions: ourUC, + } + } + cs, _ := c.ConsensusTipState() + for i, sce := range utxos { + txn.Signatures[i] = wallet.StandardTransactionSignature(sce.ID) + wallet.SignTransaction(cs, &txn, i, ourKey) + } + if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { + log.Fatal(err) + } + log.Println("Broadcast", txn.ID(), "successfully") + } +} + +func printTestnetEvents(seed wallet.Seed, events []wallet.Event) { + ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) + for _, e := range events { + switch t := e.Val.(type) { + case *wallet.EventTransaction: + if len(t.SiafundInputs) == 0 || len(t.SiacoinOutputs) == 0 { + continue + } + sco := t.SiacoinOutputs[0].SiacoinOutput + sci := t.SiacoinInputs[0].SiacoinOutput + if sci.Address == ourAddr { + fmt.Printf("%v: Sent %v (+ %v fee) to %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) + } else { + fmt.Printf("%v: Received %v from %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) + } + case *wallet.EventMinerPayout: + sco := t.SiacoinOutput.SiacoinOutput + fmt.Printf("%v: Earned %v miner payout from block %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) + } + } +} diff --git a/go.mod b/go.mod index e912807..0e8add0 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.12.0 golang.org/x/term v0.6.0 + lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 ) diff --git a/go.sum b/go.sum index cbfab16..ef73c41 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= +lukechampine.com/flagg v1.1.1/go.mod h1:a9ZuZu5LSPXELWSJrabRD00ort+lDXSOQu34xWgEoDI= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= lukechampine.com/upnp v0.3.0 h1:UVCD6eD6fmJmwak6DVE3vGN+L46Fk8edTcC6XYCb6C4= From c2e5f1518dc389b378a0f56ab910f4ddf58716df Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 29 Nov 2023 12:15:12 -0500 Subject: [PATCH 002/630] main: Fix mining divide-by-zero --- cmd/walletd/testnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 00100c5..3496478 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -136,7 +136,7 @@ outer: n := big.NewInt(int64(hashes)) n.Mul(n, big.NewInt(int64(24*time.Hour))) d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) - d.Mul(d, big.NewInt(int64(elapsed))) + d.Mul(d, big.NewInt(int64(1+elapsed))) r, _ := new(big.Rat).SetFrac(n, d).Float64() log.Printf("Mining...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) From f662fd57bd7e16ffcb652665b28d097c10deb26d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 29 Nov 2023 12:13:59 -0800 Subject: [PATCH 003/630] ci: publish anagami --- .github/actions/test/action.yml | 10 +++++----- .github/workflows/publish.yml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index c44a2c0..de33d2c 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -11,11 +11,11 @@ runs: # uses: golangci/golangci-lint-action@v3 # with: # skip-cache: true - - name: Analyze - uses: SiaFoundation/action-golang-analysis@HEAD - with: - analyzers: | - go.sia.tech/jape.Analyzer +# - name: Analyze +# uses: SiaFoundation/action-golang-analysis@HEAD +# with: +# analyzers: | +# go.sia.tech/jape.Analyzer - name: Test uses: n8maninger/action-golang-test@v1 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e912f45..12c19cb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,6 +6,7 @@ on: push: branches: - master + - its-happening tags: - 'v[0-9]+.[0-9]+.[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+-**' From c29bcb7558bbb95e56cece9910a247b85b3e8695 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 30 Nov 2023 23:32:54 -0500 Subject: [PATCH 004/630] main: Add anagami bootstrap peers --- cmd/walletd/node.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 97d8229..991b101 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -60,6 +60,28 @@ var zenBootstrap = []string{ var anagamiBootstrap = []string{ "147.135.16.182:9781", + "98.180.237.163:9981", + "98.180.237.163:11981", + "98.180.237.163:10981", + "94.130.139.59:9801", + "84.86.11.238:9801", + "69.131.14.86:9981", + "68.108.89.92:9981", + "62.30.63.93:9981", + "46.173.150.154:9111", + "195.252.198.117:9981", + "185.65.135.189:9981", + "185.213.154.206:9981", + "185.204.1.222:9981", + "174.174.206.214:9981", + "172.58.232.54:9981", + "172.58.229.31:9981", + "172.56.200.90:9981", + "172.56.162.155:9981", + "163.172.13.180:9981", + "154.47.25.194:9981", + "138.201.19.49:9981", + "100.34.20.44:9981", } type boltDB struct { From d7f8b5abbccaf01be030957430bb369eba38607c Mon Sep 17 00:00:00 2001 From: lukechampine Date: Fri, 1 Dec 2023 10:24:06 -0500 Subject: [PATCH 005/630] main: Fix txns display bug --- cmd/walletd/testnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 3496478..56a259e 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -281,7 +281,7 @@ func printTestnetEvents(seed wallet.Seed, events []wallet.Event) { for _, e := range events { switch t := e.Val.(type) { case *wallet.EventTransaction: - if len(t.SiafundInputs) == 0 || len(t.SiacoinOutputs) == 0 { + if len(t.SiacoinInputs) == 0 || len(t.SiacoinOutputs) == 0 { continue } sco := t.SiacoinOutputs[0].SiacoinOutput From bf4a26d346f8b6c24c39b00b961c4c174577ca71 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 3 Dec 2023 19:52:43 -0500 Subject: [PATCH 006/630] syncer: Initialize strikes map --- syncer/syncer.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index ec1597b..65a0b00 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -910,13 +910,14 @@ func New(l net.Listener, cm ChainManager, pm PeerStore, header gateway.Header, o opt(&config) } return &Syncer{ - l: l, - cm: cm, - pm: pm, - header: header, - config: config, - log: config.Logger, - peers: make(map[string]*gateway.Peer), - synced: make(map[string]bool), + l: l, + cm: cm, + pm: pm, + header: header, + config: config, + log: config.Logger, + peers: make(map[string]*gateway.Peer), + synced: make(map[string]bool), + strikes: make(map[string]int), } } From db00172e7f626f8392544c774a2c1ba20fbe90ec Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 3 Dec 2023 23:05:05 -0500 Subject: [PATCH 007/630] main: Add txpool command --- cmd/walletd/main.go | 21 ++++++++++++++--- cmd/walletd/testnet.go | 51 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 24aaecf..fb03ce8 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -108,6 +108,12 @@ Sends a simple testnet transaction. walletd txns Lists testnet transactions and miner rewards. +` + txpoolUsage = `Usage: + walletd txpool + +Lists unconfirmed testnet transactions in the txpool. +Note that only transactions relevant to the wallet are shown. ` ) @@ -132,6 +138,7 @@ func main() { sendCmd := flagg.New("send", sendUsage) sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") txnsCmd := flagg.New("txns", txnsUsage) + txpoolCmd := flagg.New("txpool", txpoolUsage) cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, @@ -142,6 +149,7 @@ func main() { {Cmd: balanceCmd}, {Cmd: sendCmd}, {Cmd: txnsCmd}, + {Cmd: txpoolCmd}, }, }) @@ -231,8 +239,15 @@ func main() { } seed := loadTestnetSeed(seed) c := initTestnetClient(apiAddr, network, seed) - events, err := c.Wallet("primary").Events(0, -1) - check("Couldn't get events:", err) - printTestnetEvents(seed, events) + printTestnetEvents(c, seed) + + case txpoolCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + seed := loadTestnetSeed(seed) + c := initTestnetClient(apiAddr, network, seed) + printTestnetTxpool(c, seed) } } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 56a259e..4965ab5 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -276,24 +276,63 @@ func sendTestnet(c *api.Client, seed wallet.Seed, amount types.Currency, dest ty } } -func printTestnetEvents(seed wallet.Seed, events []wallet.Event) { +func printTestnetEvents(c *api.Client, seed wallet.Seed) { ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - for _, e := range events { + events, err := c.Wallet("primary").Events(0, -1) + check("Couldn't get events:", err) + for i := range events { + e := events[len(events)-1-i] switch t := e.Val.(type) { case *wallet.EventTransaction: if len(t.SiacoinInputs) == 0 || len(t.SiacoinOutputs) == 0 { continue } - sco := t.SiacoinOutputs[0].SiacoinOutput sci := t.SiacoinInputs[0].SiacoinOutput + sco := t.SiacoinOutputs[0].SiacoinOutput if sci.Address == ourAddr { - fmt.Printf("%v: Sent %v (+ %v fee) to %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) + fmt.Printf("%v (%v): Sent %v (+ %v fee) to %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) } else { - fmt.Printf("%v: Received %v from %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) + fmt.Printf("%v (%v): Received %v from %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) } case *wallet.EventMinerPayout: sco := t.SiacoinOutput.SiacoinOutput - fmt.Printf("%v: Earned %v miner payout from block %v\n", e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) + fmt.Printf("%v (%v): Earned %v miner payout from block %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) + } + } +} + +func printTestnetTxpool(c *api.Client, seed wallet.Seed) { + ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) + txns, v2txns, err := c.TxpoolTransactions() + check("Couldn't get txpool transactions:", err) + if len(txns) == 0 && len(v2txns) == 0 { + fmt.Println("No transactions in txpool.") + return + } + for _, txn := range txns { + if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { + continue + } + id := txn.ID() + sci := txn.SiacoinInputs[0] + sco := txn.SiacoinOutputs[0] + if sci.UnlockConditions.UnlockHash() == ourAddr { + fmt.Printf("%x (v1): Sending %v (+ %v fee) to %v\n", id[:4], sco.Value, txn.TotalFees(), sco.Address) + } else if sco.Address == ourAddr { + fmt.Printf("%x (v1): Receiving %v from %v\n", id[:4], sco.Value, sci.UnlockConditions.UnlockHash()) + } + } + for _, txn := range v2txns { + if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { + continue + } + id := txn.ID() + sci := txn.SiacoinInputs[0].Parent.SiacoinOutput + sco := txn.SiacoinOutputs[0] + if sci.Address == ourAddr { + fmt.Printf("%x (v2): Sending %v (+ %v fee) to %v\n", id[:4], sco.Value, txn.MinerFee, sco.Address) + } else if sco.Address == ourAddr { + fmt.Printf("%x (v2): Receiving %v from %v\n", id[:4], sco.Value, sci.Address) } } } From 91f2d6b2babdad889a8cb9912b4f8e67b874e4dc Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 3 Dec 2023 23:05:31 -0500 Subject: [PATCH 008/630] main: Don't double-spend in send command --- cmd/walletd/testnet.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 4965ab5..f65b2be 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -195,18 +195,33 @@ func sendTestnet(c *api.Client, seed wallet.Seed, amount types.Currency, dest ty ourUC := types.StandardUnlockConditions(seed.PublicKey(0)) ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - cs, _ := c.ConsensusTipState() + cs, err := c.ConsensusTipState() + check("Couldn't get consensus tip state:", err) utxos, _, err := c.Wallet("primary").Outputs() + check("Couldn't get outputs:", err) + txns, v2txns, err := c.TxpoolTransactions() if err != nil { log.Fatal(err) } + inPool := make(map[types.Hash256]bool) + for _, ptxn := range txns { + for _, in := range ptxn.SiacoinInputs { + inPool[types.Hash256(in.ParentID)] = true + } + } + for _, ptxn := range v2txns { + for _, in := range ptxn.SiacoinInputs { + inPool[in.Parent.ID] = true + } + } + frand.Shuffle(len(utxos), reflect.Swapper(utxos)) var inputSum types.Currency rem := utxos[:0] for _, utxo := range utxos { if inputSum.Cmp(amount) >= 0 { break - } else if cs.Index.Height > utxo.MaturityHeight { + } else if cs.Index.Height > utxo.MaturityHeight && !inPool[utxo.ID] { rem = append(rem, utxo) inputSum = inputSum.Add(utxo.SiacoinOutput.Value) } From 351e71d27d34849773e7de84373547e66f5ff75d Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 4 Dec 2023 10:58:05 -0500 Subject: [PATCH 009/630] syncer: Check addBlocks err in v2 sync --- syncer/syncer.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index 65a0b00..68cf7fa 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -338,7 +338,7 @@ func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Pee func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { if _, ok := h.s.cm.Block(bo.ParentID); !ok { - h.s.log.Printf("peer %v relayed a header with unknown parent (%v); triggering a resync", origin, bo.ParentID) + h.s.log.Printf("peer %v relayed a v2 outline with unknown parent (%v); triggering a resync", origin, bo.ParentID) h.s.mu.Lock() h.s.synced[origin.Addr] = false h.s.mu.Unlock() @@ -352,13 +352,13 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate } else if bo.ParentID != cs.Index.ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) + h.s.log.Printf("peer %v relayed a v2 outline that does not attach to our tip; triggering a resync", origin) h.s.mu.Lock() h.s.synced[origin.Addr] = false h.s.mu.Unlock() return } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) + h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) return } @@ -747,7 +747,7 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { blocks, rem, err := p.SendV2Blocks(history, s.config.MaxSendBlocks, s.config.SendBlocksTimeout) if err != nil { return err - } else if addBlocks(blocks); err != nil { + } else if err := addBlocks(blocks); err != nil { return err } else if rem == 0 { return nil From 87ad61ce0ae63b3e62878f47ff27aea16285f9a1 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 4 Dec 2023 17:30:24 -0500 Subject: [PATCH 010/630] mod: Update core dependency --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0e8add0..4008ad6 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 + go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3 go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.12.0 + go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 diff --git a/go.sum b/go.sum index ef73c41..8165738 100644 --- a/go.sum +++ b/go.sum @@ -7,16 +7,16 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 h1:2nKOKa99g9h9m3hL5UortAbmnwuwXhDcTHIhzmqBae8= -go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3 h1:W66DLFVMj1jnZPohpkeG4wulTKpx3kCttCRRTPjz+H0= +go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.12.0 h1:8eq5228RJQ5+MOiLLj0+0xVkkNvDHajfC816BUxdBh0= -go.sia.tech/web/walletd v0.12.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.10.0 h1:DomJMDo9hpbky8vtqp3HpATeOWejwIRxsdn1xKoaiJ8= +go.sia.tech/web/walletd v0.10.0/go.mod h1:zfiPJGTwHjYyYGJNhjYTFn3OSJPPQkVL4nXp1M/Lhmg= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= From 3f770690cb71c5d89b9b06f68f5943832bec0bf4 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 6 Dec 2023 14:41:38 -0500 Subject: [PATCH 011/630] syncer: Fix Subnet normalization and log bans --- internal/syncerutil/store.go | 10 +++++----- syncer/syncer.go | 15 ++++++++------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go index a486291..40ff1a1 100644 --- a/internal/syncerutil/store.go +++ b/internal/syncerutil/store.go @@ -28,11 +28,11 @@ func (eps *EphemeralPeerStore) banned(peer string) bool { return false // shouldn't happen } for _, s := range []string{ - peer, // 1.2.3.4:5678 - syncer.Subnet(host + "/32"), // 1.2.3.4:* - syncer.Subnet(host + "/24"), // 1.2.3.* - syncer.Subnet(host + "/16"), // 1.2.* - syncer.Subnet(host + "/8"), // 1.* + peer, // 1.2.3.4:5678 + syncer.Subnet(host, "/32"), // 1.2.3.4:* + syncer.Subnet(host, "/24"), // 1.2.3.* + syncer.Subnet(host, "/16"), // 1.2.* + syncer.Subnet(host, "/8"), // 1.* } { if b, ok := eps.bans[s]; ok { if time.Until(b.Expiry) <= 0 { diff --git a/syncer/syncer.go b/syncer/syncer.go index 68cf7fa..195e204 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -55,12 +55,12 @@ type PeerStore interface { } // Subnet normalizes the provided CIDR subnet string. -func Subnet(cidr string) string { - ip, ipnet, err := net.ParseCIDR(cidr) +func Subnet(addr, mask string) string { + ip, ipnet, err := net.ParseCIDR(addr + mask) if err != nil { return "" // shouldn't happen } - return ip.Mask(ipnet.Mask).String() + cidr + return ip.Mask(ipnet.Mask).String() + mask } type config struct { @@ -419,6 +419,7 @@ add: } func (s *Syncer) ban(p *gateway.Peer, err error) { + s.log.Printf("banning %v: %v", p, err) p.SetErr(errors.New("banned")) s.pm.Ban(p.ConnAddr, 24*time.Hour, err.Error()) @@ -428,10 +429,10 @@ func (s *Syncer) ban(p *gateway.Peer, err error) { } // add a strike to each subnet for subnet, maxStrikes := range map[string]int{ - Subnet(host + "/32"): 2, // 1.2.3.4:* - Subnet(host + "/24"): 8, // 1.2.3.* - Subnet(host + "/16"): 64, // 1.2.* - Subnet(host + "/8"): 512, // 1.* + Subnet(host, "/32"): 2, // 1.2.3.4:* + Subnet(host, "/24"): 8, // 1.2.3.* + Subnet(host, "/16"): 64, // 1.2.* + Subnet(host, "/8"): 512, // 1.* } { s.mu.Lock() ban := (s.strikes[subnet] + 1) >= maxStrikes From 8ea4b58b1ba7129e10f7e607330e064122d47b9e Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 10 Dec 2023 18:54:43 -0500 Subject: [PATCH 012/630] main: Update bootstrap list --- cmd/walletd/node.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 991b101..4d7e232 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -70,9 +70,6 @@ var anagamiBootstrap = []string{ "62.30.63.93:9981", "46.173.150.154:9111", "195.252.198.117:9981", - "185.65.135.189:9981", - "185.213.154.206:9981", - "185.204.1.222:9981", "174.174.206.214:9981", "172.58.232.54:9981", "172.58.229.31:9981", From 650510b02ee74ebd4018eef37a2d98a8bd2501a5 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 10 Dec 2023 18:58:15 -0500 Subject: [PATCH 013/630] walletutil: Don't ignore subscription errors --- internal/walletutil/store.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go index 0340464..0ef2357 100644 --- a/internal/walletutil/store.go +++ b/internal/walletutil/store.go @@ -363,17 +363,16 @@ func (s *JSONStore) load() error { // ProcessChainApplyUpdate implements chain.Subscriber. func (s *JSONStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { - s.EphemeralStore.ProcessChainApplyUpdate(cau, mayCommit) - if mayCommit { - return s.save() + err := s.EphemeralStore.ProcessChainApplyUpdate(cau, mayCommit) + if err == nil && mayCommit { + err = s.save() } - return nil + return err } // ProcessChainRevertUpdate implements chain.Subscriber. func (s *JSONStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - s.EphemeralStore.ProcessChainRevertUpdate(cru) - return nil + return s.EphemeralStore.ProcessChainRevertUpdate(cru) } // AddAddress implements api.Wallet. From 0bba38c7d86a7376e8427f5af369bf37cf7cb7e8 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 10 Dec 2023 18:59:44 -0500 Subject: [PATCH 014/630] syncer: Tweak logs --- syncer/syncer.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index 195e204..162768f 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -323,7 +323,7 @@ func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Pee h.s.mu.Unlock() return } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) + h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) return } @@ -725,10 +725,12 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { oldTime := time.Now() lastPrint := time.Now() startTime, startHeight := oldTime, oldTip.Height + var sentBlocks uint64 addBlocks := func(blocks []types.Block) error { if err := s.cm.AddBlocks(blocks); err != nil { return err } + sentBlocks += uint64(len(blocks)) endTime, endHeight := time.Now(), s.cm.Tip().Height s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { info.SyncedBlocks += endHeight - startHeight @@ -765,7 +767,7 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { } else if newTip := s.cm.Tip(); newTip != oldTip { s.log.Printf("finished syncing %v blocks with %v, tip now %v", totalBlocks, p, newTip) } else { - s.log.Printf("finished syncing with %v, tip unchanged", p) + s.log.Printf("finished syncing %v blocks with %v, tip unchanged", sentBlocks, p) } } } From f477395821e50f00f5cda02e89bc785977e28324 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 11 Dec 2023 09:17:23 -0500 Subject: [PATCH 015/630] main: Log to walletd.log by default --- cmd/walletd/node.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 4d7e232..d40c8a4 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -3,8 +3,10 @@ package main import ( "context" "errors" + "io" "log" "net" + "os" "path/filepath" "strconv" "time" @@ -226,7 +228,12 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Default())) + logFile, err := os.OpenFile(filepath.Join(dir, "walletd.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + log.Fatal(err) + } + logger := log.New(io.MultiWriter(os.Stderr, logFile), "", log.LstdFlags) + s := syncer.New(l, cm, ps, header, syncer.WithLogger(logger)) wm, err := walletutil.NewJSONWalletManager(dir, cm) if err != nil { From b08e34ea538b5728d8d35354bec83342458c84fc Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 11 Dec 2023 09:17:50 -0500 Subject: [PATCH 016/630] main: 2nd Anagami era --- cmd/walletd/testnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index f65b2be..c2bda51 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -38,7 +38,7 @@ func TestnetAnagami() (*consensus.Network, types.Block) { n.HardforkOak.Height = 5 n.HardforkOak.FixHeight = 8 - n.HardforkOak.GenesisTimestamp = time.Unix(1697100000, 0) // Oct 12, 2023 @ 08:40 GMT + n.HardforkOak.GenesisTimestamp = time.Unix(1702300000, 0) // Dec 11, 2023 @ 13:06 GMT n.HardforkASIC.Height = 13 n.HardforkASIC.OakTime = 10 * time.Minute From 7667515425f2d3635a5cb2c040ce1ce4522f4ae8 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 11 Dec 2023 12:52:30 -0500 Subject: [PATCH 017/630] main: Add immature and unconfirmed to balance cmd --- api/api.go | 5 +++-- api/server.go | 14 ++++++++++---- cmd/walletd/main.go | 14 +++++++++++++- cmd/walletd/testnet.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/api/api.go b/api/api.go index d894469..f5cb51b 100644 --- a/api/api.go +++ b/api/api.go @@ -32,8 +32,9 @@ type TxpoolTransactionsResponse struct { // WalletBalanceResponse is the response type for /wallets/:name/balance. type WalletBalanceResponse struct { - Siacoins types.Currency `json:"siacoins"` - Siafunds uint64 `json:"siafunds"` + Siacoins types.Currency `json:"siacoins"` + ImmatureSiacoins types.Currency `json:"immatureSiacoins"` + Siafunds uint64 `json:"siafunds"` } // WalletOutputsResponse is the response type for /wallets/:name/outputs. diff --git a/api/server.go b/api/server.go index 83321ea..46e770a 100644 --- a/api/server.go +++ b/api/server.go @@ -236,17 +236,23 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { if jc.Check("couldn't load outputs", err) != nil { return } - var sc types.Currency + height := s.cm.TipState().Index.Height + var sc, immature types.Currency var sf uint64 for _, sco := range scos { - sc = sc.Add(sco.SiacoinOutput.Value) + if height >= sco.MaturityHeight { + sc = sc.Add(sco.SiacoinOutput.Value) + } else { + immature = immature.Add(sco.SiacoinOutput.Value) + } } for _, sfo := range sfos { sf += sfo.SiafundOutput.Value } jc.Encode(WalletBalanceResponse{ - Siacoins: sc, - Siafunds: sf, + Siacoins: sc, + ImmatureSiacoins: immature, + Siafunds: sf, }) } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index fb03ce8..ec506c4 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -217,7 +217,19 @@ func main() { c := initTestnetClient(apiAddr, network, seed) b, err := c.Wallet("primary").Balance() check("Couldn't get balance:", err) - fmt.Println(b.Siacoins) + out := fmt.Sprint(b.Siacoins) + if !b.ImmatureSiacoins.IsZero() { + out += fmt.Sprintf(" + %v immature", b.ImmatureSiacoins) + } + poolGained, poolLost := testnetTxpoolBalance(c, seed) + if !poolGained.IsZero() || !poolLost.IsZero() { + if poolGained.Cmp(poolLost) >= 0 { + out += fmt.Sprintf(" + %v unconfirmed", poolGained.Sub(poolLost)) + } else { + out += fmt.Sprintf(" - %v unconfirmed", poolLost.Sub(poolGained)) + } + } + fmt.Println(out) case sendCmd: if len(cmd.Args()) != 2 { diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index c2bda51..d19e667 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -316,6 +316,35 @@ func printTestnetEvents(c *api.Client, seed wallet.Seed) { } } +func testnetTxpoolBalance(c *api.Client, seed wallet.Seed) (gained, lost types.Currency) { + ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) + txns, v2txns, err := c.TxpoolTransactions() + check("Couldn't get txpool transactions:", err) + for _, txn := range txns { + if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { + continue + } + sco := txn.SiacoinOutputs[0] + if txn.SiacoinInputs[0].UnlockConditions.UnlockHash() == ourAddr { + lost = lost.Add(sco.Value) + } else if sco.Address == ourAddr { + gained = gained.Add(sco.Value) + } + } + for _, txn := range v2txns { + if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { + continue + } + sco := txn.SiacoinOutputs[0] + if txn.SiacoinInputs[0].Parent.SiacoinOutput.Address == ourAddr { + lost = lost.Add(sco.Value) + } else if sco.Address == ourAddr { + gained = gained.Add(sco.Value) + } + } + return +} + func printTestnetTxpool(c *api.Client, seed wallet.Seed) { ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) txns, v2txns, err := c.TxpoolTransactions() From be1716001b4e7db7ae2edf4eb5bd5b08df42b8c2 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 11 Dec 2023 19:05:42 -0500 Subject: [PATCH 018/630] main: Include txn fees in unconfirmed balance --- cmd/walletd/testnet.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index d19e667..8d477cd 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -326,7 +326,7 @@ func testnetTxpoolBalance(c *api.Client, seed wallet.Seed) (gained, lost types.C } sco := txn.SiacoinOutputs[0] if txn.SiacoinInputs[0].UnlockConditions.UnlockHash() == ourAddr { - lost = lost.Add(sco.Value) + lost = lost.Add(sco.Value).Add(txn.TotalFees()) } else if sco.Address == ourAddr { gained = gained.Add(sco.Value) } @@ -337,7 +337,7 @@ func testnetTxpoolBalance(c *api.Client, seed wallet.Seed) (gained, lost types.C } sco := txn.SiacoinOutputs[0] if txn.SiacoinInputs[0].Parent.SiacoinOutput.Address == ourAddr { - lost = lost.Add(sco.Value) + lost = lost.Add(sco.Value).Add(txn.MinerFee) } else if sco.Address == ourAddr { gained = gained.Add(sco.Value) } From 200ab7e6a290c4ff0b72bc8e873481bf57f9c47a Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 12 Dec 2023 13:11:58 -0500 Subject: [PATCH 019/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4008ad6..3c9739d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3 + go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 8165738..22ae4b3 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3 h1:W66DLFVMj1jnZPohpkeG4wulTKpx3kCttCRRTPjz+H0= -go.sia.tech/core v0.1.12-0.20231204221602-d66e812ff1b3/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206 h1:tG5fk7fyx7KnSs172weYqn7ILNzwPFWukDbOsfwMt9k= +go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From da086c4c7eef6018d24e8881dd9c30f04733ccbc Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 12 Dec 2023 13:12:16 -0500 Subject: [PATCH 020/630] main: Automatically fixup consensus.db tree --- cmd/walletd/node.go | 1 + cmd/walletd/testnet.go | 58 ++++++++++++++++++++++++++++++++++ internal/walletutil/manager.go | 3 ++ 3 files changed, 62 insertions(+) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index d40c8a4..008c835 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -167,6 +167,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) case "anagami": network, genesisBlock = TestnetAnagami() bootstrapPeers = anagamiBootstrap + testnetFixDBTree(dir) default: return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 8d477cd..82a6892 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -6,9 +6,12 @@ import ( "log" "math/big" "os" + "path/filepath" "reflect" "time" + bolt "go.etcd.io/bbolt" + "go.sia.tech/core/chain" "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/walletd/api" @@ -380,3 +383,58 @@ func printTestnetTxpool(c *api.Client, seed wallet.Seed) { } } } + +func testnetFixDBTree(dir string) { + if _, err := os.Stat(filepath.Join(dir, "consensus.db")); err != nil { + log.Fatal(err) + } + bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + if err != nil { + log.Fatal(err) + } + db := &boltDB{db: bdb} + defer db.Close() + if db.Bucket([]byte("tree-fix")) != nil { + return + } + + fmt.Print("Fixing consensus.db Merkle tree...") + + network, genesisBlock := TestnetAnagami() + dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) + if err != nil { + log.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + bdb2, err := bolt.Open(filepath.Join(dir, "consensus.db-fixed"), 0600, nil) + if err != nil { + log.Fatal(err) + } + db2 := &boltDB{db: bdb2} + defer db2.Close() + dbstore2, tipState2, err := chain.NewDBStore(db2, network, genesisBlock) + if err != nil { + log.Fatal(err) + } + cm2 := chain.NewManager(dbstore2, tipState2) + + for cm2.Tip() != cm.Tip() { + index, _ := cm.BestIndex(cm2.Tip().Height + 1) + b, _ := cm.Block(index.ID) + if err := cm2.AddBlocks([]types.Block{b}); err != nil { + log.Fatal(err) + } + } + + if _, err := db2.CreateBucket([]byte("tree-fix")); err != nil { + log.Fatal(err) + } else if err := db.Close(); err != nil { + log.Fatal(err) + } else if err := db2.Close(); err != nil { + log.Fatal(err) + } else if err := os.Rename(filepath.Join(dir, "consensus.db-fixed"), filepath.Join(dir, "consensus.db")); err != nil { + log.Fatal(err) + } + fmt.Println("done.") +} diff --git a/internal/walletutil/manager.go b/internal/walletutil/manager.go index 77b8d1f..185006b 100644 --- a/internal/walletutil/manager.go +++ b/internal/walletutil/manager.go @@ -244,6 +244,9 @@ func (wm *JSONWalletManager) AddWallet(name string, info json.RawMessage) error // update existing wallet mw.info = info return wm.save() + } else if _, err := os.Stat(filepath.Join(wm.dir, "wallets", name+".json")); err == nil { + // shouldn't happen in normal conditions + return errors.New("a wallet with that name already exists, but is absent from wallets.json") } store, _, err := NewJSONStore(filepath.Join(wm.dir, "wallets", name+".json")) if err != nil { From 8baeb7bb6aca986631fa0a04e0631674f978e2fc Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 12 Dec 2023 16:59:30 -0500 Subject: [PATCH 021/630] main: Less naive mining loop --- cmd/walletd/testnet.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 82a6892..7cbb1ba 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -1,6 +1,7 @@ package main import ( + "encoding/binary" "encoding/hex" "fmt" "log" @@ -165,14 +166,27 @@ outer: } b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) } + + buf := make([]byte, 32+8+8+32) + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) + if b.V2 != nil { + copy(buf[:32], "sia/id/block|") + copy(buf[48:], b.V2.Commitment[:]) + } else { + root := b.MerkleRoot() // NOTE: expensive! + copy(buf[:32], b.ParentID[:]) + copy(buf[48:], root[:]) + } startBlock := time.Now() - for b.ID().CmpWork(cs.ChildTarget) < 0 { + for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { b.Nonce += cs.NonceFactor() // ensure nonce meets factor requirement for b.Nonce%cs.NonceFactor() != 0 { b.Nonce++ } hashes++ + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) if time.Since(startBlock) > 30*time.Second { continue outer } @@ -308,13 +322,13 @@ func printTestnetEvents(c *api.Client, seed wallet.Seed) { sci := t.SiacoinInputs[0].SiacoinOutput sco := t.SiacoinOutputs[0].SiacoinOutput if sci.Address == ourAddr { - fmt.Printf("%v (%v): Sent %v (+ %v fee) to %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) + fmt.Printf("%14v (%v): Sent %v (+ %v fee) to %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) } else { - fmt.Printf("%v (%v): Received %v from %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) + fmt.Printf("%14v (%v): Received %v from %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) } case *wallet.EventMinerPayout: sco := t.SiacoinOutput.SiacoinOutput - fmt.Printf("%v (%v): Earned %v miner payout from block %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) + fmt.Printf("%14v (%v): Earned %v miner payout from block %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) } } } From c025901ad0279b61452ae3bf11e8184a27754baa Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 12 Dec 2023 17:22:10 -0500 Subject: [PATCH 022/630] main: More mining speedups --- cmd/walletd/testnet.go | 55 +++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 7cbb1ba..ab8c6c9 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -125,6 +125,31 @@ func initTestnetClient(addr string, network string, seed wallet.Seed) *api.Clien return c } +func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { + buf := make([]byte, 32+8+8+32) + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) + if b.V2 != nil { + copy(buf[:32], "sia/id/block|") + copy(buf[48:], b.V2.Commitment[:]) + } else { + root := b.MerkleRoot() + copy(buf[:32], b.ParentID[:]) + copy(buf[48:], root[:]) + } + factor := cs.NonceFactor() + startBlock := time.Now() + for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { + b.Nonce += factor + hashes++ + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + if time.Since(startBlock) > 10*time.Second { + return hashes, false + } + } + return hashes, true +} + func runTestnetMiner(c *api.Client, seed wallet.Seed) { minerAddr := types.StandardUnlockHash(seed.PublicKey(0)) log.Println("Started mining into", minerAddr) @@ -148,7 +173,7 @@ outer: check("Couldn't get txpool transactions:", err) b := types.Block{ ParentID: cs.Index.ID, - Nonce: cs.NonceFactor() * frand.Uint64n(100000), + Nonce: cs.NonceFactor() * frand.Uint64n(100), Timestamp: types.CurrentTimestamp(), MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: cs.BlockReward()}}, Transactions: txns, @@ -166,30 +191,10 @@ outer: } b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) } - - buf := make([]byte, 32+8+8+32) - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) - if b.V2 != nil { - copy(buf[:32], "sia/id/block|") - copy(buf[48:], b.V2.Commitment[:]) - } else { - root := b.MerkleRoot() // NOTE: expensive! - copy(buf[:32], b.ParentID[:]) - copy(buf[48:], root[:]) - } - startBlock := time.Now() - for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - // ensure nonce meets factor requirement - for b.Nonce%cs.NonceFactor() != 0 { - b.Nonce++ - } - hashes++ - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - if time.Since(startBlock) > 30*time.Second { - continue outer - } + h, ok := mineBlock(cs, &b) + hashes += float64(h) + if !ok { + continue outer } blocks++ index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} From e701126689e0e23a497e16d9b64e53c96f7320e5 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 12 Dec 2023 18:48:07 -0500 Subject: [PATCH 023/630] main: Remove harmful sanity check --- cmd/walletd/testnet.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index ab8c6c9..3d95eb6 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -404,9 +404,6 @@ func printTestnetTxpool(c *api.Client, seed wallet.Seed) { } func testnetFixDBTree(dir string) { - if _, err := os.Stat(filepath.Join(dir, "consensus.db")); err != nil { - log.Fatal(err) - } bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) if err != nil { log.Fatal(err) From 4453d86ddcc001e906005460d61d5cc0b71ea98d Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 14 Dec 2023 20:39:24 -0500 Subject: [PATCH 024/630] wallet: Fix relevant address handling --- wallet/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 1d788b5..ecc9754 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -265,7 +265,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f unique := relevant[:0] for _, addr := range relevant { if !seen[addr] { - relevant = append(relevant, addr) + unique = append(unique, addr) seen[addr] = true } } From 6bbbcf81e5b56b0e86c1302a26b94edc84b1c4f5 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Tue, 12 Dec 2023 13:36:45 -0500 Subject: [PATCH 025/630] ui: v0.15.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5855b34..1bfccc4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( go.etcd.io/bbolt v1.3.7 go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.14.0 + go.sia.tech/web/walletd v0.15.0 golang.org/x/term v0.6.0 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 diff --git a/go.sum b/go.sum index 6db5c74..ec5131c 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.14.0 h1:GcNDv5HrLoMPKB8LcGIMKEoare+zUTkvRUkYFmq5KTE= -go.sia.tech/web/walletd v0.14.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.15.0 h1:OlhY4603TjEYBGXT6YG5dBv81Rw3KIpSvqx9ONo9xDc= +go.sia.tech/web/walletd v0.15.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= From 2b1d123dc7843ae8451fef64268337987a451e11 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 17 Dec 2023 09:16:33 -0500 Subject: [PATCH 026/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c9739d..f480ca4 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206 + go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 22ae4b3..b6adc97 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206 h1:tG5fk7fyx7KnSs172weYqn7ILNzwPFWukDbOsfwMt9k= -go.sia.tech/core v0.1.12-0.20231212170807-3d9547b51206/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf h1:lf2rX8WiDqPqz0PGWFfs6/jMpS7yewYT5ORAKQ9C97g= +go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 755aa3b9bc87e083f513e465520afdf796ddb934 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 26 Dec 2023 21:35:38 -0500 Subject: [PATCH 027/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f480ca4..bd544fd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf + go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index b6adc97..29df26d 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf h1:lf2rX8WiDqPqz0PGWFfs6/jMpS7yewYT5ORAKQ9C97g= -go.sia.tech/core v0.1.12-0.20231217141231-562d4f3f50bf/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f h1:+FdBnJnXR6XAr6PiUMuwuBqSohWDnZ7cbqSElO2Wwc0= +go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From de2a05296586dbde9f1bcea43e2377249ecb0092 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 26 Dec 2023 21:40:01 -0500 Subject: [PATCH 028/630] main: Automatically fixup consensus.db tree (again) --- cmd/walletd/testnet.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 3d95eb6..5e51d79 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -410,7 +410,7 @@ func testnetFixDBTree(dir string) { } db := &boltDB{db: bdb} defer db.Close() - if db.Bucket([]byte("tree-fix")) != nil { + if db.Bucket([]byte("tree-fix-2")) != nil { return } @@ -436,14 +436,16 @@ func testnetFixDBTree(dir string) { cm2 := chain.NewManager(dbstore2, tipState2) for cm2.Tip() != cm.Tip() { + fmt.Printf("\rFixing consensus.db Merkle tree...%v/%v", cm2.Tip().Height, cm.Tip().Height) index, _ := cm.BestIndex(cm2.Tip().Height + 1) b, _ := cm.Block(index.ID) if err := cm2.AddBlocks([]types.Block{b}); err != nil { - log.Fatal(err) + break } } + fmt.Println() - if _, err := db2.CreateBucket([]byte("tree-fix")); err != nil { + if _, err := db2.CreateBucket([]byte("tree-fix-2")); err != nil { log.Fatal(err) } else if err := db.Close(); err != nil { log.Fatal(err) @@ -452,5 +454,10 @@ func testnetFixDBTree(dir string) { } else if err := os.Rename(filepath.Join(dir, "consensus.db-fixed"), filepath.Join(dir, "consensus.db")); err != nil { log.Fatal(err) } + + fmt.Print("Backing up old wallet state...") + os.Rename(filepath.Join(dir, "wallets.json"), filepath.Join(dir, "wallets.json-bck")) + os.Rename(filepath.Join(dir, "wallets"), filepath.Join(dir, "wallets-bck")) fmt.Println("done.") + fmt.Println("NOTE: Your wallet will resync automatically on first use; this may take a few seconds.") } From c563ff175802199418fe53a0de2d0bddaa078820 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 27 Dec 2023 14:47:46 -0500 Subject: [PATCH 029/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd544fd..bf9f87f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f + go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 29df26d..0181055 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f h1:+FdBnJnXR6XAr6PiUMuwuBqSohWDnZ7cbqSElO2Wwc0= -go.sia.tech/core v0.1.12-0.20231227020339-6db2904e192f/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f h1:oRjijJ7/Op9FREgWkrVwOMX+0ou9Ivdlh5mLvAdYZw4= +go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 0677269c714438ae558fe7f6fa0752ec573473a7 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 27 Dec 2023 14:48:41 -0500 Subject: [PATCH 030/630] main: Add checkdb command --- cmd/walletd/main.go | 9 ++++++ cmd/walletd/testnet.go | 70 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index ec506c4..594c0cc 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -139,6 +139,7 @@ func main() { sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") txnsCmd := flagg.New("txns", txnsUsage) txpoolCmd := flagg.New("txpool", txpoolUsage) + dbCheckCmd := flagg.New("checkdb", "check consensus.db for errors") cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, @@ -150,6 +151,7 @@ func main() { {Cmd: sendCmd}, {Cmd: txnsCmd}, {Cmd: txpoolCmd}, + {Cmd: dbCheckCmd}, }, }) @@ -261,5 +263,12 @@ func main() { seed := loadTestnetSeed(seed) c := initTestnetClient(apiAddr, network, seed) printTestnetTxpool(c, seed) + + case dbCheckCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + testnetCheckDB(dir) } } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 5e51d79..77b197a 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -112,8 +112,10 @@ func initTestnetClient(addr string, network string, seed wallet.Seed) *api.Clien } if ws, _ := c.Wallets(); len(ws) == 0 { fmt.Print("Initializing testnet wallet...") - c.AddWallet("primary", nil) - if err := wc.AddAddress(ourAddr, nil); err != nil { + if err := c.AddWallet("primary", nil); err != nil { + fmt.Println() + log.Fatal(err) + } else if err := wc.AddAddress(ourAddr, nil); err != nil { fmt.Println() log.Fatal(err) } else if err := wc.Subscribe(0); err != nil { @@ -157,17 +159,22 @@ func runTestnetMiner(c *api.Client, seed wallet.Seed) { var hashes float64 var blocks uint64 + var last types.ChainIndex outer: for { elapsed := time.Since(start) cs, err := c.ConsensusTipState() check("Couldn't get consensus tip state:", err) + if cs.Index == last { + fmt.Println("Tip now", cs.Index) + last = cs.Index + } n := big.NewInt(int64(hashes)) n.Mul(n, big.NewInt(int64(24*time.Hour))) d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) d.Mul(d, big.NewInt(int64(1+elapsed))) r, _ := new(big.Rat).SetFrac(n, d).Float64() - log.Printf("Mining...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) + fmt.Printf("\rMining block %4v...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", cs.Index.Height+1, hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) txns, v2txns, err := c.TxpoolTransactions() check("Couldn't get txpool transactions:", err) @@ -201,13 +208,13 @@ outer: tip, err := c.ConsensusTip() check("Couldn't get consensus tip:", err) if tip != cs.Index { - log.Printf("Mined %v but tip changed, starting over", index) + fmt.Printf("\nMined %v but tip changed, starting over\n", index) } else if err := c.SyncerBroadcastBlock(b); err != nil { - log.Println("Mined invalid block:", err) + fmt.Printf("\nMined invalid block: %v\n", err) } else if b.V2 == nil { - log.Printf("Found v1 block %v", index) + fmt.Printf("\nFound v1 block %v\n", index) } else { - log.Printf("Found v2 block %v", index) + fmt.Printf("\nFound v2 block %v\n", index) } } } @@ -456,8 +463,57 @@ func testnetFixDBTree(dir string) { } fmt.Print("Backing up old wallet state...") + os.RemoveAll(filepath.Join(dir, "wallets.json-bck")) os.Rename(filepath.Join(dir, "wallets.json"), filepath.Join(dir, "wallets.json-bck")) + os.RemoveAll(filepath.Join(dir, "wallets-bck")) os.Rename(filepath.Join(dir, "wallets"), filepath.Join(dir, "wallets-bck")) fmt.Println("done.") fmt.Println("NOTE: Your wallet will resync automatically on first use; this may take a few seconds.") } + +func testnetCheckDB(dir string) { + bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + if err != nil { + log.Fatal(err) + } + db := &boltDB{db: bdb} + defer db.Close() + + fmt.Print("Reapplying blocks...") + + network, genesisBlock := TestnetAnagami() + dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) + if err != nil { + log.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + dbstore2, tipState2, err := chain.NewDBStore(chain.NewMemDB(), network, genesisBlock) + if err != nil { + log.Fatal(err) + } + cm2 := chain.NewManager(dbstore2, tipState2) + + for cm2.Tip() != cm.Tip() { + fmt.Printf("\rReapplying blocks...%v/%v", cm2.Tip().Height, cm.Tip().Height) + index, _ := cm.BestIndex(cm2.Tip().Height + 1) + b, _ := cm.Block(index.ID) + if err := cm2.AddBlocks([]types.Block{b}); err != nil { + break + } + } + fmt.Println() + if cm.Tip() != cm2.Tip() { + fmt.Printf("Could not apply all blocks (%v/%v); marking consensus.db as corrupt\n", cm2.Tip().Height, cm.Tip().Height) + db.newTx() + db.tx.DeleteBucket([]byte("tree-fix-2")) + return + } + if cm.TipState().Commitment(types.Hash256{}, types.VoidAddress) != cm2.TipState().Commitment(types.Hash256{}, types.VoidAddress) { + fmt.Println("Final state differs from consensus.db; marking consensus.db as corrupt") + db.newTx() + db.tx.DeleteBucket([]byte("tree-fix-2")) + return + } + fmt.Println("No problems detected.") +} From 9772cb8696bc14e87a22cd496d146979279bfdb6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 28 Dec 2023 12:25:08 -0800 Subject: [PATCH 031/630] ci: remove test step from publish --- .github/workflows/publish.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12c19cb..546b4c4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,17 +12,6 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+-**' jobs: - test: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Test - uses: ./.github/actions/test docker: runs-on: ubuntu-latest needs: [ test ] From c298022eff554cce35d5bf9c6632d24375028e93 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 28 Dec 2023 12:26:15 -0800 Subject: [PATCH 032/630] ci: remove test step --- .github/workflows/publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 546b4c4..1b3cb03 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,6 @@ on: jobs: docker: runs-on: ubuntu-latest - needs: [ test ] permissions: packages: write contents: read @@ -44,7 +43,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} build-linux: runs-on: ubuntu-latest - needs: [ test ] steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 @@ -84,7 +82,6 @@ jobs: path: release/ build-mac: runs-on: macos-latest - needs: [ test ] steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 @@ -169,7 +166,6 @@ jobs: path: release/ build-windows: runs-on: windows-latest - needs: [ test ] steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 From e3244b3a391e207f66209d329d2b4ddd38a73043 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sat, 30 Dec 2023 01:16:45 -0500 Subject: [PATCH 033/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bf9f87f..f9c29c0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f + go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 0181055..eec3b91 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f h1:oRjijJ7/Op9FREgWkrVwOMX+0ou9Ivdlh5mLvAdYZw4= -go.sia.tech/core v0.1.12-0.20231227192623-093cc498401f/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827 h1:RoCf8gU2dwICUcP0IiWSL9bKoDOAQtrmcD5lhDLB6aI= +go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From e6393f0d56bc561603becb16d3ae571bb90fb66f Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 31 Dec 2023 00:31:58 -0500 Subject: [PATCH 034/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f9c29c0..54a71e8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827 + go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index eec3b91..e9a5252 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827 h1:RoCf8gU2dwICUcP0IiWSL9bKoDOAQtrmcD5lhDLB6aI= -go.sia.tech/core v0.1.12-0.20231230053358-7505ca1f7827/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9 h1:C2bBKgPcHhaGA93htmUZfPEPgeWqGY8LhI9/JYaTQN0= +go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From e3c12d22b84c7c8020a498c83c1fb1d6dfe26d2f Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 2 Jan 2024 17:53:51 -0500 Subject: [PATCH 035/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 54a71e8..a7c22ab 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9 + go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index e9a5252..9a99e00 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9 h1:C2bBKgPcHhaGA93htmUZfPEPgeWqGY8LhI9/JYaTQN0= -go.sia.tech/core v0.1.12-0.20231231053054-38ae2e7cf9b9/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05 h1:oMICBdw/0vVCf2wLvjQmP7uO5nkq9GY0eAlaS4181yA= +go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From b5ffd4e70e70e22980c44d4ddfdfbd65a7868ac7 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 3 Jan 2024 13:15:14 -0500 Subject: [PATCH 036/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a7c22ab..3094a62 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05 + go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 9a99e00..9fc8109 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05 h1:oMICBdw/0vVCf2wLvjQmP7uO5nkq9GY0eAlaS4181yA= go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7 h1:/3BeAUjphPhI4BT9jcnB3lDgqwkbajcR4Pg/WGj1lig= +go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From d962ecdf33179962f8ae68a834baffea6101d8ea Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 3 Jan 2024 14:04:44 -0500 Subject: [PATCH 037/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3094a62..f2daa47 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7 + go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 9fc8109..1f44c82 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05 h1:oMICBdw/0vVCf2wLvjQmP7uO5nkq9GY0eAlaS4181yA= -go.sia.tech/core v0.1.12-0.20240102225118-01d51483ce05/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= -go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7 h1:/3BeAUjphPhI4BT9jcnB3lDgqwkbajcR4Pg/WGj1lig= -go.sia.tech/core v0.1.12-0.20240103180928-a4b68d2633a7/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8 h1:R+GMKScY0+gb8smswxBgcgjCmB1QBBUqYLRK3vWAdgY= +go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From d304cf4177956266453ee8db9c17c7cff32f5ac1 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Wed, 3 Jan 2024 16:29:35 -0500 Subject: [PATCH 038/630] ui: ci action for updating ui --- .github/workflows/ui.yml | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/ui.yml diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml new file mode 100644 index 0000000..74dc4dc --- /dev/null +++ b/.github/workflows/ui.yml @@ -0,0 +1,71 @@ +name: Update UI + +on: + # Run daily + schedule: + - cron: '0 0 * * *' + # Enable manual trigger + workflow_dispatch: + +jobs: + update-ui: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '1.20.0' + + - name: Check for new walletd tag in SiaFoundation/web + id: check-tag + env: + GH_TOKEN: ${{ github.token }} + run: | + # Fetch tags with pagination + TAGS_JSON=$(gh api --paginate repos/SiaFoundation/web/tags) + + # Extract tags that start with "walletd/", sort them in version order, and pick the highest version + LATEST_WALLETD_GO_TAG=$(echo "$TAGS_JSON" | jq -r '.[] | select(.name | startswith("walletd/")).name' | sort -Vr | head -n 1) + LATEST_WALLETD_VERSION=$(echo "$LATEST_WALLETD_GO_TAG" | sed 's/walletd\///') + + echo "Latest walletd tag is $LATEST_WALLETD_GO_TAG" + echo "GO_TAG=$LATEST_WALLETD_GO_TAG" >> $GITHUB_ENV + echo "VERSION=$LATEST_WALLETD_VERSION" >> $GITHUB_ENV + + - name: Fetch release notes for the release + id: release-notes + env: + GH_TOKEN: ${{ github.token }} + if: env.GO_TAG != 'null' + run: | + RELEASE_TAG_FORMATTED=$(echo "$GO_TAG" | sed 's/\/v/@/') + RELEASES_JSON=$(gh api --paginate repos/SiaFoundation/web/releases) + + RELEASE_NOTES=$(echo "$RELEASES_JSON" | jq -r --arg TAG_NAME "$RELEASE_TAG_FORMATTED" '.[] | select(.name == $TAG_NAME).body') + echo "Release notes for $RELEASE_TAG_FORMATTED: $RELEASE_NOTES" + echo "RELEASE_NOTES<> $GITHUB_ENV + echo "$RELEASE_NOTES" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Update go.mod with latest module + if: env.GO_TAG != 'null' + run: | + GO_MODULE_FORMATTED=$(echo "$GO_TAG" | sed 's/\//@/') + echo "Updating go.mod to use $GO_MODULE_FORMATTED" + go clean -modcache + go get go.sia.tech/web/$GO_MODULE_FORMATTED + go mod tidy + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + if: env.GO_TAG != 'null' + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "ui: ${{ env.VERSION }}" + title: "ui: ${{ env.VERSION }}" + body: ${{ env.RELEASE_NOTES }} + branch: "ui/update" + delete-branch: true From d35d32bd912c0e06fa895cfcee02eeeaeaa4851c Mon Sep 17 00:00:00 2001 From: alexfreska Date: Wed, 3 Jan 2024 21:31:38 +0000 Subject: [PATCH 039/630] ui: v0.16.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1bfccc4..84ec484 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( go.etcd.io/bbolt v1.3.7 go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.15.0 + go.sia.tech/web/walletd v0.16.0 golang.org/x/term v0.6.0 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 diff --git a/go.sum b/go.sum index ec5131c..5b68cd2 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.15.0 h1:OlhY4603TjEYBGXT6YG5dBv81Rw3KIpSvqx9ONo9xDc= -go.sia.tech/web/walletd v0.15.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.16.0 h1:tCERgjsz4orokM94kt7PH2tNweHdOwK5aoPsCXes5HM= +go.sia.tech/web/walletd v0.16.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= From eb99414015e24c26f05bbf5a30cd268d48a6e3ff Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 3 Jan 2024 17:51:43 -0500 Subject: [PATCH 040/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2daa47..50ffa00 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8 + go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 1f44c82..a2d873a 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8 h1:R+GMKScY0+gb8smswxBgcgjCmB1QBBUqYLRK3vWAdgY= -go.sia.tech/core v0.1.12-0.20240103190338-62b60d69add8/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054 h1:+7OK1zJTCEy0tQmrlytA/vo0Xit3rlhQQx2x+hKMVUw= +go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 8d6da8892cc2692b19570f16b3e67c1998e6cd1d Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 3 Jan 2024 18:49:49 -0500 Subject: [PATCH 041/630] mod: Update core dependency --- api/server.go | 9 +++++---- go.mod | 2 +- go.sum | 4 ++-- syncer/syncer.go | 33 +++++++++++++++------------------ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/api/server.go b/api/server.go index 46e770a..c38c4d6 100644 --- a/api/server.go +++ b/api/server.go @@ -28,7 +28,7 @@ type ( PoolTransactions() []types.Transaction V2PoolTransactions() []types.V2Transaction AddPoolTransactions(txns []types.Transaction) error - AddV2PoolTransactions(txns []types.V2Transaction) error + AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) error UnconfirmedParents(txn types.Transaction) []types.Transaction } @@ -40,7 +40,7 @@ type ( Connect(addr string) (*gateway.Peer, error) BroadcastHeader(bh gateway.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) - BroadcastV2TransactionSet(txns []types.V2Transaction) + BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) BroadcastV2BlockOutline(bo gateway.V2BlockOutline) } @@ -154,10 +154,11 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { s.s.BroadcastTransactionSet(tbr.Transactions) } if len(tbr.V2Transactions) != 0 { - if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(tbr.V2Transactions)) != nil { + index := s.cm.TipState().Index + if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(index, tbr.V2Transactions)) != nil { return } - s.s.BroadcastV2TransactionSet(tbr.V2Transactions) + s.s.BroadcastV2TransactionSet(index, tbr.V2Transactions) } } diff --git a/go.mod b/go.mod index 50ffa00..75a4cb6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054 + go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index a2d873a..0b167f9 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054 h1:+7OK1zJTCEy0tQmrlytA/vo0Xit3rlhQQx2x+hKMVUw= -go.sia.tech/core v0.1.12-0.20240103201313-cb031ab57054/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029 h1:oYriMRrX0rSaDj9rvupmw2elGAnmjDxAE8zWVVFGuJg= +go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/syncer/syncer.go b/syncer/syncer.go index 162768f..ac842b2 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -29,7 +29,7 @@ type ChainManager interface { PoolTransaction(txid types.TransactionID) (types.Transaction, bool) AddPoolTransactions(txns []types.Transaction) error V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) - AddV2PoolTransactions(txns []types.V2Transaction) error + AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) error TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) } @@ -397,25 +397,22 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate h.s.relayV2BlockOutline(bo, origin) // non-blocking } -func (h *rpcHandler) RelayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { +func (h *rpcHandler) RelayV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { // if we've already seen these transactions, don't relay them again + allSeen := true for _, txn := range txns { - if _, ok := h.s.cm.V2PoolTransaction(txn.ID()); !ok { - goto add + if _, allSeen = h.s.cm.V2PoolTransaction(txn.ID()); !allSeen { + break } } - return - -add: - if err := h.s.cm.AddV2PoolTransactions(txns); err != nil { - // too risky to ban here (txns are probably just outdated), but at least - // log it if we think we're synced - if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - } + if allSeen { + return + } + if err := h.s.cm.AddV2PoolTransactions(index, txns); err != nil { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) return } - h.s.relayV2TransactionSet(txns, origin) // non-blocking + h.s.relayV2TransactionSet(index, txns, origin) // non-blocking } func (s *Syncer) ban(p *gateway.Peer, err error) { @@ -531,14 +528,14 @@ func (s *Syncer) relayV2BlockOutline(pb gateway.V2BlockOutline, origin *gateway. } } -func (s *Syncer) relayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { +func (s *Syncer) relayV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { s.mu.Lock() defer s.mu.Unlock() for _, p := range s.peers { if p == origin || !p.SupportsV2() { continue } - go p.RelayV2TransactionSet(txns, s.config.RelayTransactionSetTimeout) + go p.RelayV2TransactionSet(index, txns, s.config.RelayTransactionSetTimeout) } } @@ -862,8 +859,8 @@ func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2Bl func (s *Syncer) BroadcastTransactionSet(txns []types.Transaction) { s.relayTransactionSet(txns, nil) } // BroadcastV2TransactionSet broadcasts a v2 transaction set to all peers. -func (s *Syncer) BroadcastV2TransactionSet(txns []types.V2Transaction) { - s.relayV2TransactionSet(txns, nil) +func (s *Syncer) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) { + s.relayV2TransactionSet(index, txns, nil) } // Peers returns the set of currently-connected peers. From 6ea33bd82928c538812ded21afd25883635ec67d Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 4 Jan 2024 16:45:56 -0500 Subject: [PATCH 042/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- syncer/syncer.go | 15 +++++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 75a4cb6..0eb2c68 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029 + go.sia.tech/core v0.1.12-0.20240104213000-41097337c139 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 0b167f9..ae6c2a8 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029 h1:oYriMRrX0rSaDj9rvupmw2elGAnmjDxAE8zWVVFGuJg= -go.sia.tech/core v0.1.12-0.20240103234000-bae2b8fd3029/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240104213000-41097337c139 h1:7/cRTofOXmp150CvEiJQWsYZ+psmzn8bKmA2Jp7V5+0= +go.sia.tech/core v0.1.12-0.20240104213000-41097337c139/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/syncer/syncer.go b/syncer/syncer.go index ac842b2..5d7ec3f 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -29,7 +29,7 @@ type ChainManager interface { PoolTransaction(txid types.TransactionID) (types.Transaction, bool) AddPoolTransactions(txns []types.Transaction) error V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) - AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) error + AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2Transaction) error TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) } @@ -397,7 +397,7 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate h.s.relayV2BlockOutline(bo, origin) // non-blocking } -func (h *rpcHandler) RelayV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { +func (h *rpcHandler) RelayV2TransactionSet(basis types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { // if we've already seen these transactions, don't relay them again allSeen := true for _, txn := range txns { @@ -408,11 +408,18 @@ func (h *rpcHandler) RelayV2TransactionSet(index types.ChainIndex, txns []types. if allSeen { return } - if err := h.s.cm.AddV2PoolTransactions(index, txns); err != nil { + if _, ok := h.s.cm.Block(basis.ID); !ok { + h.s.log.Printf("peer %v relayed a v2 transaction set with unknown basis (%v); triggering a resync", origin, basis) + h.s.mu.Lock() + h.s.synced[origin.Addr] = false + h.s.mu.Unlock() + return + } + if err := h.s.cm.AddV2PoolTransactions(basis, txns); err != nil { h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) return } - h.s.relayV2TransactionSet(index, txns, origin) // non-blocking + h.s.relayV2TransactionSet(basis, txns, origin) // non-blocking } func (s *Syncer) ban(p *gateway.Peer, err error) { From 39d73b990f05997eb9bcc98971d7207dc5b35ecb Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 4 Jan 2024 18:55:27 -0500 Subject: [PATCH 043/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0eb2c68..fa08664 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240104213000-41097337c139 + go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index ae6c2a8..10181de 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240104213000-41097337c139 h1:7/cRTofOXmp150CvEiJQWsYZ+psmzn8bKmA2Jp7V5+0= -go.sia.tech/core v0.1.12-0.20240104213000-41097337c139/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569 h1:6URaowQhbzrNdI+WL95ElD+7fmKa9z25zqKekiGEiM8= +go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 5d749b279604290d92fb673699545a4a46a33050 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 4 Jan 2024 23:44:41 -0500 Subject: [PATCH 044/630] mod: Update core dependency --- cmd/walletd/multiproof.go | 270 ++++++++++++++++++++++++++++++++++++++ cmd/walletd/node.go | 1 + go.mod | 2 +- go.sum | 4 +- 4 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 cmd/walletd/multiproof.go diff --git a/cmd/walletd/multiproof.go b/cmd/walletd/multiproof.go new file mode 100644 index 0000000..45f1e0c --- /dev/null +++ b/cmd/walletd/multiproof.go @@ -0,0 +1,270 @@ +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "log" + "math/bits" + "path/filepath" + "sort" + + bolt "go.etcd.io/bbolt" + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" +) + +// copied from types/multiproof.go + +type elementLeaf struct { + *types.StateElement + ElementHash types.Hash256 +} + +func (l elementLeaf) hash() types.Hash256 { + buf := make([]byte, 1+32+8+1) + buf[0] = 0x00 // leafHashPrefix + copy(buf[1:], l.ElementHash[:]) + binary.LittleEndian.PutUint64(buf[33:], l.LeafIndex) + buf[41] = 0 // spent (always false for multiproofs) + return types.HashBytes(buf) +} + +func hashAll(elems ...interface{}) [32]byte { + h := types.NewHasher() + for _, e := range elems { + if et, ok := e.(types.EncoderTo); ok { + et.EncodeTo(h.E) + } else { + switch e := e.(type) { + case string: + h.WriteDistinguisher(e) + case uint64: + h.E.WriteUint64(e) + } + } + } + return h.Sum() +} + +func chainIndexLeaf(e *types.ChainIndexElement) elementLeaf { + return elementLeaf{&e.StateElement, hashAll("leaf/chainindex", e.ID, e.ChainIndex)} +} + +func siacoinLeaf(e *types.SiacoinElement) elementLeaf { + return elementLeaf{&e.StateElement, hashAll("leaf/siacoin", e.ID, e.SiacoinOutput, e.MaturityHeight)} +} + +func siafundLeaf(e *types.SiafundElement) elementLeaf { + return elementLeaf{&e.StateElement, hashAll("leaf/siafund", e.ID, e.SiafundOutput, e.ClaimStart)} +} + +func v2FileContractLeaf(e *types.V2FileContractElement) elementLeaf { + return elementLeaf{&e.StateElement, hashAll("leaf/v2filecontract", e.ID, e.V2FileContract)} +} + +func splitLeaves(ls []elementLeaf, mid uint64) (left, right []elementLeaf) { + split := sort.Search(len(ls), func(i int) bool { return ls[i].LeafIndex >= mid }) + return ls[:split], ls[split:] +} + +func forEachElementLeaf(txns []types.V2Transaction, fn func(l elementLeaf)) { + visit := func(l elementLeaf) { + if l.LeafIndex != types.EphemeralLeafIndex { + fn(l) + } + } + for _, txn := range txns { + for i := range txn.SiacoinInputs { + visit(siacoinLeaf(&txn.SiacoinInputs[i].Parent)) + } + for i := range txn.SiafundInputs { + visit(siafundLeaf(&txn.SiafundInputs[i].Parent)) + } + for i := range txn.FileContractRevisions { + visit(v2FileContractLeaf(&txn.FileContractRevisions[i].Parent)) + } + for i := range txn.FileContractResolutions { + visit(v2FileContractLeaf(&txn.FileContractResolutions[i].Parent)) + if r, ok := txn.FileContractResolutions[i].Resolution.(*types.V2StorageProof); ok { + visit(chainIndexLeaf(&r.ProofIndex)) + } + } + } +} + +func forEachTree(txns []types.V2Transaction, fn func(i, j uint64, leaves []elementLeaf)) { + clearBits := func(x uint64, n int) uint64 { return x &^ (1<= 64 { + d.SetErr(errors.New("invalid Merkle proof size")) + } + }) + if d.Err() != nil { + return + } + multiproof := make([]types.Hash256, multiproofSize(*txns)) + for i := range multiproof { + multiproof[i].DecodeFrom(d) + } + expandMultiproof(*txns, multiproof) +} + +func testnetFixMultiproofs(dir string) { + bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + if err != nil { + log.Fatal(err) + } + defer bdb.Close() + var needUpdate bool + bdb.Update(func(tx *bolt.Tx) error { + needUpdate = tx.Bucket([]byte("multiproof-fix")) == nil + return nil + }) + if !needUpdate { + return + } + + fmt.Println("Fixing consensus.db multiproofs...") + + type supplementedBlock struct { + Block types.Block + Supplement *consensus.V1BlockSupplement + } + + decodeBlock := func(v []byte) (sb supplementedBlock, err error) { + d := types.NewBufDecoder(v) + if v := d.ReadUint8(); v != 2 { + d.SetErr(fmt.Errorf("incompatible version (%d)", v)) + } + (*types.V1Block)(&sb.Block).DecodeFrom(d) + if d.ReadBool() { + sb.Block.V2 = new(types.V2BlockData) + sb.Block.V2.Height = d.ReadUint64() + sb.Block.V2.Commitment.DecodeFrom(d) + (*V2TransactionsMultiproof)(&sb.Block.V2.Transactions).DecodeFrom(d) + } + if d.ReadBool() { + sb.Supplement = new(consensus.V1BlockSupplement) + sb.Supplement.DecodeFrom(d) + } + err = d.Err() + return + } + encodeBlock := func(sb supplementedBlock) []byte { + var buf bytes.Buffer + e := types.NewEncoder(&buf) + e.WriteUint8(2) + (types.V2Block)(sb.Block).EncodeTo(e) + e.WriteBool(sb.Supplement != nil) + if sb.Supplement != nil { + sb.Supplement.EncodeTo(e) + } + e.Flush() + return buf.Bytes() + } + + err = bdb.Update(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte("Blocks")) + var keys []string + bucket.ForEach(func(k, v []byte) error { + keys = append(keys, string(k)) + return nil + }) + for _, k := range keys { + fmt.Printf("\r%x...", k) + b, err := decodeBlock(bucket.Get([]byte(k))) + if err != nil { + return err + } + if err := bucket.Put([]byte(k), encodeBlock(b)); err != nil { + return err + } + } + _, err := tx.CreateBucket([]byte("multiproof-fix")) + return err + }) + if err != nil { + fmt.Println() + log.Fatal(err) + } + fmt.Println("done.") +} diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 008c835..7000064 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -168,6 +168,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) network, genesisBlock = TestnetAnagami() bootstrapPeers = anagamiBootstrap testnetFixDBTree(dir) + testnetFixMultiproofs(dir) default: return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } diff --git a/go.mod b/go.mod index fa08664..caba7ce 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569 + go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 10181de..dbcb098 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569 h1:6URaowQhbzrNdI+WL95ElD+7fmKa9z25zqKekiGEiM8= -go.sia.tech/core v0.1.12-0.20240104235401-e42d6d5b2569/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774 h1:neyQGoGFqedOFgI0NcNPAdAv0A3lFvwthWm10Cw4qr8= +go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 9bbc5966696124058fc43101060f7162824ca6ca Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sat, 6 Jan 2024 11:46:24 -0500 Subject: [PATCH 045/630] syncer: Add resync helper --- syncer/syncer.go | 46 ++++++++++++++++++---------------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index 5d7ec3f..87b3211 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -3,6 +3,7 @@ package syncer import ( "context" "errors" + "fmt" "io" "log" "net" @@ -193,6 +194,16 @@ type rpcHandler struct { s *Syncer } +func (h *rpcHandler) resync(p *gateway.Peer, reason string) { + h.s.mu.Lock() + alreadyResyncing := !h.s.synced[p.Addr] + h.s.synced[p.Addr] = false + h.s.mu.Unlock() + if !alreadyResyncing { + h.s.log.Printf("triggering resync with %v: %v", p, reason) + } +} + func (h *rpcHandler) PeersForShare() (peers []string) { peers = h.s.pm.Peers() if len(peers) > 10 { @@ -249,18 +260,12 @@ func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { if _, ok := h.s.cm.Block(bh.ID()); ok { return // already seen } else if _, ok := h.s.cm.Block(bh.ParentID); !ok { - h.s.log.Printf("peer %v relayed a header with unknown parent (%v); triggering a resync", origin, bh.ParentID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, fmt.Sprintf("peer relayed a header with unknown parent (%v)", bh.ParentID)) return } else if cs := h.s.cm.TipState(); bh.ParentID != cs.Index.ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, "peer relayed a header that does not attach to our tip") return } else if bh.ID().CmpWork(cs.ChildTarget) < 0 { h.s.ban(origin, errors.New("peer sent header with insufficient work")) @@ -303,10 +308,7 @@ add: func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { if _, ok := h.s.cm.Block(bh.Parent.ID); !ok { - h.s.log.Printf("peer %v relayed a v2 header with unknown parent (%v); triggering a resync", origin, bh.Parent.ID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", bh.Parent.ID)) return } cs := h.s.cm.TipState() @@ -317,10 +319,7 @@ func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Pee } else if bh.Parent.ID != cs.Index.ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, "peer relayed a v2 header that does not attach to our tip") return } else if bid.CmpWork(cs.ChildTarget) < 0 { h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) @@ -338,10 +337,7 @@ func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Pee func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { if _, ok := h.s.cm.Block(bo.ParentID); !ok { - h.s.log.Printf("peer %v relayed a v2 outline with unknown parent (%v); triggering a resync", origin, bo.ParentID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, fmt.Sprintf("peer relayed a v2 outline with unknown parent (%v)", bo.ParentID)) return } cs := h.s.cm.TipState() @@ -352,10 +348,7 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate } else if bo.ParentID != cs.Index.ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain - h.s.log.Printf("peer %v relayed a v2 outline that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, "peer relayed a v2 outline that does not attach to our tip") return } else if bid.CmpWork(cs.ChildTarget) < 0 { h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) @@ -409,10 +402,7 @@ func (h *rpcHandler) RelayV2TransactionSet(basis types.ChainIndex, txns []types. return } if _, ok := h.s.cm.Block(basis.ID); !ok { - h.s.log.Printf("peer %v relayed a v2 transaction set with unknown basis (%v); triggering a resync", origin, basis) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() + h.resync(origin, fmt.Sprintf("peer %v relayed a v2 transaction set with unknown basis (%v); triggering a resync", origin, basis)) return } if err := h.s.cm.AddV2PoolTransactions(basis, txns); err != nil { From 1efdf7fe5746ae86adfd810d0046b22fb68ea47b Mon Sep 17 00:00:00 2001 From: Michael Bulanov <72828450+mike76-dev@users.noreply.github.com> Date: Sat, 6 Jan 2024 18:02:49 +0100 Subject: [PATCH 046/630] Remove unreachable code check() already calls log.Fatal if err != nil, so this code is redundant --- cmd/walletd/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index da11cc1..0223e0d 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -51,9 +51,6 @@ func getAPIPassword() string { pw, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() check("Could not read API password:", err) - if err != nil { - log.Fatal(err) - } apiPassword = string(pw) } return apiPassword From b29b002aa00aaae58e804b6eb510ff7526a08996 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sun, 7 Jan 2024 00:01:29 -0500 Subject: [PATCH 047/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index caba7ce..8c2f43e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774 + go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index dbcb098..657e621 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774 h1:neyQGoGFqedOFgI0NcNPAdAv0A3lFvwthWm10Cw4qr8= -go.sia.tech/core v0.1.12-0.20240105034614-a052129f7774/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7 h1:Y22/f07Vd9VzGyKbV+BasMbpKj9sPfeEjRWGKUMhgNY= +go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From f877848d12cae022d241c97f7c95f3af83014806 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 8 Jan 2024 10:48:17 -0500 Subject: [PATCH 048/630] mod: Update core dependency --- api/server.go | 10 +++++---- go.mod | 2 +- go.sum | 4 ++-- syncer/syncer.go | 57 +++++++++++++++++++----------------------------- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/api/server.go b/api/server.go index c38c4d6..61dc08a 100644 --- a/api/server.go +++ b/api/server.go @@ -27,8 +27,8 @@ type ( RecommendedFee() types.Currency PoolTransactions() []types.Transaction V2PoolTransactions() []types.V2Transaction - AddPoolTransactions(txns []types.Transaction) error - AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) error + AddPoolTransactions(txns []types.Transaction) (bool, error) + AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) (bool, error) UnconfirmedParents(txn types.Transaction) []types.Transaction } @@ -148,14 +148,16 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { return } if len(tbr.Transactions) != 0 { - if jc.Check("invalid transaction set", s.cm.AddPoolTransactions(tbr.Transactions)) != nil { + _, err := s.cm.AddPoolTransactions(tbr.Transactions) + if jc.Check("invalid transaction set", err) != nil { return } s.s.BroadcastTransactionSet(tbr.Transactions) } if len(tbr.V2Transactions) != 0 { index := s.cm.TipState().Index - if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(index, tbr.V2Transactions)) != nil { + _, err := s.cm.AddV2PoolTransactions(index, tbr.V2Transactions) + if jc.Check("invalid v2 transaction set", err) != nil { return } s.s.BroadcastV2TransactionSet(index, tbr.V2Transactions) diff --git a/go.mod b/go.mod index 8c2f43e..62d4407 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7 + go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 657e621..511f07a 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7 h1:Y22/f07Vd9VzGyKbV+BasMbpKj9sPfeEjRWGKUMhgNY= -go.sia.tech/core v0.1.12-0.20240107050014-f660ae1b34d7/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202 h1:dVE3mN3DZGSV4nGBhc1W/nCx/lHCMe9Kb95e37Msj3Y= +go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/syncer/syncer.go b/syncer/syncer.go index 87b3211..8de17a6 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -28,9 +28,9 @@ type ChainManager interface { TipState() consensus.State PoolTransaction(txid types.TransactionID) (types.Transaction, bool) - AddPoolTransactions(txns []types.Transaction) error + AddPoolTransactions(txns []types.Transaction) (bool, error) V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) - AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2Transaction) error + AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2Transaction) (bool, error) TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) } @@ -286,24 +286,19 @@ func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { } func (h *rpcHandler) RelayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { - // if we've already seen these transactions, don't relay them again - for _, txn := range txns { - if _, ok := h.s.cm.PoolTransaction(txn.ID()); !ok { - goto add - } - } - return - -add: - if err := h.s.cm.AddPoolTransactions(txns); err != nil { - // too risky to ban here (txns are probably just outdated), but at least - // log it if we think we're synced - if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + if len(txns) == 0 { + h.s.ban(origin, errors.New("peer sent an empty transaction set")) + } else if known, err := h.s.cm.AddPoolTransactions(txns); !known { + if err != nil { + // too risky to ban here (txns are probably just outdated), but at least + // log it if we think we're synced + if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + } + } else { + h.s.relayTransactionSet(txns, origin) // non-blocking } - return } - h.s.relayTransactionSet(txns, origin) // non-blocking } func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { @@ -391,25 +386,17 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate } func (h *rpcHandler) RelayV2TransactionSet(basis types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { - // if we've already seen these transactions, don't relay them again - allSeen := true - for _, txn := range txns { - if _, allSeen = h.s.cm.V2PoolTransaction(txn.ID()); !allSeen { - break - } - } - if allSeen { - return - } if _, ok := h.s.cm.Block(basis.ID); !ok { - h.resync(origin, fmt.Sprintf("peer %v relayed a v2 transaction set with unknown basis (%v); triggering a resync", origin, basis)) - return - } - if err := h.s.cm.AddV2PoolTransactions(basis, txns); err != nil { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - return + h.resync(origin, fmt.Sprintf("peer %v relayed a v2 transaction set with unknown basis (%v)", origin, basis)) + } else if len(txns) == 0 { + h.s.ban(origin, errors.New("peer sent an empty transaction set")) + } else if known, err := h.s.cm.AddV2PoolTransactions(basis, txns); !known { + if err != nil { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + } else { + h.s.relayV2TransactionSet(basis, txns, origin) // non-blocking + } } - h.s.relayV2TransactionSet(basis, txns, origin) // non-blocking } func (s *Syncer) ban(p *gateway.Peer, err error) { From d7eea4c458985371f898322d90f6b74de609eaa3 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 8 Jan 2024 13:29:42 -0500 Subject: [PATCH 049/630] main: Add deletev1 command --- cmd/walletd/main.go | 8 ++-- cmd/walletd/testnet.go | 85 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 92 insertions(+), 7 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 594c0cc..c817539 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -139,7 +139,7 @@ func main() { sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") txnsCmd := flagg.New("txns", txnsUsage) txpoolCmd := flagg.New("txpool", txpoolUsage) - dbCheckCmd := flagg.New("checkdb", "check consensus.db for errors") + dbDeleteCmd := flagg.New("deletev1", "delete v1 state from consensus.db") cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, @@ -151,7 +151,7 @@ func main() { {Cmd: sendCmd}, {Cmd: txnsCmd}, {Cmd: txpoolCmd}, - {Cmd: dbCheckCmd}, + {Cmd: dbDeleteCmd}, }, }) @@ -264,11 +264,11 @@ func main() { c := initTestnetClient(apiAddr, network, seed) printTestnetTxpool(c, seed) - case dbCheckCmd: + case dbDeleteCmd: if len(cmd.Args()) != 0 { cmd.Usage() return } - testnetCheckDB(dir) + testnetDeleteV1DBState(dir) } } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 77b197a..81282a3 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/binary" "encoding/hex" "fmt" @@ -517,3 +518,87 @@ func testnetCheckDB(dir string) { } fmt.Println("No problems detected.") } + +func testnetDeleteV1DBState(dir string) { + bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + if err != nil { + log.Fatal(err) + } + var needUpdate bool + bdb.View(func(tx *bolt.Tx) error { + needUpdate = tx.Bucket([]byte("Tree")) != nil + return nil + }) + if !needUpdate { + return + } + + fmt.Println("Deleting unneeded v1 state...") + + var blockIDs []types.BlockID + bdb.View(func(tx *bolt.Tx) error { + return tx.Bucket([]byte("Blocks")).ForEach(func(k, v []byte) error { + blockIDs = append(blockIDs, *(*types.BlockID)(k)) + return nil + }) + }) + var total int + err = bdb.Update(func(tx *bolt.Tx) error { + for _, bucket := range []struct { + name string + elemName string + }{ + {"SiacoinElements", "siacoin elements"}, + {"SiafundElements", "siafund elements"}, + {"FileContracts", "file contract elements"}, + {"AncestorTimestamps", "ancestor timestamps"}, + {"Tree", "Merkle tree hashes"}, + } { + b := tx.Bucket([]byte(bucket.name)) + if b == nil { + continue + } + b.ForEach(func(k, v []byte) error { + fmt.Printf("\rDeleting %v...%x", bucket.elemName, k) + total += len(k) + len(v) + return b.Delete(k) + }) + tx.DeleteBucket([]byte(bucket.name)) + fmt.Println("done.") + } + return nil + }) + if err != nil { + log.Fatal(err) + } + + db := &boltDB{db: bdb} + defer db.Close() + network, genesisBlock := TestnetAnagami() + dbstore, _, err := chain.NewDBStore(db, network, genesisBlock) + if err != nil { + log.Fatal(err) + } + + var buf bytes.Buffer + e := types.NewEncoder(&buf) + for _, id := range blockIDs { + fmt.Printf("\rDeleting v1 block supplements...%v", id) + if b, bs, _ := dbstore.Block(id); bs != nil { + buf.Reset() + for _, txn := range bs.Transactions { + txn.EncodeTo(e) + } + for _, fc := range bs.ExpiringFileContracts { + fc.EncodeTo(e) + } + e.Flush() + total += buf.Len() + bs.Transactions = nil + bs.ExpiringFileContracts = nil + dbstore.AddBlock(b, bs) + } + } + fmt.Println("done.") + fmt.Printf("All v1 state deleted. Your consensus.db is now %v MB lighter!\n", total/1e6) +} diff --git a/go.mod b/go.mod index 62d4407..b542f45 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202 + go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index 511f07a..f88cde8 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202 h1:dVE3mN3DZGSV4nGBhc1W/nCx/lHCMe9Kb95e37Msj3Y= -go.sia.tech/core v0.1.12-0.20240108152323-e78806dec202/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0 h1:IJyShwpMA1gAWPmlUIeLrfPIFD0nNH5VznrdzWG44O4= +go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 8ede5d226cc0c1065b37e8b2a318783f96f1ff40 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 9 Jan 2024 12:07:54 -0500 Subject: [PATCH 050/630] syncerutil: Clear expired bans before save --- internal/syncerutil/store.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go index 40ff1a1..8e0f6fa 100644 --- a/internal/syncerutil/store.go +++ b/internal/syncerutil/store.go @@ -149,6 +149,12 @@ func (jps *JSONPeerStore) save() error { return nil } defer func() { jps.lastSave = time.Now() }() + // clear out expired bans + for peer, b := range jps.EphemeralPeerStore.bans { + if time.Until(b.Expiry) <= 0 { + delete(jps.EphemeralPeerStore.bans, peer) + } + } p := jsonPersist{ Peers: jps.EphemeralPeerStore.peers, Bans: jps.EphemeralPeerStore.bans, From 69f23b550f19c523fcb98c391a6c0b823536c88e Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 10 Jan 2024 18:33:34 -0500 Subject: [PATCH 051/630] mod: Update core dependency --- go.mod | 2 +- go.sum | 4 ++-- syncer/syncer.go | 59 ++++++++++++++++++++++++------------------------ 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index b542f45..e73f354 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0 + go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 diff --git a/go.sum b/go.sum index f88cde8..da86cc5 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0 h1:IJyShwpMA1gAWPmlUIeLrfPIFD0nNH5VznrdzWG44O4= -go.sia.tech/core v0.1.12-0.20240108182830-2ffe1cc4b4f0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4 h1:OfBI3wNFj86VwVFTVkQ8RRQB9g5uFul91NAxWAJ8mO0= +go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/syncer/syncer.go b/syncer/syncer.go index 8de17a6..d3b3054 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -22,7 +22,7 @@ type ChainManager interface { History() ([32]types.BlockID, error) BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) Block(id types.BlockID) (types.Block, bool) - SyncCheckpoint(index types.ChainIndex) (types.Block, consensus.State, bool) + State(id types.BlockID) (consensus.State, bool) AddBlocks(blocks []types.Block) error Tip() types.ChainIndex TipState() consensus.State @@ -249,30 +249,33 @@ func (h *rpcHandler) Transactions(index types.ChainIndex, txnHashes []types.Hash } func (h *rpcHandler) Checkpoint(index types.ChainIndex) (types.Block, consensus.State, error) { - b, cs, ok := h.s.cm.SyncCheckpoint(index) - if !ok { + b, ok1 := h.s.cm.Block(index.ID) + cs, ok2 := h.s.cm.State(b.ParentID) + if !ok1 || !ok2 { return types.Block{}, consensus.State{}, errors.New("checkpoint not found") } return b, cs, nil } func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bh.ID()); ok { - return // already seen - } else if _, ok := h.s.cm.Block(bh.ParentID); !ok { + cs, ok := h.s.cm.State(bh.ParentID) + if !ok { h.resync(origin, fmt.Sprintf("peer relayed a header with unknown parent (%v)", bh.ParentID)) return - } else if cs := h.s.cm.TipState(); bh.ParentID != cs.Index.ID { + } + bid := bh.ID() + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent header with insufficient work")) + return + } else if bh.ParentID != h.s.cm.Tip().ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain h.resync(origin, "peer relayed a header that does not attach to our tip") return - } else if bh.ID().CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) - return } - - // header is valid and attaches to our tip; request + validate full block + // request + validate full block if b, err := origin.SendBlock(bh.ID(), h.s.config.SendBlockTimeout); err != nil { // log-worthy, but not ban-worthy h.s.log.Printf("couldn't retrieve new block %v after header relay from %v: %v", bh.ID(), origin, err) @@ -302,23 +305,22 @@ func (h *rpcHandler) RelayTransactionSet(txns []types.Transaction, origin *gatew } func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bh.Parent.ID); !ok { + cs, ok := h.s.cm.State(bh.Parent.ID) + if !ok { h.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", bh.Parent.ID)) return } - cs := h.s.cm.TipState() bid := bh.ID(cs) - if _, ok := h.s.cm.Block(bid); ok { - // already seen + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) return - } else if bh.Parent.ID != cs.Index.ID { + } else if bh.Parent != h.s.cm.Tip() { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain h.resync(origin, "peer relayed a v2 header that does not attach to our tip") return - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) - return } // header is sufficiently valid; relay it @@ -331,23 +333,22 @@ func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Pee } func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bo.ParentID); !ok { + cs, ok := h.s.cm.State(bo.ParentID) + if !ok { h.resync(origin, fmt.Sprintf("peer relayed a v2 outline with unknown parent (%v)", bo.ParentID)) return } - cs := h.s.cm.TipState() bid := bo.ID(cs) - if _, ok := h.s.cm.Block(bid); ok { - // already seen + if _, ok := h.s.cm.State(bid); ok { + return // already seen + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) return - } else if bo.ParentID != cs.Index.ID { + } else if bo.ParentID != h.s.cm.Tip().ID { // block extends a sidechain, which peer (if honest) believes to be the // heaviest chain h.resync(origin, "peer relayed a v2 outline that does not attach to our tip") return - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) - return } // block has sufficient work and attaches to our tip, but may be missing @@ -356,7 +357,7 @@ func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gate txns, v2txns := h.s.cm.TransactionsForPartialBlock(bo.Missing()) b, missing := bo.Complete(cs, txns, v2txns) if len(missing) > 0 { - index := types.ChainIndex{ID: bid, Height: cs.Index.Height + 1} + index := types.ChainIndex{Height: bo.Height, ID: bid} txns, v2txns, err := origin.SendTransactions(index, missing, h.s.config.SendTransactionsTimeout) if err != nil { // log-worthy, but not ban-worthy From 5beb0235bf3ba388fd2ceef8718fdcffa4b26f8b Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 10 Jan 2024 18:47:41 -0500 Subject: [PATCH 052/630] syncer: Wait for peer goroutines before returning from (Syncer).Run --- syncer/syncer.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index d3b3054..ed378c1 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -773,13 +773,23 @@ func (s *Syncer) Run() error { s.l.Close() s.mu.Lock() s.l = nil - for addr, p := range s.peers { + for _, p := range s.peers { p.Close() - delete(s.peers, addr) } s.mu.Unlock() <-errChan <-errChan + + // wait for all peer goroutines to exit + // TODO: a cond would be nicer than polling here + s.mu.Lock() + for len(s.peers) != 0 { + s.mu.Unlock() + time.Sleep(100 * time.Millisecond) + s.mu.Lock() + } + s.mu.Unlock() + if errors.Is(err, net.ErrClosed) { return nil // graceful shutdown } From 9742e21bb6d5b742ff86a7a25f9c655a199e7e79 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 16 Jan 2024 18:13:52 -0500 Subject: [PATCH 053/630] all: Refactor for coreutils --- api/api_test.go | 4 +- api/server.go | 2 +- cmd/walletd/main.go | 9 - cmd/walletd/multiproof.go | 270 ---------- cmd/walletd/node.go | 77 +-- cmd/walletd/testnet.go | 196 ------- go.mod | 7 +- go.sum | 15 +- internal/syncerutil/store.go | 2 +- internal/walletutil/manager.go | 2 +- internal/walletutil/store.go | 2 +- syncer/syncer.go | 918 --------------------------------- wallet/seed.go | 6 +- 13 files changed, 30 insertions(+), 1480 deletions(-) delete mode 100644 cmd/walletd/multiproof.go delete mode 100644 syncer/syncer.go diff --git a/api/api_test.go b/api/api_test.go index 335807a..37ad08d 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -6,15 +6,15 @@ import ( "testing" "time" - "go.sia.tech/core/chain" "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" "go.sia.tech/jape" "go.sia.tech/walletd/api" "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/internal/walletutil" - "go.sia.tech/walletd/syncer" "go.sia.tech/walletd/wallet" "lukechampine.com/frand" ) diff --git a/api/server.go b/api/server.go index 61dc08a..f2ee9ec 100644 --- a/api/server.go +++ b/api/server.go @@ -15,7 +15,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" - "go.sia.tech/walletd/syncer" + "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/wallet" ) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index c817539..ec506c4 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -139,7 +139,6 @@ func main() { sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") txnsCmd := flagg.New("txns", txnsUsage) txpoolCmd := flagg.New("txpool", txpoolUsage) - dbDeleteCmd := flagg.New("deletev1", "delete v1 state from consensus.db") cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, @@ -151,7 +150,6 @@ func main() { {Cmd: sendCmd}, {Cmd: txnsCmd}, {Cmd: txpoolCmd}, - {Cmd: dbDeleteCmd}, }, }) @@ -263,12 +261,5 @@ func main() { seed := loadTestnetSeed(seed) c := initTestnetClient(apiAddr, network, seed) printTestnetTxpool(c, seed) - - case dbDeleteCmd: - if len(cmd.Args()) != 0 { - cmd.Usage() - return - } - testnetDeleteV1DBState(dir) } } diff --git a/cmd/walletd/multiproof.go b/cmd/walletd/multiproof.go deleted file mode 100644 index 45f1e0c..0000000 --- a/cmd/walletd/multiproof.go +++ /dev/null @@ -1,270 +0,0 @@ -package main - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "log" - "math/bits" - "path/filepath" - "sort" - - bolt "go.etcd.io/bbolt" - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" -) - -// copied from types/multiproof.go - -type elementLeaf struct { - *types.StateElement - ElementHash types.Hash256 -} - -func (l elementLeaf) hash() types.Hash256 { - buf := make([]byte, 1+32+8+1) - buf[0] = 0x00 // leafHashPrefix - copy(buf[1:], l.ElementHash[:]) - binary.LittleEndian.PutUint64(buf[33:], l.LeafIndex) - buf[41] = 0 // spent (always false for multiproofs) - return types.HashBytes(buf) -} - -func hashAll(elems ...interface{}) [32]byte { - h := types.NewHasher() - for _, e := range elems { - if et, ok := e.(types.EncoderTo); ok { - et.EncodeTo(h.E) - } else { - switch e := e.(type) { - case string: - h.WriteDistinguisher(e) - case uint64: - h.E.WriteUint64(e) - } - } - } - return h.Sum() -} - -func chainIndexLeaf(e *types.ChainIndexElement) elementLeaf { - return elementLeaf{&e.StateElement, hashAll("leaf/chainindex", e.ID, e.ChainIndex)} -} - -func siacoinLeaf(e *types.SiacoinElement) elementLeaf { - return elementLeaf{&e.StateElement, hashAll("leaf/siacoin", e.ID, e.SiacoinOutput, e.MaturityHeight)} -} - -func siafundLeaf(e *types.SiafundElement) elementLeaf { - return elementLeaf{&e.StateElement, hashAll("leaf/siafund", e.ID, e.SiafundOutput, e.ClaimStart)} -} - -func v2FileContractLeaf(e *types.V2FileContractElement) elementLeaf { - return elementLeaf{&e.StateElement, hashAll("leaf/v2filecontract", e.ID, e.V2FileContract)} -} - -func splitLeaves(ls []elementLeaf, mid uint64) (left, right []elementLeaf) { - split := sort.Search(len(ls), func(i int) bool { return ls[i].LeafIndex >= mid }) - return ls[:split], ls[split:] -} - -func forEachElementLeaf(txns []types.V2Transaction, fn func(l elementLeaf)) { - visit := func(l elementLeaf) { - if l.LeafIndex != types.EphemeralLeafIndex { - fn(l) - } - } - for _, txn := range txns { - for i := range txn.SiacoinInputs { - visit(siacoinLeaf(&txn.SiacoinInputs[i].Parent)) - } - for i := range txn.SiafundInputs { - visit(siafundLeaf(&txn.SiafundInputs[i].Parent)) - } - for i := range txn.FileContractRevisions { - visit(v2FileContractLeaf(&txn.FileContractRevisions[i].Parent)) - } - for i := range txn.FileContractResolutions { - visit(v2FileContractLeaf(&txn.FileContractResolutions[i].Parent)) - if r, ok := txn.FileContractResolutions[i].Resolution.(*types.V2StorageProof); ok { - visit(chainIndexLeaf(&r.ProofIndex)) - } - } - } -} - -func forEachTree(txns []types.V2Transaction, fn func(i, j uint64, leaves []elementLeaf)) { - clearBits := func(x uint64, n int) uint64 { return x &^ (1<= 64 { - d.SetErr(errors.New("invalid Merkle proof size")) - } - }) - if d.Err() != nil { - return - } - multiproof := make([]types.Hash256, multiproofSize(*txns)) - for i := range multiproof { - multiproof[i].DecodeFrom(d) - } - expandMultiproof(*txns, multiproof) -} - -func testnetFixMultiproofs(dir string) { - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) - if err != nil { - log.Fatal(err) - } - defer bdb.Close() - var needUpdate bool - bdb.Update(func(tx *bolt.Tx) error { - needUpdate = tx.Bucket([]byte("multiproof-fix")) == nil - return nil - }) - if !needUpdate { - return - } - - fmt.Println("Fixing consensus.db multiproofs...") - - type supplementedBlock struct { - Block types.Block - Supplement *consensus.V1BlockSupplement - } - - decodeBlock := func(v []byte) (sb supplementedBlock, err error) { - d := types.NewBufDecoder(v) - if v := d.ReadUint8(); v != 2 { - d.SetErr(fmt.Errorf("incompatible version (%d)", v)) - } - (*types.V1Block)(&sb.Block).DecodeFrom(d) - if d.ReadBool() { - sb.Block.V2 = new(types.V2BlockData) - sb.Block.V2.Height = d.ReadUint64() - sb.Block.V2.Commitment.DecodeFrom(d) - (*V2TransactionsMultiproof)(&sb.Block.V2.Transactions).DecodeFrom(d) - } - if d.ReadBool() { - sb.Supplement = new(consensus.V1BlockSupplement) - sb.Supplement.DecodeFrom(d) - } - err = d.Err() - return - } - encodeBlock := func(sb supplementedBlock) []byte { - var buf bytes.Buffer - e := types.NewEncoder(&buf) - e.WriteUint8(2) - (types.V2Block)(sb.Block).EncodeTo(e) - e.WriteBool(sb.Supplement != nil) - if sb.Supplement != nil { - sb.Supplement.EncodeTo(e) - } - e.Flush() - return buf.Bytes() - } - - err = bdb.Update(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte("Blocks")) - var keys []string - bucket.ForEach(func(k, v []byte) error { - keys = append(keys, string(k)) - return nil - }) - for _, k := range keys { - fmt.Printf("\r%x...", k) - b, err := decodeBlock(bucket.Get([]byte(k))) - if err != nil { - return err - } - if err := bucket.Put([]byte(k), encodeBlock(b)); err != nil { - return err - } - } - _, err := tx.CreateBucket([]byte("multiproof-fix")) - return err - }) - if err != nil { - fmt.Println() - log.Fatal(err) - } - fmt.Println("done.") -} diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 7000064..7326a32 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -11,14 +11,14 @@ import ( "strconv" "time" - bolt "go.etcd.io/bbolt" - "go.sia.tech/core/chain" "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/internal/walletutil" - "go.sia.tech/walletd/syncer" "lukechampine.com/upnp" ) @@ -83,68 +83,6 @@ var anagamiBootstrap = []string{ "100.34.20.44:9981", } -type boltDB struct { - tx *bolt.Tx - db *bolt.DB -} - -func (db *boltDB) newTx() (err error) { - if db.tx == nil { - db.tx, err = db.db.Begin(true) - } - return -} - -func (db *boltDB) Bucket(name []byte) chain.DBBucket { - if err := db.newTx(); err != nil { - panic(err) - } - - b := db.tx.Bucket(name) - if b == nil { - return nil - } - return b -} - -func (db *boltDB) CreateBucket(name []byte) (chain.DBBucket, error) { - if err := db.newTx(); err != nil { - return nil, err - } - - b, err := db.tx.CreateBucket(name) - if b == nil { - return nil, err - } - return b, nil -} - -func (db *boltDB) Flush() error { - if db.tx == nil { - return nil - } - - if err := db.tx.Commit(); err != nil { - return err - } - db.tx = nil - return nil -} - -func (db *boltDB) Cancel() { - if db.tx == nil { - return - } - - db.tx.Rollback() - db.tx = nil -} - -func (db *boltDB) Close() error { - db.Flush() - return db.db.Close() -} - type node struct { cm *chain.Manager s *syncer.Syncer @@ -167,18 +105,15 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) case "anagami": network, genesisBlock = TestnetAnagami() bootstrapPeers = anagamiBootstrap - testnetFixDBTree(dir) - testnetFixMultiproofs(dir) default: return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) if err != nil { log.Fatal(err) } - db := &boltDB{db: bdb} - dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { return nil, err } @@ -255,7 +190,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) return func() { l.Close() <-ch - db.Close() + bdb.Close() } }, }, nil diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 81282a3..254d7a0 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -1,19 +1,15 @@ package main import ( - "bytes" "encoding/binary" "encoding/hex" "fmt" "log" "math/big" "os" - "path/filepath" "reflect" "time" - bolt "go.etcd.io/bbolt" - "go.sia.tech/core/chain" "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/walletd/api" @@ -410,195 +406,3 @@ func printTestnetTxpool(c *api.Client, seed wallet.Seed) { } } } - -func testnetFixDBTree(dir string) { - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) - if err != nil { - log.Fatal(err) - } - db := &boltDB{db: bdb} - defer db.Close() - if db.Bucket([]byte("tree-fix-2")) != nil { - return - } - - fmt.Print("Fixing consensus.db Merkle tree...") - - network, genesisBlock := TestnetAnagami() - dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) - if err != nil { - log.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - bdb2, err := bolt.Open(filepath.Join(dir, "consensus.db-fixed"), 0600, nil) - if err != nil { - log.Fatal(err) - } - db2 := &boltDB{db: bdb2} - defer db2.Close() - dbstore2, tipState2, err := chain.NewDBStore(db2, network, genesisBlock) - if err != nil { - log.Fatal(err) - } - cm2 := chain.NewManager(dbstore2, tipState2) - - for cm2.Tip() != cm.Tip() { - fmt.Printf("\rFixing consensus.db Merkle tree...%v/%v", cm2.Tip().Height, cm.Tip().Height) - index, _ := cm.BestIndex(cm2.Tip().Height + 1) - b, _ := cm.Block(index.ID) - if err := cm2.AddBlocks([]types.Block{b}); err != nil { - break - } - } - fmt.Println() - - if _, err := db2.CreateBucket([]byte("tree-fix-2")); err != nil { - log.Fatal(err) - } else if err := db.Close(); err != nil { - log.Fatal(err) - } else if err := db2.Close(); err != nil { - log.Fatal(err) - } else if err := os.Rename(filepath.Join(dir, "consensus.db-fixed"), filepath.Join(dir, "consensus.db")); err != nil { - log.Fatal(err) - } - - fmt.Print("Backing up old wallet state...") - os.RemoveAll(filepath.Join(dir, "wallets.json-bck")) - os.Rename(filepath.Join(dir, "wallets.json"), filepath.Join(dir, "wallets.json-bck")) - os.RemoveAll(filepath.Join(dir, "wallets-bck")) - os.Rename(filepath.Join(dir, "wallets"), filepath.Join(dir, "wallets-bck")) - fmt.Println("done.") - fmt.Println("NOTE: Your wallet will resync automatically on first use; this may take a few seconds.") -} - -func testnetCheckDB(dir string) { - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) - if err != nil { - log.Fatal(err) - } - db := &boltDB{db: bdb} - defer db.Close() - - fmt.Print("Reapplying blocks...") - - network, genesisBlock := TestnetAnagami() - dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) - if err != nil { - log.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - dbstore2, tipState2, err := chain.NewDBStore(chain.NewMemDB(), network, genesisBlock) - if err != nil { - log.Fatal(err) - } - cm2 := chain.NewManager(dbstore2, tipState2) - - for cm2.Tip() != cm.Tip() { - fmt.Printf("\rReapplying blocks...%v/%v", cm2.Tip().Height, cm.Tip().Height) - index, _ := cm.BestIndex(cm2.Tip().Height + 1) - b, _ := cm.Block(index.ID) - if err := cm2.AddBlocks([]types.Block{b}); err != nil { - break - } - } - fmt.Println() - if cm.Tip() != cm2.Tip() { - fmt.Printf("Could not apply all blocks (%v/%v); marking consensus.db as corrupt\n", cm2.Tip().Height, cm.Tip().Height) - db.newTx() - db.tx.DeleteBucket([]byte("tree-fix-2")) - return - } - if cm.TipState().Commitment(types.Hash256{}, types.VoidAddress) != cm2.TipState().Commitment(types.Hash256{}, types.VoidAddress) { - fmt.Println("Final state differs from consensus.db; marking consensus.db as corrupt") - db.newTx() - db.tx.DeleteBucket([]byte("tree-fix-2")) - return - } - fmt.Println("No problems detected.") -} - -func testnetDeleteV1DBState(dir string) { - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) - if err != nil { - log.Fatal(err) - } - var needUpdate bool - bdb.View(func(tx *bolt.Tx) error { - needUpdate = tx.Bucket([]byte("Tree")) != nil - return nil - }) - if !needUpdate { - return - } - - fmt.Println("Deleting unneeded v1 state...") - - var blockIDs []types.BlockID - bdb.View(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("Blocks")).ForEach(func(k, v []byte) error { - blockIDs = append(blockIDs, *(*types.BlockID)(k)) - return nil - }) - }) - var total int - err = bdb.Update(func(tx *bolt.Tx) error { - for _, bucket := range []struct { - name string - elemName string - }{ - {"SiacoinElements", "siacoin elements"}, - {"SiafundElements", "siafund elements"}, - {"FileContracts", "file contract elements"}, - {"AncestorTimestamps", "ancestor timestamps"}, - {"Tree", "Merkle tree hashes"}, - } { - b := tx.Bucket([]byte(bucket.name)) - if b == nil { - continue - } - b.ForEach(func(k, v []byte) error { - fmt.Printf("\rDeleting %v...%x", bucket.elemName, k) - total += len(k) + len(v) - return b.Delete(k) - }) - tx.DeleteBucket([]byte(bucket.name)) - fmt.Println("done.") - } - return nil - }) - if err != nil { - log.Fatal(err) - } - - db := &boltDB{db: bdb} - defer db.Close() - network, genesisBlock := TestnetAnagami() - dbstore, _, err := chain.NewDBStore(db, network, genesisBlock) - if err != nil { - log.Fatal(err) - } - - var buf bytes.Buffer - e := types.NewEncoder(&buf) - for _, id := range blockIDs { - fmt.Printf("\rDeleting v1 block supplements...%v", id) - if b, bs, _ := dbstore.Block(id); bs != nil { - buf.Reset() - for _, txn := range bs.Transactions { - txn.EncodeTo(e) - } - for _, fc := range bs.ExpiringFileContracts { - fc.EncodeTo(e) - } - e.Flush() - total += buf.Len() - bs.Transactions = nil - bs.ExpiringFileContracts = nil - dbstore.AddBlock(b, bs) - } - } - fmt.Println("done.") - fmt.Printf("All v1 state deleted. Your consensus.db is now %v MB lighter!\n", total/1e6) -} diff --git a/go.mod b/go.mod index e73f354..741bb2d 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module go.sia.tech/walletd -go 1.18 +go 1.21 require ( - go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4 + go.sia.tech/core v0.2.0 + go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.10.0 golang.org/x/term v0.6.0 @@ -16,6 +16,7 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + go.etcd.io/bbolt v1.3.8 // indirect go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect diff --git a/go.sum b/go.sum index da86cc5..ca04bda 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,19 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4 h1:OfBI3wNFj86VwVFTVkQ8RRQB9g5uFul91NAxWAJ8mO0= -go.sia.tech/core v0.1.12-0.20240110232951-32f441a776b4/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.sia.tech/core v0.2.0 h1:+J/QylNueFmg5kCJCIfwqnCtKKoC/JN5wasPLy85QZI= +go.sia.tech/core v0.2.0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f h1:o+crEJc7ZJIOstRLpNVaIKUk3x+4Ki4UBfC9CFGZxs8= +go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f/go.mod h1:TqUzs1E/84w7Cq7IjH1RLjpIpv5haKc55Zdzsv6ciPA= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -20,6 +25,7 @@ go.sia.tech/web/walletd v0.10.0/go.mod h1:zfiPJGTwHjYyYGJNhjYTFn3OSJPPQkVL4nXp1M golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -28,6 +34,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= lukechampine.com/flagg v1.1.1/go.mod h1:a9ZuZu5LSPXELWSJrabRD00ort+lDXSOQu34xWgEoDI= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go index 8e0f6fa..6456c6b 100644 --- a/internal/syncerutil/store.go +++ b/internal/syncerutil/store.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "go.sia.tech/walletd/syncer" + "go.sia.tech/coreutils/syncer" ) type peerBan struct { diff --git a/internal/walletutil/manager.go b/internal/walletutil/manager.go index 185006b..be5e962 100644 --- a/internal/walletutil/manager.go +++ b/internal/walletutil/manager.go @@ -7,7 +7,7 @@ import ( "path/filepath" "sync" - "go.sia.tech/core/chain" + "go.sia.tech/coreutils/chain" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" ) diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go index 0ef2357..8a2ffef 100644 --- a/internal/walletutil/store.go +++ b/internal/walletutil/store.go @@ -6,7 +6,7 @@ import ( "os" "sync" - "go.sia.tech/core/chain" + "go.sia.tech/coreutils/chain" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" ) diff --git a/syncer/syncer.go b/syncer/syncer.go deleted file mode 100644 index ed378c1..0000000 --- a/syncer/syncer.go +++ /dev/null @@ -1,918 +0,0 @@ -package syncer - -import ( - "context" - "errors" - "fmt" - "io" - "log" - "net" - "reflect" - "sync" - "time" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" - "go.sia.tech/core/types" - "lukechampine.com/frand" -) - -// A ChainManager manages blockchain state. -type ChainManager interface { - History() ([32]types.BlockID, error) - BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) - Block(id types.BlockID) (types.Block, bool) - State(id types.BlockID) (consensus.State, bool) - AddBlocks(blocks []types.Block) error - Tip() types.ChainIndex - TipState() consensus.State - - PoolTransaction(txid types.TransactionID) (types.Transaction, bool) - AddPoolTransactions(txns []types.Transaction) (bool, error) - V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) - AddV2PoolTransactions(basis types.ChainIndex, txns []types.V2Transaction) (bool, error) - TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) -} - -// PeerInfo contains metadata about a peer. -type PeerInfo struct { - FirstSeen time.Time `json:"firstSeen"` - LastConnect time.Time `json:"lastConnect,omitempty"` - SyncedBlocks uint64 `json:"syncedBlocks,omitempty"` - SyncDuration time.Duration `json:"syncDuration,omitempty"` -} - -// A PeerStore stores peers and bans. -type PeerStore interface { - AddPeer(peer string) - Peers() []string - UpdatePeerInfo(peer string, fn func(*PeerInfo)) - PeerInfo(peer string) (PeerInfo, bool) - - // Ban temporarily bans one or more IPs. The addr should either be a single - // IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). - Ban(addr string, duration time.Duration, reason string) - Banned(peer string) bool -} - -// Subnet normalizes the provided CIDR subnet string. -func Subnet(addr, mask string) string { - ip, ipnet, err := net.ParseCIDR(addr + mask) - if err != nil { - return "" // shouldn't happen - } - return ip.Mask(ipnet.Mask).String() + mask -} - -type config struct { - MaxInboundPeers int - MaxOutboundPeers int - MaxInflightRPCs int - ConnectTimeout time.Duration - ShareNodesTimeout time.Duration - SendBlockTimeout time.Duration - SendTransactionsTimeout time.Duration - RelayHeaderTimeout time.Duration - RelayBlockOutlineTimeout time.Duration - RelayTransactionSetTimeout time.Duration - SendBlocksTimeout time.Duration - MaxSendBlocks uint64 - PeerDiscoveryInterval time.Duration - SyncInterval time.Duration - Logger *log.Logger -} - -// An Option modifies a Syncer's configuration. -type Option func(*config) - -// WithMaxInboundPeers sets the maximum number of inbound connections. The -// default is 8. -func WithMaxInboundPeers(n int) Option { - return func(c *config) { c.MaxInboundPeers = n } -} - -// WithMaxOutboundPeers sets the maximum number of outbound connections. The -// default is 8. -func WithMaxOutboundPeers(n int) Option { - return func(c *config) { c.MaxOutboundPeers = n } -} - -// WithMaxInflightRPCs sets the maximum number of concurrent RPCs per peer. The -// default is 3. -func WithMaxInflightRPCs(n int) Option { - return func(c *config) { c.MaxInflightRPCs = n } -} - -// WithConnectTimeout sets the timeout when connecting to a peer. The default is -// 5 seconds. -func WithConnectTimeout(d time.Duration) Option { - return func(c *config) { c.ConnectTimeout = d } -} - -// WithShareNodesTimeout sets the timeout for the ShareNodes RPC. The default is -// 5 seconds. -func WithShareNodesTimeout(d time.Duration) Option { - return func(c *config) { c.ShareNodesTimeout = d } -} - -// WithSendBlockTimeout sets the timeout for the SendBlock RPC. The default is -// 60 seconds. -func WithSendBlockTimeout(d time.Duration) Option { - return func(c *config) { c.SendBlockTimeout = d } -} - -// WithSendBlocksTimeout sets the timeout for the SendBlocks RPC. The default is -// 120 seconds. -func WithSendBlocksTimeout(d time.Duration) Option { - return func(c *config) { c.SendBlocksTimeout = d } -} - -// WithMaxSendBlocks sets the maximum number of blocks requested per SendBlocks -// RPC. The default is 10. -func WithMaxSendBlocks(n uint64) Option { - return func(c *config) { c.MaxSendBlocks = n } -} - -// WithSendTransactionsTimeout sets the timeout for the SendTransactions RPC. -// The default is 60 seconds. -func WithSendTransactionsTimeout(d time.Duration) Option { - return func(c *config) { c.SendTransactionsTimeout = d } -} - -// WithRelayHeaderTimeout sets the timeout for the RelayHeader and RelayV2Header -// RPCs. The default is 5 seconds. -func WithRelayHeaderTimeout(d time.Duration) Option { - return func(c *config) { c.RelayHeaderTimeout = d } -} - -// WithRelayBlockOutlineTimeout sets the timeout for the RelayV2BlockOutline -// RPC. The default is 60 seconds. -func WithRelayBlockOutlineTimeout(d time.Duration) Option { - return func(c *config) { c.RelayBlockOutlineTimeout = d } -} - -// WithRelayTransactionSetTimeout sets the timeout for the RelayTransactionSet -// RPC. The default is 60 seconds. -func WithRelayTransactionSetTimeout(d time.Duration) Option { - return func(c *config) { c.RelayTransactionSetTimeout = d } -} - -// WithPeerDiscoveryInterval sets the frequency at which the syncer attempts to -// discover and connect to new peers. The default is 5 seconds. -func WithPeerDiscoveryInterval(d time.Duration) Option { - return func(c *config) { c.PeerDiscoveryInterval = d } -} - -// WithSyncInterval sets the frequency at which the syncer attempts to sync with -// peers. The default is 5 seconds. -func WithSyncInterval(d time.Duration) Option { - return func(c *config) { c.SyncInterval = d } -} - -// WithLogger sets the logger used by a Syncer. The default is a logger that -// outputs to io.Discard. -func WithLogger(l *log.Logger) Option { - return func(c *config) { c.Logger = l } -} - -// A Syncer synchronizes blockchain data with peers. -type Syncer struct { - l net.Listener - cm ChainManager - pm PeerStore - header gateway.Header - config config - log *log.Logger // redundant, but convenient - - mu sync.Mutex - peers map[string]*gateway.Peer - synced map[string]bool - strikes map[string]int -} - -type rpcHandler struct { - s *Syncer -} - -func (h *rpcHandler) resync(p *gateway.Peer, reason string) { - h.s.mu.Lock() - alreadyResyncing := !h.s.synced[p.Addr] - h.s.synced[p.Addr] = false - h.s.mu.Unlock() - if !alreadyResyncing { - h.s.log.Printf("triggering resync with %v: %v", p, reason) - } -} - -func (h *rpcHandler) PeersForShare() (peers []string) { - peers = h.s.pm.Peers() - if len(peers) > 10 { - frand.Shuffle(len(peers), reflect.Swapper(peers)) - peers = peers[:10] - } - return peers -} - -func (h *rpcHandler) Block(id types.BlockID) (types.Block, error) { - b, ok := h.s.cm.Block(id) - if !ok { - return types.Block{}, errors.New("block not found") - } - return b, nil -} - -func (h *rpcHandler) BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) { - return h.s.cm.BlocksForHistory(history, max) -} - -func (h *rpcHandler) Transactions(index types.ChainIndex, txnHashes []types.Hash256) (txns []types.Transaction, v2txns []types.V2Transaction, _ error) { - if b, ok := h.s.cm.Block(index.ID); ok { - // get txns from block - want := make(map[types.Hash256]bool) - for _, h := range txnHashes { - want[h] = true - } - for _, txn := range b.Transactions { - if want[txn.FullHash()] { - txns = append(txns, txn) - } - } - for _, txn := range b.V2Transactions() { - if want[txn.FullHash()] { - v2txns = append(v2txns, txn) - } - } - return - } - txns, v2txns = h.s.cm.TransactionsForPartialBlock(txnHashes) - return -} - -func (h *rpcHandler) Checkpoint(index types.ChainIndex) (types.Block, consensus.State, error) { - b, ok1 := h.s.cm.Block(index.ID) - cs, ok2 := h.s.cm.State(b.ParentID) - if !ok1 || !ok2 { - return types.Block{}, consensus.State{}, errors.New("checkpoint not found") - } - return b, cs, nil -} - -func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { - cs, ok := h.s.cm.State(bh.ParentID) - if !ok { - h.resync(origin, fmt.Sprintf("peer relayed a header with unknown parent (%v)", bh.ParentID)) - return - } - bid := bh.ID() - if _, ok := h.s.cm.State(bid); ok { - return // already seen - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) - return - } else if bh.ParentID != h.s.cm.Tip().ID { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.resync(origin, "peer relayed a header that does not attach to our tip") - return - } - // request + validate full block - if b, err := origin.SendBlock(bh.ID(), h.s.config.SendBlockTimeout); err != nil { - // log-worthy, but not ban-worthy - h.s.log.Printf("couldn't retrieve new block %v after header relay from %v: %v", bh.ID(), origin, err) - return - } else if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { - h.s.ban(origin, err) - return - } - - h.s.relayHeader(bh, origin) // non-blocking -} - -func (h *rpcHandler) RelayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { - if len(txns) == 0 { - h.s.ban(origin, errors.New("peer sent an empty transaction set")) - } else if known, err := h.s.cm.AddPoolTransactions(txns); !known { - if err != nil { - // too risky to ban here (txns are probably just outdated), but at least - // log it if we think we're synced - if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - } - } else { - h.s.relayTransactionSet(txns, origin) // non-blocking - } - } -} - -func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { - cs, ok := h.s.cm.State(bh.Parent.ID) - if !ok { - h.resync(origin, fmt.Sprintf("peer relayed a v2 header with unknown parent (%v)", bh.Parent.ID)) - return - } - bid := bh.ID(cs) - if _, ok := h.s.cm.State(bid); ok { - return // already seen - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent v2 header with insufficient work")) - return - } else if bh.Parent != h.s.cm.Tip() { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.resync(origin, "peer relayed a v2 header that does not attach to our tip") - return - } - - // header is sufficiently valid; relay it - // - // NOTE: The purpose of header announcements is to inform the network as - // quickly as possible that a new block has been found. A proper - // BlockOutline should follow soon after, allowing peers to obtain the - // actual block. As such, we take no action here other than relaying. - h.s.relayV2Header(bh, origin) // non-blocking -} - -func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { - cs, ok := h.s.cm.State(bo.ParentID) - if !ok { - h.resync(origin, fmt.Sprintf("peer relayed a v2 outline with unknown parent (%v)", bo.ParentID)) - return - } - bid := bo.ID(cs) - if _, ok := h.s.cm.State(bid); ok { - return // already seen - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent v2 outline with insufficient work")) - return - } else if bo.ParentID != h.s.cm.Tip().ID { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.resync(origin, "peer relayed a v2 outline that does not attach to our tip") - return - } - - // block has sufficient work and attaches to our tip, but may be missing - // transactions; first, check for them in our txpool; then, if block is - // still incomplete, request remaining transactions from the peer - txns, v2txns := h.s.cm.TransactionsForPartialBlock(bo.Missing()) - b, missing := bo.Complete(cs, txns, v2txns) - if len(missing) > 0 { - index := types.ChainIndex{Height: bo.Height, ID: bid} - txns, v2txns, err := origin.SendTransactions(index, missing, h.s.config.SendTransactionsTimeout) - if err != nil { - // log-worthy, but not ban-worthy - h.s.log.Printf("couldn't retrieve missing transactions of %v after relay from %v: %v", bid, origin, err) - return - } - b, missing = bo.Complete(cs, txns, v2txns) - if len(missing) > 0 { - // inexcusable - h.s.ban(origin, errors.New("peer sent wrong missing transactions for a block it relayed")) - return - } - } - if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { - h.s.ban(origin, err) - return - } - - // when we forward the block, exclude any txns that were in our txpool, - // since they're probably present in our peers' txpools as well - // - // NOTE: crucially, we do NOT exclude any txns we had to request from the - // sending peer, since other peers probably don't have them either - bo.RemoveTransactions(txns, v2txns) - - h.s.relayV2BlockOutline(bo, origin) // non-blocking -} - -func (h *rpcHandler) RelayV2TransactionSet(basis types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(basis.ID); !ok { - h.resync(origin, fmt.Sprintf("peer %v relayed a v2 transaction set with unknown basis (%v)", origin, basis)) - } else if len(txns) == 0 { - h.s.ban(origin, errors.New("peer sent an empty transaction set")) - } else if known, err := h.s.cm.AddV2PoolTransactions(basis, txns); !known { - if err != nil { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - } else { - h.s.relayV2TransactionSet(basis, txns, origin) // non-blocking - } - } -} - -func (s *Syncer) ban(p *gateway.Peer, err error) { - s.log.Printf("banning %v: %v", p, err) - p.SetErr(errors.New("banned")) - s.pm.Ban(p.ConnAddr, 24*time.Hour, err.Error()) - - host, _, err := net.SplitHostPort(p.ConnAddr) - if err != nil { - return // shouldn't happen - } - // add a strike to each subnet - for subnet, maxStrikes := range map[string]int{ - Subnet(host, "/32"): 2, // 1.2.3.4:* - Subnet(host, "/24"): 8, // 1.2.3.* - Subnet(host, "/16"): 64, // 1.2.* - Subnet(host, "/8"): 512, // 1.* - } { - s.mu.Lock() - ban := (s.strikes[subnet] + 1) >= maxStrikes - if ban { - delete(s.strikes, subnet) - } else { - s.strikes[subnet]++ - } - s.mu.Unlock() - if ban { - s.pm.Ban(subnet, 24*time.Hour, "too many strikes") - } - } -} - -func (s *Syncer) runPeer(p *gateway.Peer) { - s.pm.AddPeer(p.Addr) - s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { - info.LastConnect = time.Now() - }) - s.mu.Lock() - s.peers[p.Addr] = p - s.mu.Unlock() - defer func() { - s.mu.Lock() - delete(s.peers, p.Addr) - s.mu.Unlock() - }() - - h := &rpcHandler{s: s} - inflight := make(chan struct{}, s.config.MaxInflightRPCs) - for { - if p.Err() != nil { - return - } - id, stream, err := p.AcceptRPC() - if err != nil { - p.SetErr(err) - return - } - inflight <- struct{}{} - go func() { - defer stream.Close() - // NOTE: we do not set any deadlines on the stream. If a peer is - // slow, fine; we don't need to worry about resource exhaustion - // unless we have tons of peers. - if err := p.HandleRPC(id, stream, h); err != nil { - s.log.Printf("incoming RPC %v from peer %v failed: %v", id, p, err) - } - <-inflight - }() - } -} - -func (s *Syncer) relayHeader(h gateway.BlockHeader, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin { - continue - } - go p.RelayHeader(h, s.config.RelayHeaderTimeout) - } -} - -func (s *Syncer) relayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin { - continue - } - go p.RelayTransactionSet(txns, s.config.RelayTransactionSetTimeout) - } -} - -func (s *Syncer) relayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2Header(bh, s.config.RelayHeaderTimeout) - } -} - -func (s *Syncer) relayV2BlockOutline(pb gateway.V2BlockOutline, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2BlockOutline(pb, s.config.RelayBlockOutlineTimeout) - } -} - -func (s *Syncer) relayV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2TransactionSet(index, txns, s.config.RelayTransactionSetTimeout) - } -} - -func (s *Syncer) allowConnect(peer string, inbound bool) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.l == nil { - return errors.New("syncer is shutting down") - } - if s.pm.Banned(peer) { - return errors.New("banned") - } - var in, out int - for _, p := range s.peers { - if p.Inbound { - in++ - } else { - out++ - } - } - // TODO: subnet-based limits - if inbound && in >= s.config.MaxInboundPeers { - return errors.New("too many inbound peers") - } else if !inbound && out >= s.config.MaxOutboundPeers { - return errors.New("too many outbound peers") - } - return nil -} - -func (s *Syncer) alreadyConnected(peer *gateway.Peer) bool { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p.UniqueID == peer.UniqueID { - return true - } - } - return false -} - -func (s *Syncer) acceptLoop() error { - for { - conn, err := s.l.Accept() - if err != nil { - return err - } - go func() { - defer conn.Close() - if err := s.allowConnect(conn.RemoteAddr().String(), true); err != nil { - s.log.Printf("rejected inbound connection from %v: %v", conn.RemoteAddr(), err) - } else if p, err := gateway.Accept(conn, s.header); err != nil { - s.log.Printf("failed to accept inbound connection from %v: %v", conn.RemoteAddr(), err) - } else if s.alreadyConnected(p) { - s.log.Printf("rejected inbound connection from %v: already connected", conn.RemoteAddr()) - } else { - s.runPeer(p) - } - }() - } -} - -func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { - numOutbound := func() (n int) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if !p.Inbound { - n++ - } - } - return - } - - lastTried := make(map[string]time.Time) - peersForConnect := func() (peers []string) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.pm.Peers() { - // TODO: don't include port in comparison - if _, ok := s.peers[p]; !ok && time.Since(lastTried[p]) > 5*time.Minute { - peers = append(peers, p) - } - } - // TODO: weighted random selection? - frand.Shuffle(len(peers), reflect.Swapper(peers)) - return peers - } - discoverPeers := func() { - // try up to three randomly-chosen peers - var peers []*gateway.Peer - s.mu.Lock() - for _, p := range s.peers { - if peers = append(peers, p); len(peers) >= 3 { - break - } - } - s.mu.Unlock() - for _, p := range peers { - nodes, err := p.ShareNodes(s.config.ShareNodesTimeout) - if err != nil { - continue - } - for _, n := range nodes { - s.pm.AddPeer(n) - } - } - } - - ticker := time.NewTicker(s.config.PeerDiscoveryInterval) - defer ticker.Stop() - sleep := func() bool { - select { - case <-ticker.C: - return true - case <-closeChan: - return false - } - } - closing := func() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.l == nil - } - for fst := true; fst || sleep(); fst = false { - if numOutbound() >= s.config.MaxOutboundPeers { - continue - } - candidates := peersForConnect() - if len(candidates) == 0 { - discoverPeers() - continue - } - for _, p := range candidates { - if numOutbound() >= s.config.MaxOutboundPeers || closing() { - break - } - if _, err := s.Connect(p); err == nil { - s.log.Printf("formed outbound connection to %v", p) - } else { - s.log.Printf("failed to form outbound connection to %v: %v", p, err) - } - lastTried[p] = time.Now() - } - } - return nil -} - -func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { - peersForSync := func() (peers []*gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if s.synced[p.Addr] { - continue - } - if peers = append(peers, p); len(peers) >= 3 { - break - } - } - return - } - - ticker := time.NewTicker(s.config.SyncInterval) - defer ticker.Stop() - sleep := func() bool { - select { - case <-ticker.C: - return true - case <-closeChan: - return false - } - } - for fst := true; fst || sleep(); fst = false { - for _, p := range peersForSync() { - history, err := s.cm.History() - if err != nil { - return err // generally fatal - } - s.mu.Lock() - s.synced[p.Addr] = true - s.mu.Unlock() - s.log.Printf("starting sync with %v", p) - oldTip := s.cm.Tip() - oldTime := time.Now() - lastPrint := time.Now() - startTime, startHeight := oldTime, oldTip.Height - var sentBlocks uint64 - addBlocks := func(blocks []types.Block) error { - if err := s.cm.AddBlocks(blocks); err != nil { - return err - } - sentBlocks += uint64(len(blocks)) - endTime, endHeight := time.Now(), s.cm.Tip().Height - s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { - info.SyncedBlocks += endHeight - startHeight - info.SyncDuration += endTime.Sub(startTime) - }) - startTime, startHeight = endTime, endHeight - if time.Since(lastPrint) > 30*time.Second { - s.log.Printf("syncing with %v, tip now %v (avg %.2f blocks/s)", p, s.cm.Tip(), float64(s.cm.Tip().Height-oldTip.Height)/endTime.Sub(oldTime).Seconds()) - lastPrint = time.Now() - } - return nil - } - if p.SupportsV2() { - history := history[:] - err = func() error { - for { - blocks, rem, err := p.SendV2Blocks(history, s.config.MaxSendBlocks, s.config.SendBlocksTimeout) - if err != nil { - return err - } else if err := addBlocks(blocks); err != nil { - return err - } else if rem == 0 { - return nil - } - history = []types.BlockID{blocks[len(blocks)-1].ID()} - } - }() - } else { - err = p.SendBlocks(history, s.config.SendBlocksTimeout, addBlocks) - } - totalBlocks := s.cm.Tip().Height - oldTip.Height - if err != nil { - s.log.Printf("syncing with %v failed after %v blocks: %v", p, totalBlocks, err) - } else if newTip := s.cm.Tip(); newTip != oldTip { - s.log.Printf("finished syncing %v blocks with %v, tip now %v", totalBlocks, p, newTip) - } else { - s.log.Printf("finished syncing %v blocks with %v, tip unchanged", sentBlocks, p) - } - } - } - return nil -} - -// Run spawns goroutines for accepting inbound connections, forming outbound -// connections, and syncing the blockchain from active peers. It blocks until an -// error occurs, upon which all connections are closed and goroutines are -// terminated. To gracefully shutdown a Syncer, close its net.Listener. -func (s *Syncer) Run() error { - errChan := make(chan error) - closeChan := make(chan struct{}) - go func() { errChan <- s.acceptLoop() }() - go func() { errChan <- s.peerLoop(closeChan) }() - go func() { errChan <- s.syncLoop(closeChan) }() - err := <-errChan - - // when one goroutine exits, shutdown and wait for the others - close(closeChan) - s.l.Close() - s.mu.Lock() - s.l = nil - for _, p := range s.peers { - p.Close() - } - s.mu.Unlock() - <-errChan - <-errChan - - // wait for all peer goroutines to exit - // TODO: a cond would be nicer than polling here - s.mu.Lock() - for len(s.peers) != 0 { - s.mu.Unlock() - time.Sleep(100 * time.Millisecond) - s.mu.Lock() - } - s.mu.Unlock() - - if errors.Is(err, net.ErrClosed) { - return nil // graceful shutdown - } - return err -} - -// Connect forms an outbound connection to a peer. -func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { - if err := s.allowConnect(addr, false); err != nil { - return nil, err - } - ctx, cancel := context.WithTimeout(context.Background(), s.config.ConnectTimeout) - defer cancel() - // slightly gross polling hack so that we shutdown quickly - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(100 * time.Millisecond): - s.mu.Lock() - if s.l == nil { - cancel() - } - s.mu.Unlock() - } - } - }() - conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) - if err != nil { - return nil, err - } - conn.SetDeadline(time.Now().Add(s.config.ConnectTimeout)) - defer conn.SetDeadline(time.Time{}) - p, err := gateway.Dial(conn, s.header) - if err != nil { - conn.Close() - return nil, err - } else if s.alreadyConnected(p) { - conn.Close() - return nil, errors.New("already connected") - } - go s.runPeer(p) - - // runPeer does this too, but doing it outside the goroutine prevents a race - s.mu.Lock() - s.peers[p.Addr] = p - s.mu.Unlock() - return p, nil -} - -// BroadcastHeader broadcasts a header to all peers. -func (s *Syncer) BroadcastHeader(h gateway.BlockHeader) { s.relayHeader(h, nil) } - -// BroadcastV2Header broadcasts a v2 header to all peers. -func (s *Syncer) BroadcastV2Header(h gateway.V2BlockHeader) { s.relayV2Header(h, nil) } - -// BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. -func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } - -// BroadcastTransactionSet broadcasts a transaction set to all peers. -func (s *Syncer) BroadcastTransactionSet(txns []types.Transaction) { s.relayTransactionSet(txns, nil) } - -// BroadcastV2TransactionSet broadcasts a v2 transaction set to all peers. -func (s *Syncer) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) { - s.relayV2TransactionSet(index, txns, nil) -} - -// Peers returns the set of currently-connected peers. -func (s *Syncer) Peers() []*gateway.Peer { - s.mu.Lock() - defer s.mu.Unlock() - var peers []*gateway.Peer - for _, p := range s.peers { - peers = append(peers, p) - } - return peers -} - -// PeerInfo returns metadata about the specified peer. -func (s *Syncer) PeerInfo(peer string) (PeerInfo, bool) { - s.mu.Lock() - defer s.mu.Unlock() - info, ok := s.pm.PeerInfo(peer) - return info, ok -} - -// Addr returns the address of the Syncer. -func (s *Syncer) Addr() string { - return s.l.Addr().String() -} - -// New returns a new Syncer. -func New(l net.Listener, cm ChainManager, pm PeerStore, header gateway.Header, opts ...Option) *Syncer { - config := config{ - MaxInboundPeers: 8, - MaxOutboundPeers: 8, - MaxInflightRPCs: 3, - ConnectTimeout: 5 * time.Second, - ShareNodesTimeout: 5 * time.Second, - SendBlockTimeout: 60 * time.Second, - SendTransactionsTimeout: 60 * time.Second, - RelayHeaderTimeout: 5 * time.Second, - RelayBlockOutlineTimeout: 60 * time.Second, - RelayTransactionSetTimeout: 60 * time.Second, - SendBlocksTimeout: 120 * time.Second, - MaxSendBlocks: 10, - PeerDiscoveryInterval: 5 * time.Second, - SyncInterval: 5 * time.Second, - Logger: log.New(io.Discard, "", 0), - } - for _, opt := range opts { - opt(&config) - } - return &Syncer{ - l: l, - cm: cm, - pm: pm, - header: header, - config: config, - log: config.Logger, - peers: make(map[string]*gateway.Peer), - synced: make(map[string]bool), - strikes: make(map[string]int), - } -} diff --git a/wallet/seed.go b/wallet/seed.go index 4a9046a..3a54c03 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -8,7 +8,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" - "go.sia.tech/core/wallet" + "go.sia.tech/coreutils" "lukechampine.com/frand" ) @@ -20,14 +20,14 @@ type Seed struct { // PublicKey derives the public key for the specified index. func (s Seed) PublicKey(index uint64) (pk types.PublicKey) { - key := wallet.KeyFromSeed(s.entropy, index) + key := coreutils.KeyFromSeed(s.entropy, index) copy(pk[:], key[len(key)-ed25519.PublicKeySize:]) return } // PrivateKey derives the private key for the specified index. func (s Seed) PrivateKey(index uint64) types.PrivateKey { - key := wallet.KeyFromSeed(s.entropy, index) + key := coreutils.KeyFromSeed(s.entropy, index) return key[:] } From eec8eec026838f05bbab426653b299bb83360877 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Tue, 30 Jan 2024 17:03:14 -0400 Subject: [PATCH 054/630] mod: Update coreutils dependency --- api/server.go | 10 +++++----- cmd/walletd/node.go | 10 ++-------- go.mod | 6 ++++-- go.sum | 14 ++++++++++---- wallet/seed.go | 6 +++--- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/api/server.go b/api/server.go index f2ee9ec..898ba29 100644 --- a/api/server.go +++ b/api/server.go @@ -35,9 +35,9 @@ type ( // A Syncer can connect to other peers and synchronize the blockchain. Syncer interface { Addr() string - Peers() []*gateway.Peer + Peers() []*syncer.Peer PeerInfo(peer string) (syncer.PeerInfo, bool) - Connect(addr string) (*gateway.Peer, error) + Connect(addr string) (*syncer.Peer, error) BroadcastHeader(bh gateway.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) @@ -85,14 +85,14 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { func (s *server) syncerPeersHandler(jc jape.Context) { var peers []GatewayPeer for _, p := range s.s.Peers() { - info, ok := s.s.PeerInfo(p.Addr) + info, ok := s.s.PeerInfo(p.Addr()) if !ok { continue } peers = append(peers, GatewayPeer{ - Addr: p.Addr, + Addr: p.Addr(), Inbound: p.Inbound, - Version: p.Version, + Version: p.Version(), FirstSeen: info.FirstSeen, ConnectedSince: info.LastConnect, diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 7326a32..eaf0d5d 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -3,10 +3,8 @@ package main import ( "context" "errors" - "io" "log" "net" - "os" "path/filepath" "strconv" "time" @@ -165,12 +163,8 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - logFile, err := os.OpenFile(filepath.Join(dir, "walletd.log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - log.Fatal(err) - } - logger := log.New(io.MultiWriter(os.Stderr, logFile), "", log.LstdFlags) - s := syncer.New(l, cm, ps, header, syncer.WithLogger(logger)) + + s := syncer.New(l, cm, ps, header) wm, err := walletutil.NewJSONWalletManager(dir, cm) if err != nil { diff --git a/go.mod b/go.mod index dec1ce8..e6df28b 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module go.sia.tech/walletd go 1.21 require ( - go.sia.tech/core v0.2.0 - go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f + go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc + go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.16.0 golang.org/x/term v0.6.0 @@ -19,6 +19,8 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index f7a5e84..f3a3ac8 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.2.0 h1:+J/QylNueFmg5kCJCIfwqnCtKKoC/JN5wasPLy85QZI= -go.sia.tech/core v0.2.0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= -go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f h1:o+crEJc7ZJIOstRLpNVaIKUk3x+4Ki4UBfC9CFGZxs8= -go.sia.tech/coreutils v0.0.0-20240116230957-2e4a0b211d4f/go.mod h1:TqUzs1E/84w7Cq7IjH1RLjpIpv5haKc55Zdzsv6ciPA= +go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc h1:oUCCTOatQIwYkJ2FUWRvJtgU+i/BwlzmzCxoSvmmJVQ= +go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 h1:G2l6fRzAdNZy2z7+FhoG2y8ARtFpR6PkXXTB5tkdfZ8= +go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7/go.mod h1:3Mb206QDd3NtRiaHZ2kN87/HKXhcBF6lHVatS7PkViY= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -22,6 +22,12 @@ go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= go.sia.tech/web/walletd v0.16.0 h1:tCERgjsz4orokM94kt7PH2tNweHdOwK5aoPsCXes5HM= go.sia.tech/web/walletd v0.16.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= diff --git a/wallet/seed.go b/wallet/seed.go index 3a54c03..09288bb 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -8,7 +8,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" - "go.sia.tech/coreutils" + "go.sia.tech/coreutils/wallet" "lukechampine.com/frand" ) @@ -20,14 +20,14 @@ type Seed struct { // PublicKey derives the public key for the specified index. func (s Seed) PublicKey(index uint64) (pk types.PublicKey) { - key := coreutils.KeyFromSeed(s.entropy, index) + key := wallet.KeyFromSeed(s.entropy, index) copy(pk[:], key[len(key)-ed25519.PublicKeySize:]) return } // PrivateKey derives the private key for the specified index. func (s Seed) PrivateKey(index uint64) types.PrivateKey { - key := coreutils.KeyFromSeed(s.entropy, index) + key := wallet.KeyFromSeed(s.entropy, index) return key[:] } From 4895bc723267d94200202e66b70b75f305133a36 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Jan 2024 12:09:01 -0800 Subject: [PATCH 055/630] deps: update go version, add sqlite --- cmd/walletd/node.go | 5 +- go.mod | 3 +- go.sum | 2 + persist/sqlite/consensus.go | 510 +++++++++++++++++++++++++++++++ persist/sqlite/consts_default.go | 12 + persist/sqlite/consts_testing.go | 12 + persist/sqlite/init.go | 89 ++++++ persist/sqlite/init.sql | 70 +++++ persist/sqlite/migrations.go | 10 + persist/sqlite/sql.go | 232 ++++++++++++++ persist/sqlite/store.go | 120 ++++++++ persist/sqlite/types.go | 135 ++++++++ persist/sqlite/wallet.go | 292 ++++++++++++++++++ wallet/state.go | 83 +++++ wallet/wallet.go | 25 +- 15 files changed, 1587 insertions(+), 13 deletions(-) create mode 100644 persist/sqlite/consensus.go create mode 100644 persist/sqlite/consts_default.go create mode 100644 persist/sqlite/consts_testing.go create mode 100644 persist/sqlite/init.go create mode 100644 persist/sqlite/init.sql create mode 100644 persist/sqlite/migrations.go create mode 100644 persist/sqlite/sql.go create mode 100644 persist/sqlite/store.go create mode 100644 persist/sqlite/types.go create mode 100644 persist/sqlite/wallet.go create mode 100644 wallet/state.go diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index eaf0d5d..3249fe6 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -17,6 +17,7 @@ import ( "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/internal/walletutil" + "go.uber.org/zap" "lukechampine.com/upnp" ) @@ -163,9 +164,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - - s := syncer.New(l, cm, ps, header) - + s := syncer.New(l, cm, ps, header, syncer.WithLogger(zap.NewNop())) wm, err := walletutil.NewJSONWalletManager(dir, cm) if err != nil { return nil, err diff --git a/go.mod b/go.mod index e6df28b..b50ae10 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module go.sia.tech/walletd go 1.21 require ( + github.com/mattn/go-sqlite3 v1.14.21 go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.16.0 + go.uber.org/zap v1.26.0 golang.org/x/term v0.6.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 @@ -20,7 +22,6 @@ require ( go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index f3a3ac8..e25fb79 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/mattn/go-sqlite3 v1.14.21 h1:IXocQLOykluc3xPE0Lvy8FtggMz1G+U3mEjg+0zGizc= +github.com/mattn/go-sqlite3 v1.14.21/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go new file mode 100644 index 0000000..03f8f0a --- /dev/null +++ b/persist/sqlite/consensus.go @@ -0,0 +1,510 @@ +package sqlite + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/wallet" +) + +const updateProofBatchSize = 1000 + +type proofUpdater interface { + UpdateElementProof(*types.StateElement) +} + +func insertChainIndex(tx txn, index types.ChainIndex) (id int64, err error) { + err = tx.QueryRow(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`, index.Height, encode(index.ID)).Scan(&id) + return +} + +func applyEvents(tx txn, events []wallet.Event) error { + stmt, err := tx.Prepare(`INSERT INTO events (date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4) RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + addRelevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id, block_height) VALUES ($1, $2, $3)`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer addRelevantAddrStmt.Close() + + for _, event := range events { + id, err := insertChainIndex(tx, event.Index) + if err != nil { + return fmt.Errorf("failed to create chain index: %w", err) + } + + buf, err := json.Marshal(event.Val) + if err != nil { + return fmt.Errorf("failed to marshal event: %w", err) + } + + var eventID int64 + err = stmt.QueryRow(sqlTime(event.Timestamp), id, event.Val.EventType(), buf).Scan(&eventID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + + for _, addr := range event.Relevant { + addressID, err := insertAddress(tx, addr) + if err != nil { + return fmt.Errorf("failed to insert address: %w", err) + } else if _, err := addRelevantAddrStmt.Exec(eventID, addressID, event.Index.Height); err != nil { + return fmt.Errorf("failed to add relevant address: %w", err) + } + log.Println("added relevant address", eventID, addr) + } + } + return nil +} + +func deleteSiacoinOutputs(tx txn, spent []types.SiacoinElement) error { + addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare lookup statement: %w", err) + } + defer addrStmt.Close() + + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateBalanceStmt.Close() + + deleteStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer deleteStmt.Close() + + for _, se := range spent { + // query the address database ID and balance + var addressID int64 + var balance types.Currency + err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, (*sqlCurrency)(&balance)) + if err != nil { + return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) + } + + // update the balance + balance = balance.Sub(se.SiacoinOutput.Value) + _, err = updateBalanceStmt.Exec((*sqlCurrency)(&balance), addressID) + if err != nil { + return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) + } + + var dummy types.Hash256 + err = deleteStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy, 32)) + if err != nil { + return fmt.Errorf("failed to delete output %q: %w", se.ID, err) + } + } + return nil +} + +func applySiacoinOutputs(tx txn, added map[types.Hash256]types.SiacoinElement) error { + addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare lookup statement: %w", err) + } + defer addrStmt.Close() + + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateBalanceStmt.Close() + + addStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, address_id, siacoin_value, merkle_proof, leaf_index, maturity_height) VALUES ($1, $2, $3, $4, $5, $6)`) + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer addStmt.Close() + + for _, se := range added { + // query the address database ID and balance + var addressID int64 + var balance types.Currency + err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, (*sqlCurrency)(&balance)) + if err != nil { + return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) + } + + // update the balance + balance = balance.Add(se.SiacoinOutput.Value) + _, err = updateBalanceStmt.Exec((*sqlCurrency)(&balance), addressID) + if err != nil { + return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) + } + + // insert the created utxo + _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.MaturityHeight, se.LeafIndex) + if err != nil { + return fmt.Errorf("failed to insert output %q: %w", se.ID, err) + } + } + return nil +} + +func deleteSiafundOutputs(tx txn, spent []types.SiafundElement) error { + addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare lookup statement: %w", err) + } + defer addrStmt.Close() + + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateBalanceStmt.Close() + + spendStmt, err := tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer spendStmt.Close() + + for _, se := range spent { + // query the address database ID and balance + var addressID int64 + var balance uint64 + err := addrStmt.QueryRow(encode(se.SiafundOutput.Address)).Scan(&addressID, balance) + if err != nil { + return fmt.Errorf("failed to lookup address %q: %w", se.SiafundOutput.Address, err) + } + + // update the balance + if balance < se.SiafundOutput.Value { + panic("siafund balance is negative") // developer error + } + balance -= se.SiafundOutput.Value + _, err = updateBalanceStmt.Exec(balance, addressID) + if err != nil { + return fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) + } + + var dummy types.Hash256 + err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy, 32)) + if err != nil { + return fmt.Errorf("failed to delete output %q: %w", se.ID, err) + } + } + return nil +} + +func applySiafundOutputs(tx txn, added map[types.Hash256]types.SiafundElement) error { + addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare lookup statement: %w", err) + } + defer addrStmt.Close() + + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateBalanceStmt.Close() + + addStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, address_id, claim_start, siafund_value, merkle_proof, leaf_index) VALUES ($1, $2, $3, $4, $5, $6)`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer addStmt.Close() + + for _, se := range added { + // query the address database ID and balance + var addressID int64 + var balance uint64 + err := addrStmt.QueryRow(encode(se.SiafundOutput.Address)).Scan(&addressID, balance) + if err != nil { + return fmt.Errorf("failed to lookup address %q: %w", se.SiafundOutput.Address, err) + } + + // update the balance + if balance < se.SiafundOutput.Value { + panic("siafund balance is negative") // developer error + } + balance -= se.SiafundOutput.Value + _, err = updateBalanceStmt.Exec(balance, addressID) + if err != nil { + return fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) + } + + _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.ClaimStart), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex) + if err != nil { + return fmt.Errorf("failed to insert output %q: %w", se.ID, err) + } + } + return nil +} + +func updateLastIndexedTip(tx txn, tip types.ChainIndex) error { + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(tip.ID)) + return err +} + +// how slow is this going to be 😬? +// +// todo: determine if it's feasible for exchange mode to keep everything in +// memory. +func updateElementProofs(tx txn, table string, updater proofUpdater) error { + stmt, err := tx.Prepare(`SELECT id, merkle_proof, leaf_index FROM ` + table + ` LIMIT $1 OFFSET $2`) + if err != nil { + return fmt.Errorf("failed to prepare batch statement: %w", err) + } + defer stmt.Close() + + updateStmt, err := tx.Prepare(`UPDATE ` + table + ` SET merkle_proof=$1, leaf_index=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateStmt.Close() + + var updated []types.StateElement + for offset := 0; ; offset += updateProofBatchSize { + updated = updated[:0] + + more, err := func(n int) (bool, error) { + rows, err := stmt.Query(updateProofBatchSize, n) + if err != nil { + return false, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var more bool + for rows.Next() { + // if we get here, there may be more rows to process + more = true + + var se types.StateElement + err := rows.Scan(decode(&se.ID, 32), decodeSlice(&se.MerkleProof, 32*1000), &se.LeafIndex) + if err != nil { + return false, fmt.Errorf("failed to scan state element: %w", err) + } + updater.UpdateElementProof(&se) + updated = append(updated, se) + } + return more, nil + }(offset) + if err != nil { + return err + } + + for _, se := range updated { + _, err := updateStmt.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) + } + } + + if !more { + break + } + } + + return nil +} + +func applyChainUpdates(tx txn, updates []*chain.ApplyUpdate) error { + stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + // note: this would be more performant for small wallets to load all + // addresses into memory. However, for larger wallets (> 10K addresses), + // this is time consuming. Instead, the database is queried for each + // address. Monitor performance and consider changing this in the + // future. From a memory perspective, it would be fine to lazy load all + // addresses into memory. + ownsAddress := func(address types.Address) bool { + var dbID int64 + err := stmt.QueryRow(encode(address)).Scan(&dbID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + panic(err) // database error + } + return err == nil + } + + for _, update := range updates { + events := wallet.AppliedEvents(update.State, update.Block, update, ownsAddress) + if err := applyEvents(tx, events); err != nil { + return fmt.Errorf("failed to apply events: %w", err) + } + + var spentSiacoinOutputs []types.SiacoinElement + newSiacoinOutputs := make(map[types.Hash256]types.SiacoinElement) + update.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if !ownsAddress(se.SiacoinOutput.Address) { + return + } + + if spent { + spentSiacoinOutputs = append(spentSiacoinOutputs, se) + delete(newSiacoinOutputs, se.ID) + } else { + newSiacoinOutputs[se.ID] = se + } + }) + + if err := deleteSiacoinOutputs(tx, spentSiacoinOutputs); err != nil { + return fmt.Errorf("failed to delete siacoin outputs: %w", err) + } else if err := applySiacoinOutputs(tx, newSiacoinOutputs); err != nil { + return fmt.Errorf("failed to apply siacoin outputs: %w", err) + } + + var spentSiafundOutputs []types.SiafundElement + newSiafundOutputs := make(map[types.Hash256]types.SiafundElement) + update.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { + if !ownsAddress(sf.SiafundOutput.Address) { + return + } + + if spent { + spentSiafundOutputs = append(spentSiafundOutputs, sf) + delete(newSiafundOutputs, sf.ID) + } else { + newSiafundOutputs[sf.ID] = sf + } + }) + + if err := deleteSiafundOutputs(tx, spentSiafundOutputs); err != nil { + return fmt.Errorf("failed to delete siafund outputs: %w", err) + } else if err := applySiafundOutputs(tx, newSiafundOutputs); err != nil { + return fmt.Errorf("failed to apply siafund outputs: %w", err) + } + + // update proofs + if err := updateElementProofs(tx, "siacoin_elements", update); err != nil { + return fmt.Errorf("failed to update siacoin element proofs: %w", err) + } else if err := updateElementProofs(tx, "siafund_elements", update); err != nil { + return fmt.Errorf("failed to update siafund element proofs: %w", err) + } + } + + lastTip := updates[len(updates)-1].State.Index + if err := updateLastIndexedTip(tx, lastTip); err != nil { + return fmt.Errorf("failed to update last indexed tip: %w", err) + } + return nil +} + +func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { + s.updates = append(s.updates, cau) + + if mayCommit { + return s.transaction(func(tx txn) error { + if err := applyChainUpdates(tx, s.updates); err != nil { + return err + } + s.updates = nil + return nil + }) + } + return nil +} + +func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { + // update hasn't been committed yet + if len(s.updates) > 0 && s.updates[len(s.updates)-1].Block.ID() == cru.Block.ID() { + s.updates = s.updates[:len(s.updates)-1] + return nil + } + + // update has been committed, revert it + return s.transaction(func(tx txn) error { + stmt, err := tx.Prepare(`SELECT sia_address FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + // note: this would be more performant for small wallets to load all + // addresses into memory. However, for larger wallets (> 10K addresses), + // this is time consuming. Instead, the database is queried for each + // address. Monitor performance and consider changing this in the + // future. From a memory perspective, it would be fine to lazy load all + // addresses into memory. + ownsAddress := func(address types.Address) bool { + var dbID int64 + err := stmt.QueryRow(encode(address)).Scan(&dbID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + panic(err) // database error + } + return err == nil + } + + var spentSiacoinOutputs []types.SiacoinElement + var spentSiafundOutputs []types.SiafundElement + addedSiacoinOutputs := make(map[types.Hash256]types.SiacoinElement) + addedSiafundOutputs := make(map[types.Hash256]types.SiafundElement) + + cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if !ownsAddress(se.SiacoinOutput.Address) { + return + } + + if !spent { + spentSiacoinOutputs = append(spentSiacoinOutputs, se) + } else { + addedSiacoinOutputs[se.ID] = se + } + }) + + cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if !ownsAddress(se.SiafundOutput.Address) { + return + } + + if !spent { + spentSiafundOutputs = append(spentSiafundOutputs, se) + } else { + addedSiafundOutputs[se.ID] = se + } + }) + + // revert siacoin outputs + if err := deleteSiacoinOutputs(tx, spentSiacoinOutputs); err != nil { + return fmt.Errorf("failed to delete siacoin outputs: %w", err) + } else if err := applySiacoinOutputs(tx, addedSiacoinOutputs); err != nil { + return fmt.Errorf("failed to apply siacoin outputs: %w", err) + } + + // revert siafund outputs + if err := deleteSiafundOutputs(tx, spentSiafundOutputs); err != nil { + return fmt.Errorf("failed to delete siafund outputs: %w", err) + } else if err := applySiafundOutputs(tx, addedSiafundOutputs); err != nil { + return fmt.Errorf("failed to apply siafund outputs: %w", err) + } + + // revert events + _, err = tx.Exec(`DELETE FROM chain_indices WHERE block_id=$1`, cru.Block.ID()) + if err != nil { + return fmt.Errorf("failed to delete chain index: %w", err) + } + + // update proofs + if err := updateElementProofs(tx, "siacoin_elements", cru); err != nil { + return fmt.Errorf("failed to update siacoin element proofs: %w", err) + } else if err := updateElementProofs(tx, "siafund_elements", cru); err != nil { + return fmt.Errorf("failed to update siafund element proofs: %w", err) + } + return nil + }) +} + +// LastCommittedIndex returns the last chain index that was committed. +func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { + err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index, 40)) + return +} diff --git a/persist/sqlite/consts_default.go b/persist/sqlite/consts_default.go new file mode 100644 index 0000000..50b7330 --- /dev/null +++ b/persist/sqlite/consts_default.go @@ -0,0 +1,12 @@ +//go:build !testing + +package sqlite + +import "time" + +const ( + busyTimeout = 10000 // 10 seconds + maxRetryAttempts = 30 // 30 attempts + factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds + maxBackoff = 15 * time.Second +) diff --git a/persist/sqlite/consts_testing.go b/persist/sqlite/consts_testing.go new file mode 100644 index 0000000..f4911e3 --- /dev/null +++ b/persist/sqlite/consts_testing.go @@ -0,0 +1,12 @@ +//go:build testing + +package sqlite + +import "time" + +const ( + busyTimeout = 100 // 100ms + maxRetryAttempts = 10 // 10 attempts + factor = 2.0 // factor ^ retryAttempts = backoff time in milliseconds + maxBackoff = 15 * time.Second +) diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go new file mode 100644 index 0000000..bc6c1a4 --- /dev/null +++ b/persist/sqlite/init.go @@ -0,0 +1,89 @@ +package sqlite + +import ( + "database/sql" + _ "embed" // for init.sql + "errors" + "time" + + "fmt" + + "go.sia.tech/core/types" + "go.uber.org/zap" +) + +// init queries are run when the database is first created. +// +//go:embed init.sql +var initDatabase string + +func initializeSettings(tx txn, target int64) error { + _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_tip) VALUES (0, ?, ?)`, target, encode(types.ChainIndex{})) + return err +} + +func (s *Store) initNewDatabase(target int64) error { + return s.transaction(func(tx txn) error { + if _, err := tx.Exec(initDatabase); err != nil { + return fmt.Errorf("failed to initialize database: %w", err) + } else if err := initializeSettings(tx, target); err != nil { + return fmt.Errorf("failed to initialize settings: %w", err) + } + return nil + }) +} + +func (s *Store) upgradeDatabase(current, target int64) error { + log := s.log.Named("migrations") + log.Info("migrating database", zap.Int64("current", current), zap.Int64("target", target)) + + // disable foreign key constraints during migration + if _, err := s.db.Exec("PRAGMA foreign_keys = OFF"); err != nil { + return fmt.Errorf("failed to disable foreign key constraints: %w", err) + } + defer func() { + // re-enable foreign key constraints + if _, err := s.db.Exec("PRAGMA foreign_keys = ON"); err != nil { + log.Panic("failed to enable foreign key constraints", zap.Error(err)) + } + }() + + return s.transaction(func(tx txn) error { + for _, fn := range migrations[current-1:] { + current++ + start := time.Now() + if err := fn(tx, log.With(zap.Int64("version", current))); err != nil { + return fmt.Errorf("failed to migrate database to version %v: %w", current, err) + } + // check that no foreign key constraints were violated + if err := tx.QueryRow("PRAGMA foreign_key_check").Scan(); !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("foreign key constraints are not satisfied") + } + log.Debug("migration complete", zap.Int64("current", current), zap.Int64("target", target), zap.Duration("elapsed", time.Since(start))) + } + + // set the final database version + return setDBVersion(tx, target) + }) +} + +func (s *Store) init() error { + // calculate the expected final database version + target := int64(len(migrations) + 1) + // disable foreign key constraints during migration + if _, err := s.db.Exec("PRAGMA foreign_keys = OFF"); err != nil { + return fmt.Errorf("failed to disable foreign key constraints: %w", err) + } + + version := getDBVersion(s.db) + switch { + case version == 0: + return s.initNewDatabase(target) + case version < target: + return s.upgradeDatabase(version, target) + case version > target: + return fmt.Errorf("database version %v is newer than expected %v. database downgrades are not supported", version, target) + } + // nothing to do + return nil +} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql new file mode 100644 index 0000000..d6a8619 --- /dev/null +++ b/persist/sqlite/init.sql @@ -0,0 +1,70 @@ +CREATE TABLE chain_indices ( + id INTEGER PRIMARY KEY, + block_id BLOB UNIQUE NOT NULL, + height INTEGER UNIQUE NOT NULL +); + +CREATE TABLE sia_addresses ( + id INTEGER PRIMARY KEY, + sia_address BLOB UNIQUE NOT NULL, + siacoin_balance BLOB NOT NULL, + siafund_balance INTEGER NOT NULL +); + +CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + address_id INTEGER NOT NULL REFERENCES sia_addresses (id) +); +CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); + +CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id) +); +CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); + +CREATE TABLE wallets ( + id TEXT PRIMARY KEY NOT NULL, + extra_data BLOB NOT NULL +); + +CREATE TABLE wallet_addresses ( + wallet_id TEXT NOT NULL REFERENCES wallets (id), + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + extra_data BLOB NOT NULL, + UNIQUE (wallet_id, address_id) +); +CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + date_created INTEGER NOT NULL, + index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + event_data TEXT NOT NULL +); + +CREATE TABLE event_addresses ( + id INTEGER PRIMARY KEY, + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + block_height INTEGER NOT NULL, /* prevents extra join when querying for events */ + UNIQUE (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_block_height ON event_addresses(event_id, address_id, block_height DESC); + +CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + last_indexed_tip BLOB -- the last chain index that was processed +); diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go new file mode 100644 index 0000000..7aa3692 --- /dev/null +++ b/persist/sqlite/migrations.go @@ -0,0 +1,10 @@ +package sqlite + +import ( + "go.uber.org/zap" +) + +// migrations is a list of functions that are run to migrate the database from +// one version to the next. Migrations are used to update existing databases to +// match the schema in init.sql. +var migrations = []func(tx txn, log *zap.Logger) error{} diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go new file mode 100644 index 0000000..6ea715f --- /dev/null +++ b/persist/sqlite/sql.go @@ -0,0 +1,232 @@ +package sqlite + +import ( + "context" + "database/sql" + "math/rand" + "strings" + "time" + + _ "github.com/mattn/go-sqlite3" // import sqlite3 driver + "go.uber.org/zap" +) + +const ( + longQueryDuration = 10 * time.Millisecond + longTxnDuration = 10 * time.Millisecond +) + +type ( + // A scanner is an interface that wraps the Scan method of sql.Rows and sql.Row + scanner interface { + Scan(dest ...any) error + } + + // A txn is an interface for executing queries within a transaction. + txn interface { + // Exec executes a query without returning any rows. The args are for + // any placeholder parameters in the query. + Exec(query string, args ...any) (sql.Result, error) + // Prepare creates a prepared statement for later queries or executions. + // Multiple queries or executions may be run concurrently from the + // returned statement. The caller must call the statement's Close method + // when the statement is no longer needed. + Prepare(query string) (*loggedStmt, error) + // Query executes a query that returns rows, typically a SELECT. The + // args are for any placeholder parameters in the query. + Query(query string, args ...any) (*loggedRows, error) + // QueryRow executes a query that is expected to return at most one row. + // QueryRow always returns a non-nil value. Errors are deferred until + // Row's Scan method is called. If the query selects no rows, the *Row's + // Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the + // first selected row and discards the rest. + QueryRow(query string, args ...any) *loggedRow + } + + loggedStmt struct { + *sql.Stmt + query string + log *zap.Logger + } + + loggedTxn struct { + *sql.Tx + log *zap.Logger + } + + loggedRow struct { + *sql.Row + log *zap.Logger + } + + loggedRows struct { + *sql.Rows + log *zap.Logger + } +) + +func (lr *loggedRows) Next() bool { + start := time.Now() + next := lr.Rows.Next() + if dur := time.Since(start); dur > longQueryDuration { + lr.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return next +} + +func (lr *loggedRows) Scan(dest ...any) error { + start := time.Now() + err := lr.Rows.Scan(dest...) + if dur := time.Since(start); dur > longQueryDuration { + lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return err +} + +func (lr *loggedRow) Scan(dest ...any) error { + start := time.Now() + err := lr.Row.Scan(dest...) + if dur := time.Since(start); dur > longQueryDuration { + lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return err +} + +func (ls *loggedStmt) Exec(args ...any) (sql.Result, error) { + return ls.ExecContext(context.Background(), args...) +} + +func (ls *loggedStmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { + start := time.Now() + result, err := ls.Stmt.ExecContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + ls.log.Debug("slow exec", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return result, err +} + +func (ls *loggedStmt) Query(args ...any) (*sql.Rows, error) { + return ls.QueryContext(context.Background(), args...) +} + +func (ls *loggedStmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { + start := time.Now() + rows, err := ls.Stmt.QueryContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + ls.log.Debug("slow query", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return rows, err +} + +func (ls *loggedStmt) QueryRow(args ...any) *loggedRow { + return ls.QueryRowContext(context.Background(), args...) +} + +func (ls *loggedStmt) QueryRowContext(ctx context.Context, args ...any) *loggedRow { + start := time.Now() + row := ls.Stmt.QueryRowContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + ls.log.Debug("slow query row", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &loggedRow{row, ls.log.Named("row")} +} + +// Exec executes a query without returning any rows. The args are for +// any placeholder parameters in the query. +func (lt *loggedTxn) Exec(query string, args ...any) (sql.Result, error) { + start := time.Now() + result, err := lt.Tx.Exec(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + lt.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return result, err +} + +// Prepare creates a prepared statement for later queries or executions. +// Multiple queries or executions may be run concurrently from the +// returned statement. The caller must call the statement's Close method +// when the statement is no longer needed. +func (lt *loggedTxn) Prepare(query string) (*loggedStmt, error) { + start := time.Now() + stmt, err := lt.Tx.Prepare(query) + if dur := time.Since(start); dur > longQueryDuration { + lt.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } else if err != nil { + return nil, err + } + return &loggedStmt{ + Stmt: stmt, + query: query, + log: lt.log.Named("statement"), + }, nil +} + +// Query executes a query that returns rows, typically a SELECT. The +// args are for any placeholder parameters in the query. +func (lt *loggedTxn) Query(query string, args ...any) (*loggedRows, error) { + start := time.Now() + rows, err := lt.Tx.Query(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + lt.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &loggedRows{rows, lt.log.Named("rows")}, err +} + +// QueryRow executes a query that is expected to return at most one row. +// QueryRow always returns a non-nil value. Errors are deferred until +// Row's Scan method is called. If the query selects no rows, the *Row's +// Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the +// first selected row and discards the rest. +func (lt *loggedTxn) QueryRow(query string, args ...any) *loggedRow { + start := time.Now() + row := lt.Tx.QueryRow(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + lt.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &loggedRow{row, lt.log.Named("row")} +} + +func queryPlaceHolders(n int) string { + if n == 0 { + return "" + } else if n == 1 { + return "?" + } + var b strings.Builder + b.Grow(((n - 1) * 2) + 1) // ?,? + for i := 0; i < n-1; i++ { + b.WriteString("?,") + } + b.WriteString("?") + return b.String() +} + +func queryArgs[T any](args []T) []any { + if len(args) == 0 { + return nil + } + out := make([]any, len(args)) + for i, arg := range args { + out[i] = arg + } + return out +} + +// getDBVersion returns the current version of the database. +func getDBVersion(db *sql.DB) (version int64) { + // error is ignored -- the database may not have been initialized yet. + db.QueryRow(`SELECT db_version FROM global_settings;`).Scan(&version) + return +} + +// setDBVersion sets the current version of the database. +func setDBVersion(tx txn, version int64) error { + const query = `UPDATE global_settings SET db_version=$1 RETURNING id;` + var dbID int64 + return tx.QueryRow(query, version).Scan(&dbID) +} + +// jitterSleep sleeps for a random duration between t and t*1.5. +func jitterSleep(t time.Duration) { + time.Sleep(t + time.Duration(rand.Int63n(int64(t/2)))) +} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go new file mode 100644 index 0000000..0db301e --- /dev/null +++ b/persist/sqlite/store.go @@ -0,0 +1,120 @@ +package sqlite + +import ( + "database/sql" + "encoding/hex" + "fmt" + "math" + "strings" + "time" + + "go.sia.tech/coreutils/chain" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +type ( + // A Store is a persistent store that uses a SQL database as its backend. + Store struct { + db *sql.DB + log *zap.Logger + + updates []*chain.ApplyUpdate + } +) + +// transaction executes a function within a database transaction. If the +// function returns an error, the transaction is rolled back. Otherwise, the +// transaction is committed. If the transaction fails due to a busy error, it is +// retried up to 10 times before returning. +func (s *Store) transaction(fn func(txn) error) error { + var err error + txnID := hex.EncodeToString(frand.Bytes(4)) + log := s.log.Named("transaction").With(zap.String("id", txnID)) + start := time.Now() + attempt := 1 + for ; attempt < maxRetryAttempts; attempt++ { + attemptStart := time.Now() + log := log.With(zap.Int("attempt", attempt)) + err = doTransaction(s.db, log, fn) + if err == nil { + // no error, break out of the loop + return nil + } + + // return immediately if the error is not a busy error + if !strings.Contains(err.Error(), "database is locked") { + break + } + // exponential backoff + sleep := time.Duration(math.Pow(factor, float64(attempt))) * time.Millisecond + if sleep > maxBackoff { + sleep = maxBackoff + } + log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) + jitterSleep(sleep) + } + return fmt.Errorf("transaction failed (attempt %d): %w", attempt, err) +} + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +func sqliteFilepath(fp string) string { + params := []string{ + fmt.Sprintf("_busy_timeout=%d", busyTimeout), + "_foreign_keys=true", + "_journal_mode=WAL", + "_secure_delete=false", + "_cache_size=-65536", // 64MiB + } + return "file:" + fp + "?" + strings.Join(params, "&") +} + +// doTransaction is a helper function to execute a function within a transaction. If fn returns +// an error, the transaction is rolled back. Otherwise, the transaction is +// committed. +func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx txn) error) error { + start := time.Now() + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() + defer func() { + // log the transaction if it took longer than txn duration + if time.Since(start) > longTxnDuration { + log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) + } + }() + + ltx := &loggedTxn{ + Tx: tx, + log: log, + } + if err = fn(ltx); err != nil { + return err + } else if err = tx.Commit(); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + return nil +} + +// OpenDatabase creates a new SQLite store and initializes the database. If the +// database does not exist, it is created. +func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { + db, err := sql.Open("sqlite3", sqliteFilepath(fp)) + if err != nil { + return nil, err + } + store := &Store{ + db: db, + log: log, + } + if err := store.init(); err != nil { + return nil, fmt.Errorf("failed to initialize database: %w", err) + } + return store, nil +} diff --git a/persist/sqlite/types.go b/persist/sqlite/types.go new file mode 100644 index 0000000..44f1f10 --- /dev/null +++ b/persist/sqlite/types.go @@ -0,0 +1,135 @@ +package sqlite + +import ( + "bytes" + "database/sql" + "database/sql/driver" + "encoding/binary" + "fmt" + "io" + "time" + + "go.sia.tech/core/types" +) + +type ( + sqlCurrency types.Currency + sqlTime time.Time +) + +// Scan implements the sql.Scanner interface. +func (sc *sqlCurrency) Scan(src any) error { + buf, ok := src.([]byte) + if !ok { + return fmt.Errorf("cannot scan %T to Currency", src) + } else if len(buf) != 16 { + return fmt.Errorf("cannot scan %d bytes to Currency", len(buf)) + } + + sc.Lo = binary.LittleEndian.Uint64(buf[:8]) + sc.Hi = binary.LittleEndian.Uint64(buf[8:]) + return nil +} + +// Value implements the driver.Valuer interface. +func (sc sqlCurrency) Value() (driver.Value, error) { + buf := make([]byte, 16) + binary.LittleEndian.PutUint64(buf[:8], sc.Lo) + binary.LittleEndian.PutUint64(buf[8:], sc.Hi) + return buf, nil +} + +func (st *sqlTime) Scan(src any) error { + switch src := src.(type) { + case int64: + *st = sqlTime(time.Unix(src, 0)) + return nil + default: + return fmt.Errorf("cannot scan %T to Time", src) + } +} + +func (st sqlTime) Value() (driver.Value, error) { + return time.Time(st).Unix(), nil +} + +func encode[T types.EncoderTo](v T) []byte { + var buf bytes.Buffer + enc := types.NewEncoder(&buf) + v.EncodeTo(enc) + if err := enc.Flush(); err != nil { + panic(err) + } + return buf.Bytes() +} + +func encodeSlice[T types.EncoderTo](v []T) []byte { + var buf bytes.Buffer + enc := types.NewEncoder(&buf) + enc.WritePrefix(len(v)) + for _, e := range v { + e.EncodeTo(enc) + } + if err := enc.Flush(); err != nil { + panic(err) + } + return buf.Bytes() +} + +type decodableSlice[T any] struct { + v *[]T + n int64 +} + +func (d *decodableSlice[T]) Scan(src any) error { + switch src := src.(type) { + case []byte: + dec := types.NewDecoder(io.LimitedReader{ + R: bytes.NewReader(src), + N: d.n, + }) + s := make([]T, dec.ReadPrefix()) + for i := range s { + dv, ok := any(&s[i]).(types.DecoderFrom) + if !ok { + panic(fmt.Errorf("cannot decode %T", s[i])) + } + dv.DecodeFrom(dec) + } + if err := dec.Err(); err != nil { + return err + } + *d.v = s + return nil + default: + return fmt.Errorf("cannot scan %T to []byte", src) + } +} + +func decodeSlice[T any](v *[]T, maxLen int64) sql.Scanner { + return &decodableSlice[T]{v: v, n: maxLen} +} + +type decodable[T types.DecoderFrom] struct { + v T + n int64 +} + +func (d *decodable[T]) Scan(src any) error { + switch src := src.(type) { + case []byte: + dec := types.NewDecoder(io.LimitedReader{ + R: bytes.NewReader(src), + N: d.n, + }) + + d.v.DecodeFrom(dec) + return dec.Err() + default: + return fmt.Errorf("cannot scan %T to []byte", src) + } +} + +func decode[T types.DecoderFrom](v T, maxLen int64) sql.Scanner { + return &decodable[T]{v, maxLen} +} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go new file mode 100644 index 0000000..56ba3fd --- /dev/null +++ b/persist/sqlite/wallet.go @@ -0,0 +1,292 @@ +package sqlite + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +func insertAddress(tx txn, addr types.Address) (id int64, err error) { + const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) +VALUES ($1, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address +RETURNING id` + + err = tx.QueryRow(query, encode(addr), (*sqlCurrency)(&types.ZeroCurrency)).Scan(&id) + return +} + +func (s *Store) WalletEvents(walletID string, offset, limit int) (events []wallet.Event, err error) { + err = s.transaction(func(tx txn) error { + const query = `SELECT ev.id, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data +FROM events ev +INNER JOIN chain_indices ci ON (ev.index_id = ci.id) +WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) +ORDER BY ci.height DESC, ev.id ASC +LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, walletID, limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var eventID int64 + var event wallet.Event + var eventType string + var eventBuf []byte + + err := rows.Scan(&eventID, (*sqlTime)(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID, 32), &eventType, &eventBuf) + if err != nil { + return fmt.Errorf("failed to scan event: %w", err) + } + + switch eventType { + case wallet.EventTypeTransaction: + var tx wallet.EventTransaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + event.Val = &tx + case wallet.EventTypeMissedFileContract: + var m wallet.EventMissedFileContract + if err = json.Unmarshal(eventBuf, &m); err != nil { + return fmt.Errorf("failed to unmarshal missed file contract event: %w", err) + } + event.Val = &m + case wallet.EventTypeMinerPayout: + var m wallet.EventMinerPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return fmt.Errorf("failed to unmarshal payout event: %w", err) + } + event.Val = &m + default: + return fmt.Errorf("unknown event type: %s", eventType) + } + + // event.Relevant = relevantAddresses[eventID] + events = append(events, event) + } + return nil + }) + return +} + +func (s *Store) AddWallet(name string, info json.RawMessage) error { + return s.transaction(func(tx txn) error { + const query = `INSERT INTO wallets (id, extra_data) VALUES ($1, $2)` + + _, err := tx.Exec(query, name, info) + if err != nil { + return fmt.Errorf("failed to insert wallet: %w", err) + } + return nil + }) +} + +func (s *Store) DeleteWallet(name string) error { + return s.transaction(func(tx txn) error { + _, err := tx.Exec(`DELETE FROM wallets WHERE id=$1`, name) + return err + }) +} + +func (s *Store) Wallets() (map[string]json.RawMessage, error) { + wallets := make(map[string]json.RawMessage) + err := s.transaction(func(tx txn) error { + const query = `SELECT id, extra_data FROM wallets` + + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var friendlyName string + var extraData json.RawMessage + if err := rows.Scan(&friendlyName, &extraData); err != nil { + return fmt.Errorf("failed to scan wallet: %w", err) + } + wallets[friendlyName] = extraData + } + return nil + }) + return wallets, err +} + +func (s *Store) AddAddress(walletID string, address types.Address, info json.RawMessage) error { + return s.transaction(func(tx txn) error { + addressID, err := insertAddress(tx, address) + if err != nil { + return fmt.Errorf("failed to insert address: %w", err) + } + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, extra_data, address_id) VALUES ($1, $2, $3)`, walletID, info, addressID) + return err + }) +} + +func (s *Store) RemoveAddress(walletID string, address types.Address) error { + return s.transaction(func(tx txn) error { + const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2)` + _, err := tx.Exec(query, walletID, encode(address)) + return err + }) +} + +func (s *Store) Addresses(walletID string) (map[types.Address]json.RawMessage, error) { + addresses := make(map[types.Address]json.RawMessage) + err := s.transaction(func(tx txn) error { + const query = `SELECT sa.sia_address, wa.extra_data +FROM wallet_addresses wa +INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) +WHERE wa.wallet_id=$1` + + rows, err := tx.Query(query, walletID) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var address types.Address + var extraData json.RawMessage + if err := rows.Scan(decode(&address, 32), &extraData); err != nil { + return fmt.Errorf("failed to scan address: %w", err) + } + addresses[address] = extraData + } + return nil + }) + return addresses, err +} + +func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.SiacoinElement, err error) { + err = s.transaction(func(tx txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)` + + rows, err := tx.Query(query, walletID) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var siacoin types.SiacoinElement + var proof []byte + + err := rows.Scan(decode(&siacoin.ID, 32), &siacoin.LeafIndex, &proof, (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address, 32), &siacoin.MaturityHeight) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + siacoins = append(siacoins, siacoin) + } + return nil + }) + return +} + +func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.SiafundElement, err error) { + err = s.transaction(func(tx txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)` + + rows, err := tx.Query(query, walletID) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var siafund types.SiafundElement + var proof []byte + + err := rows.Scan(decode(&siafund.ID, 32), &siafund.LeafIndex, &proof, &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address, 32)) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + siafunds = append(siafunds, siafund) + } + return nil + }) + return +} + +// WalletBalance returns the total balance of a wallet. +func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) { + err = s.transaction(func(tx txn) error { + const query = `SELECT siacoin_balance, siafund_balance FROM sia_addresses sa + INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) + WHERE wa.wallet_id=$1` + + rows, err := tx.Query(query, walletID) + if err != nil { + return err + } + + for rows.Next() { + var siacoin types.Currency + var siafund uint64 + + if err := rows.Scan((*sqlCurrency)(&siacoin), &siafund); err != nil { + return fmt.Errorf("failed to scan address balance: %w", err) + } + sc = sc.Add(siacoin) + sf += siafund + } + return nil + }) + return +} + +// AddressBalance returns the balance of a single address. +func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { + err = s.transaction(func(tx txn) error { + const query = `SELECT siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` + return tx.QueryRow(query, encode(address)).Scan((*sqlCurrency)(&sc), &sf) + }) + return +} + +func (s *Store) Annotate(walletID string, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { + err = s.transaction(func(tx txn) error { + stmt, err := tx.Prepare(`SELECT sia_address FROM wallet_addresses WHERE wallet_id=$1 AND sia_address=$2 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + // note: this would be more performant for small wallets to load all + // addresses into memory. However, for larger wallets (> 10K addresses), + // this is time consuming. Instead, the database is queried for each + // address. Monitor performance and consider changing this in the + // future. From a memory perspective, it would be fine to lazy load all + // addresses into memory. + ownsAddress := func(address types.Address) bool { + var dbID int64 + err := stmt.QueryRow(walletID, encode(address)).Scan(dbID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + panic(err) // database error + } + return err == nil + } + + for _, txn := range txns { + ptxn := wallet.Annotate(txn, ownsAddress) + if ptxn.Type != "unrelated" { + annotated = append(annotated, ptxn) + } + } + return nil + }) + return +} diff --git a/wallet/state.go b/wallet/state.go new file mode 100644 index 0000000..219a7f2 --- /dev/null +++ b/wallet/state.go @@ -0,0 +1,83 @@ +package wallet + +import ( + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" +) + +// A Midstate is a snapshot of unapplied consensus changes. +type Midstate struct { + SpentSiacoinOutputs map[types.Hash256]bool + SpentSiafundOutputs map[types.Hash256]bool + + NewSiacoinOutputs map[types.Hash256]types.SiacoinElement + NewSiafundOutputs map[types.Hash256]types.SiafundElement + + Events []Event +} + +func (ms *Midstate) Apply(cau *chain.ApplyUpdate, ownsAddress func(types.Address) bool) { + events := AppliedEvents(cau.State, cau.Block, cau, ownsAddress) + ms.Events = append(ms.Events, events...) + + cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if !ownsAddress(se.SiacoinOutput.Address) { + return + } + + if spent { + ms.SpentSiacoinOutputs[se.ID] = true + delete(ms.NewSiacoinOutputs, se.ID) + } else { + ms.NewSiacoinOutputs[se.ID] = se + } + }) + + cau.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { + if !ownsAddress(sf.SiafundOutput.Address) { + return + } + + if spent { + ms.SpentSiafundOutputs[sf.ID] = true + delete(ms.NewSiafundOutputs, sf.ID) + } else { + ms.NewSiafundOutputs[sf.ID] = sf + } + }) +} + +func (ms *Midstate) Revert(cru *chain.RevertUpdate, ownsAddress func(types.Address) bool) { + revertedBlockID := cru.Block.ID() + for i := len(ms.Events) - 1; i >= 0; i-- { + // working backwards, revert all events until the block ID no longer + // matches. + if ms.Events[i].Index.ID != revertedBlockID { + break + } + ms.Events = ms.Events[:i] + } + + cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if !ownsAddress(se.SiacoinOutput.Address) { + return + } + + if !spent { + delete(ms.SpentSiacoinOutputs, se.ID) + } + }) + + cru.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { + if !ownsAddress(sf.SiafundOutput.Address) { + return + } + + if spent { + ms.SpentSiafundOutputs[sf.ID] = true + delete(ms.NewSiafundOutputs, sf.ID) + } else { + ms.NewSiafundOutputs[sf.ID] = sf + } + }) +} diff --git a/wallet/wallet.go b/wallet/wallet.go index ecc9754..d1a458c 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -9,6 +9,13 @@ import ( "go.sia.tech/core/types" ) +const ( + // transactions + EventTypeTransaction = "transaction" + EventTypeMinerPayout = "miner payout" + EventTypeMissedFileContract = "missed file contract" +) + // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. @@ -143,12 +150,12 @@ type Event struct { Index types.ChainIndex Timestamp time.Time Relevant []types.Address - Val interface{ eventType() string } + Val interface{ EventType() string } } -func (*EventTransaction) eventType() string { return "transaction" } -func (*EventMinerPayout) eventType() string { return "miner payout" } -func (*EventMissedFileContract) eventType() string { return "missed file contract" } +func (*EventTransaction) EventType() string { return EventTypeTransaction } +func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } +func (*EventMissedFileContract) EventType() string { return EventTypeMissedFileContract } // MarshalJSON implements json.Marshaler. func (e Event) MarshalJSON() ([]byte, error) { @@ -163,7 +170,7 @@ func (e Event) MarshalJSON() ([]byte, error) { Timestamp: e.Timestamp, Index: e.Index, Relevant: e.Relevant, - Type: e.Val.eventType(), + Type: e.Val.EventType(), Val: val, }) } @@ -184,11 +191,11 @@ func (e *Event) UnmarshalJSON(data []byte) error { e.Index = s.Index e.Relevant = s.Relevant switch s.Type { - case (*EventTransaction)(nil).eventType(): + case (*EventTransaction)(nil).EventType(): e.Val = new(EventTransaction) - case (*EventMinerPayout)(nil).eventType(): + case (*EventMinerPayout)(nil).EventType(): e.Val = new(EventMinerPayout) - case (*EventMissedFileContract)(nil).eventType(): + case (*EventMissedFileContract)(nil).EventType(): e.Val = new(EventMissedFileContract) } if e.Val == nil { @@ -259,7 +266,7 @@ type ChainUpdate interface { // AppliedEvents extracts a list of relevant events from a chain update. func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { var events []Event - addEvent := func(v interface{ eventType() string }, relevant []types.Address) { + addEvent := func(v interface{ EventType() string }, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] From 46db8cc74b1f858fd1898d32c7d6d364d969696c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 9 Jan 2024 15:50:09 -0800 Subject: [PATCH 056/630] wallet: add manager --- wallet/manager.go | 173 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 wallet/manager.go diff --git a/wallet/manager.go b/wallet/manager.go new file mode 100644 index 0000000..e4a2380 --- /dev/null +++ b/wallet/manager.go @@ -0,0 +1,173 @@ +package wallet + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "go.sia.tech/core/chain" + "go.sia.tech/core/types" + "go.uber.org/zap" +) + +type ( + ChainManager interface { + AddSubscriber(chain.Subscriber, types.ChainIndex) error + RemoveSubscriber(chain.Subscriber) + + BestIndex(height uint64) (types.ChainIndex, bool) + } + + Store interface { + chain.Subscriber + + WalletEvents(name string, offset, limit int) ([]Event, error) + AddWallet(name string, info json.RawMessage) error + DeleteWallet(name string) error + Wallets() (map[string]json.RawMessage, error) + + AddAddress(walletID string, address types.Address, info json.RawMessage) error + RemoveAddress(walletID string, address types.Address) error + Addresses(walletID string) (map[types.Address]json.RawMessage, error) + UnspentSiacoinOutputs(walletID string) ([]types.SiacoinElement, error) + UnspentSiafundOutputs(walletID string) ([]types.SiafundElement, error) + Annotate(walletID string, txns []types.Transaction) ([]PoolTransaction, error) + WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) + + AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) + + LastCommittedIndex() (types.ChainIndex, error) + } + + // A Manager manages wallets. + Manager struct { + chain ChainManager + store Store + log *zap.Logger + + mu sync.Mutex + used map[types.Hash256]bool + } +) + +// AddWallet adds the given wallet. +func (m *Manager) AddWallet(name string, info json.RawMessage) error { + return m.store.AddWallet(name, info) +} + +// DeleteWallet deletes the given wallet. +func (m *Manager) DeleteWallet(name string) error { + return m.store.DeleteWallet(name) +} + +// Wallets returns the wallets of the wallet manager. +func (m *Manager) Wallets() (map[string]json.RawMessage, error) { + return m.store.Wallets() +} + +// AddAddress adds the given address to the given wallet. +func (m *Manager) AddAddress(name string, addr types.Address, info json.RawMessage) error { + return m.store.AddAddress(name, addr, info) +} + +// RemoveAddress removes the given address from the given wallet. +func (m *Manager) RemoveAddress(name string, addr types.Address) error { + return m.store.RemoveAddress(name, addr) +} + +// Addresses returns the addresses of the given wallet. +func (m *Manager) Addresses(name string) (map[types.Address]json.RawMessage, error) { + return m.store.Addresses(name) +} + +// Events returns the events of the given wallet. +func (m *Manager) Events(name string, offset, limit int) ([]Event, error) { + return m.store.WalletEvents(name, offset, limit) +} + +// UnspentSiacoinOutputs returns the unspent siacoin outputs of the given wallet +func (m *Manager) UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) { + return m.store.UnspentSiacoinOutputs(name) +} + +// UnspentSiafundOutputs returns the unspent siafund outputs of the given wallet +func (m *Manager) UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) { + return m.store.UnspentSiafundOutputs(name) +} + +// Annotate annotates the given transactions with the wallet they belong to. +func (m *Manager) Annotate(name string, pool []types.Transaction) ([]PoolTransaction, error) { + return m.store.Annotate(name, pool) +} + +// WalletBalance returns the balance of the given wallet. +func (m *Manager) WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) { + return m.store.WalletBalance(walletID) +} + +// AddressBalance returns the balance of the given address. +func (m *Manager) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { + return m.store.AddressBalance(address) +} + +// Reserve reserves the given ids for the given duration. +func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + + // check if any of the ids are already reserved + for _, id := range ids { + if m.used[id] { + return fmt.Errorf("output %q already reserved", id) + } + } + + // reserve the ids + for _, id := range ids { + m.used[id] = true + } + + // sleep for the duration and then unreserve the ids + time.AfterFunc(duration, func() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, id := range ids { + delete(m.used, id) + } + }) + return nil +} + +// Subscribe resubscribes the indexer starting at the given height. +func (m *Manager) Subscribe(startHeight uint64) error { + var index types.ChainIndex + if startHeight > 0 { + var ok bool + index, ok = m.chain.BestIndex(startHeight - 1) + if !ok { + return errors.New("invalid height") + } + } + m.chain.RemoveSubscriber(m.store) + return m.chain.AddSubscriber(m.store, index) +} + +// NewManager creates a new wallet manager. +func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) { + m := &Manager{ + chain: cm, + store: store, + log: log, + } + + lastTip, err := store.LastCommittedIndex() + if err != nil { + return nil, fmt.Errorf("failed to get last committed index: %w", err) + } else if err := cm.AddSubscriber(store, lastTip); err != nil { + return nil, fmt.Errorf("failed to subscribe to chain manager: %w", err) + } + return m, nil +} From 591f38e8ccc7444d19732de0d8e79913678c8d53 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 9 Jan 2024 15:50:18 -0800 Subject: [PATCH 057/630] api, cmd: use sqlite3 store --- api/api_test.go | 64 ++++++++++++++++++++++++------ api/client.go | 14 +++---- api/server.go | 97 ++++++++++++++++++--------------------------- cmd/walletd/main.go | 27 ++++++++++++- cmd/walletd/node.go | 37 ++++++++++------- wallet/manager.go | 2 +- 6 files changed, 145 insertions(+), 96 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 37ad08d..9ffa789 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -3,6 +3,7 @@ package api_test import ( "net" "net/http" + "path/filepath" "testing" "time" @@ -14,8 +15,9 @@ import ( "go.sia.tech/jape" "go.sia.tech/walletd/api" "go.sia.tech/walletd/internal/syncerutil" - "go.sia.tech/walletd/internal/walletutil" + "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) @@ -48,6 +50,8 @@ func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Cl } func TestWallet(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testNetwork() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) @@ -62,7 +66,17 @@ func TestWallet(t *testing.T) { t.Fatal(err) } cm := chain.NewManager(dbstore, tipState) - wm := walletutil.NewEphemeralWalletManager(cm) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) defer shutdown() @@ -70,7 +84,7 @@ func TestWallet(t *testing.T) { t.Fatal(err) } wc := c.Wallet("primary") - if err := wc.Subscribe(0); err != nil { + if err := c.Resubscribe(0); err != nil { t.Fatal(err) } @@ -153,7 +167,7 @@ func TestWallet(t *testing.T) { } // transaction should appear in history - events, err = wc.Events(0, -1) + events, err = wc.Events(0, 100) if err != nil { t.Fatal(err) } else if len(events) == 0 { @@ -169,6 +183,8 @@ func TestWallet(t *testing.T) { } func TestV2(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testNetwork() // gift primary wallet some coins primaryPrivateKey := types.GeneratePrivateKey() @@ -184,7 +200,15 @@ func TestV2(t *testing.T) { t.Fatal(err) } cm := chain.NewManager(dbstore, tipState) - wm := walletutil.NewEphemeralWalletManager(cm) + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } c, shutdown := runServer(cm, nil, wm) defer shutdown() if err := c.AddWallet("primary", nil); err != nil { @@ -194,9 +218,6 @@ func TestV2(t *testing.T) { if err := primary.AddAddress(primaryAddress, nil); err != nil { t.Fatal(err) } - if err := primary.Subscribe(0); err != nil { - t.Fatal(err) - } if err := c.AddWallet("secondary", nil); err != nil { t.Fatal(err) } @@ -204,7 +225,7 @@ func TestV2(t *testing.T) { if err := secondary.AddAddress(secondaryAddress, nil); err != nil { t.Fatal(err) } - if err := secondary.Subscribe(0); err != nil { + if err := c.Resubscribe(0); err != nil { t.Fatal(err) } @@ -373,6 +394,7 @@ func TestV2(t *testing.T) { } func TestP2P(t *testing.T) { + log := zaptest.NewLogger(t) n, genesisBlock := testNetwork() // gift primary wallet some coins primaryPrivateKey := types.GeneratePrivateKey() @@ -388,7 +410,15 @@ func TestP2P(t *testing.T) { t.Fatal(err) } cm1 := chain.NewManager(dbstore1, tipState) - wm1 := walletutil.NewEphemeralWalletManager(cm1) + ws1, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws1.Close() + wm1, err := wallet.NewManager(cm1, ws1, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } l1, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) @@ -409,7 +439,7 @@ func TestP2P(t *testing.T) { if err := primary.AddAddress(primaryAddress, nil); err != nil { t.Fatal(err) } - if err := primary.Subscribe(0); err != nil { + if err := c1.Resubscribe(0); err != nil { t.Fatal(err) } @@ -418,7 +448,15 @@ func TestP2P(t *testing.T) { t.Fatal(err) } cm2 := chain.NewManager(dbstore2, tipState) - wm2 := walletutil.NewEphemeralWalletManager(cm2) + ws2, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws2.Close() + wm2, err := wallet.NewManager(cm2, ws2, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } l2, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) @@ -439,7 +477,7 @@ func TestP2P(t *testing.T) { if err := secondary.AddAddress(secondaryAddress, nil); err != nil { t.Fatal(err) } - if err := secondary.Subscribe(0); err != nil { + if err := c2.Resubscribe(0); err != nil { t.Fatal(err) } diff --git a/api/client.go b/api/client.go index 973f194..3258d9e 100644 --- a/api/client.go +++ b/api/client.go @@ -105,6 +105,13 @@ func (c *Client) Wallet(name string) *WalletClient { return &WalletClient{c: c.c, name: name} } +// Resubscribe subscribes the wallet to consensus updates, starting at the +// specified height. This can only be done once. +func (c *Client) Resubscribe(height uint64) (err error) { + err = c.c.POST("/resubscribe", height, nil) + return +} + // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { @@ -112,13 +119,6 @@ type WalletClient struct { name string } -// Subscribe subscribes the wallet to consensus updates, starting at the -// specified height. This can only be done once. -func (c *WalletClient) Subscribe(height uint64) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/subscribe", c.name), height, nil) - return -} - // AddAddress adds the specified address and associated metadata to the // wallet. func (c *WalletClient) AddAddress(addr types.Address, info json.RawMessage) (err error) { diff --git a/api/server.go b/api/server.go index 898ba29..f7c87d7 100644 --- a/api/server.go +++ b/api/server.go @@ -3,7 +3,6 @@ package api import ( "encoding/json" "errors" - "fmt" "net/http" "reflect" "sync" @@ -46,17 +45,23 @@ type ( // A WalletManager manages wallets, keyed by name. WalletManager interface { + Subscribe(startHeight uint64) error + AddWallet(name string, info json.RawMessage) error DeleteWallet(name string) error - Wallets() map[string]json.RawMessage - SubscribeWallet(name string, startHeight uint64) error + Wallets() (map[string]json.RawMessage, error) AddAddress(name string, addr types.Address, info json.RawMessage) error RemoveAddress(name string, addr types.Address) error Addresses(name string) (map[types.Address]json.RawMessage, error) Events(name string, offset, limit int) ([]wallet.Event, error) - UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) + UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) + UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) + WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) + + Reserve(ids []types.Hash256, duration time.Duration) error + AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) } ) @@ -165,7 +170,11 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } func (s *server) walletsHandler(jc jape.Context) { - jc.Encode(s.wm.Wallets()) + wallets, err := s.wm.Wallets() + if jc.Check("couldn't load wallets", err) != nil { + return + } + jc.Encode(wallets) } func (s *server) walletsNameHandlerPUT(jc jape.Context) { @@ -187,12 +196,11 @@ func (s *server) walletsNameHandlerDELETE(jc jape.Context) { } } -func (s *server) walletsSubscribeHandler(jc jape.Context) { - var name string +func (s *server) resubscribeHandler(jc jape.Context) { var height uint64 - if jc.DecodeParam("name", &name) != nil || jc.Decode(&height) != nil { + if jc.Decode(&height) != nil { return - } else if jc.Check("couldn't subscribe wallet", s.wm.SubscribeWallet(name, height)) != nil { + } else if jc.Check("couldn't subscribe wallet", s.wm.Subscribe(height)) != nil { return } } @@ -235,26 +243,14 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { if jc.DecodeParam("name", &name) != nil { return } - scos, sfos, err := s.wm.UnspentOutputs(name) - if jc.Check("couldn't load outputs", err) != nil { + + sc, sf, err := s.wm.WalletBalance(name) + if jc.Check("couldn't load balance", err) != nil { return } - height := s.cm.TipState().Index.Height - var sc, immature types.Currency - var sf uint64 - for _, sco := range scos { - if height >= sco.MaturityHeight { - sc = sc.Add(sco.SiacoinOutput.Value) - } else { - immature = immature.Add(sco.SiacoinOutput.Value) - } - } - for _, sfo := range sfos { - sf += sfo.SiafundOutput.Value - } jc.Encode(WalletBalanceResponse{ Siacoins: sc, - ImmatureSiacoins: immature, + ImmatureSiacoins: types.ZeroCurrency, Siafunds: sf, }) } @@ -289,8 +285,13 @@ func (s *server) walletsOutputsHandler(jc jape.Context) { if jc.DecodeParam("name", &name) != nil { return } - scos, sfos, err := s.wm.UnspentOutputs(name) - if jc.Check("couldn't load outputs", err) != nil { + scos, err := s.wm.UnspentSiacoinOutputs(name) + if jc.Check("couldn't load siacoin outputs", err) != nil { + return + } + + sfos, err := s.wm.UnspentSiafundOutputs(name) + if jc.Check("couldn't load siafund outputs", err) != nil { return } jc.Encode(WalletOutputsResponse{ @@ -300,44 +301,23 @@ func (s *server) walletsOutputsHandler(jc jape.Context) { } func (s *server) walletsReserveHandler(jc jape.Context) { - var name string var wrr WalletReserveRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wrr) != nil { + if jc.Decode(&wrr) != nil { return } - s.mu.Lock() + ids := make([]types.Hash256, 0, len(wrr.SiacoinOutputs)+len(wrr.SiafundOutputs)) for _, id := range wrr.SiacoinOutputs { - if s.used[types.Hash256(id)] { - s.mu.Unlock() - jc.Error(fmt.Errorf("output %v is already reserved", id), http.StatusBadRequest) - return - } - s.used[types.Hash256(id)] = true + ids = append(ids, types.Hash256(id)) } + for _, id := range wrr.SiafundOutputs { - if s.used[types.Hash256(id)] { - s.mu.Unlock() - jc.Error(fmt.Errorf("output %v is already reserved", id), http.StatusBadRequest) - return - } - s.used[types.Hash256(id)] = true + ids = append(ids, types.Hash256(id)) } - s.mu.Unlock() - if wrr.Duration == 0 { - wrr.Duration = 10 * time.Minute + if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids, wrr.Duration)) != nil { + return } - time.AfterFunc(wrr.Duration, func() { - s.mu.Lock() - defer s.mu.Unlock() - for _, id := range wrr.SiacoinOutputs { - delete(s.used, types.Hash256(id)) - } - for _, id := range wrr.SiafundOutputs { - delete(s.used, types.Hash256(id)) - } - }) } func (s *server) walletsReleaseHandler(jc jape.Context) { @@ -412,7 +392,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, err := s.wm.UnspentOutputs(name) + utxos, err := s.wm.UnspentSiacoinOutputs(name) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -486,7 +466,7 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { return } - _, utxos, err := s.wm.UnspentOutputs(name) + utxos, err := s.wm.UnspentSiafundOutputs(name) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -524,10 +504,11 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, + "POST /resubscribe": srv.resubscribeHandler, + "GET /wallets": srv.walletsHandler, "PUT /wallets/:name": srv.walletsNameHandlerPUT, "DELETE /wallets/:name": srv.walletsNameHandlerDELETE, - "POST /wallets/:name/subscribe": srv.walletsSubscribeHandler, "PUT /wallets/:name/addresses/:addr": srv.walletsAddressHandlerPUT, "DELETE /wallets/:name/addresses/:addr": srv.walletsAddressHandlerDELETE, "GET /wallets/:name/addresses": srv.walletsAddressesHandlerGET, diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index e87d9e8..ae4f3ec 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -11,6 +11,8 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/term" "lukechampine.com/flagg" "lukechampine.com/frand" @@ -162,7 +164,27 @@ func main() { if err != nil { log.Fatal(err) } - n, err := newNode(gatewayAddr, dir, network, upnp) + + // configure console logging note: this is configured before anything else + // to have consistent logging. File logging will be added after the cli + // flags and config is parsed + consoleCfg := zap.NewProductionEncoderConfig() + consoleCfg.TimeKey = "" // prevent duplicate timestamps + consoleCfg.EncodeTime = zapcore.RFC3339TimeEncoder + consoleCfg.EncodeDuration = zapcore.StringDurationEncoder + consoleCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + consoleCfg.StacktraceKey = "" + consoleCfg.CallerKey = "" + consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg) + + // only log info messages to console unless stdout logging is enabled + consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.InfoLevel)) + logger := zap.New(consoleCore, zap.AddCaller()) + defer logger.Sync() + // redirect stdlib log to zap + zap.RedirectStdLog(logger.Named("stdlib")) + + n, err := newNode(gatewayAddr, dir, network, upnp, logger) if err != nil { log.Fatal(err) } @@ -170,6 +192,8 @@ func main() { stop := n.Start() log.Println("api: Listening on", l.Addr()) go startWeb(l, n, apiPassword) + log.Println("api: Listening on", l.Addr()) + go startWeb(l, n, apiPassword) signalCh := make(chan os.Signal, 1) signal.Notify(signalCh, os.Interrupt) <-signalCh @@ -204,7 +228,6 @@ func main() { seed := loadTestnetSeed(seed) c := initTestnetClient(apiAddr, network, seed) runTestnetMiner(c, seed) - case balanceCmd: if len(cmd.Args()) != 0 { cmd.Usage() diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 3249fe6..bd6119b 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -3,7 +3,7 @@ package main import ( "context" "errors" - "log" + "fmt" "net" "path/filepath" "strconv" @@ -16,7 +16,8 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/internal/syncerutil" - "go.sia.tech/walletd/internal/walletutil" + "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/wallet" "go.uber.org/zap" "lukechampine.com/upnp" ) @@ -85,12 +86,12 @@ var anagamiBootstrap = []string{ type node struct { cm *chain.Manager s *syncer.Syncer - wm *walletutil.JSONWalletManager + wm *wallet.Manager Start func() (stop func()) } -func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) { +func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logger) (*node, error) { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -110,11 +111,11 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) if err != nil { - log.Fatal(err) + return nil, fmt.Errorf("failed to open consensus database: %w", err) } dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create chain store: %w", err) } cm := chain.NewManager(dbstore, tipState) @@ -127,21 +128,21 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if d, err := upnp.Discover(ctx); err != nil { - log.Println("WARN: couldn't discover UPnP device:", err) + log.Debug("couldn't discover UPnP router", zap.Error(err)) } else { _, portStr, _ := net.SplitHostPort(addr) port, _ := strconv.Atoi(portStr) if !d.IsForwarded(uint16(port), "TCP") { if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { - log.Println("WARN: couldn't forward port:", err) + log.Debug("couldn't forward port", zap.Error(err)) } else { - log.Println("p2p: Forwarded port", port) + log.Debug("upnp: forwarded p2p port", zap.Int("port", port)) } } if ip, err := d.ExternalIP(); err != nil { - log.Println("WARN: couldn't determine external IP:", err) + log.Debug("couldn't determine external IP", zap.Error(err)) } else { - log.Println("p2p: External IP is", ip) + log.Debug("external IP is", zap.String("ip", ip)) syncerAddr = net.JoinHostPort(ip, portStr) } } @@ -154,7 +155,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) ps, err := syncerutil.NewJSONPeerStore(filepath.Join(dir, "peers.json")) if err != nil { - log.Fatal(err) + return nil, fmt.Errorf("failed to open peer store: %w", err) } for _, peer := range bootstrapPeers { ps.AddPeer(peer) @@ -164,10 +165,16 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(zap.NewNop())) - wm, err := walletutil.NewJSONWalletManager(dir, cm) + s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) + + walletDB, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open wallet database: %w", err) + } + + wm, err := wallet.NewManager(cm, walletDB, log.Named("wallet")) + if err != nil { + return nil, fmt.Errorf("failed to create wallet manager: %w", err) } return &node{ diff --git a/wallet/manager.go b/wallet/manager.go index e4a2380..3e982e5 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -7,8 +7,8 @@ import ( "sync" "time" - "go.sia.tech/core/chain" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.uber.org/zap" ) From 1bec4038e955c29e7256cf807687d1641cb62f10 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 9 Jan 2024 15:50:49 -0800 Subject: [PATCH 058/630] internal: remove ephemeral and JSON wallet store --- internal/walletutil/manager.go | 403 --------------------------------- internal/walletutil/store.go | 402 -------------------------------- 2 files changed, 805 deletions(-) delete mode 100644 internal/walletutil/manager.go delete mode 100644 internal/walletutil/store.go diff --git a/internal/walletutil/manager.go b/internal/walletutil/manager.go deleted file mode 100644 index be5e962..0000000 --- a/internal/walletutil/manager.go +++ /dev/null @@ -1,403 +0,0 @@ -package walletutil - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "sync" - - "go.sia.tech/coreutils/chain" - "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" -) - -var errNoWallet = errors.New("wallet does not exist") - -type ChainManager interface { - AddSubscriber(s chain.Subscriber, tip types.ChainIndex) error - RemoveSubscriber(s chain.Subscriber) - BestIndex(height uint64) (types.ChainIndex, bool) -} - -type managedEphemeralWallet struct { - w *EphemeralStore - info json.RawMessage - subscribed bool -} - -// An EphemeralWalletManager manages multiple ephemeral wallet stores. -type EphemeralWalletManager struct { - cm ChainManager - mu sync.Mutex - wallets map[string]*managedEphemeralWallet -} - -// AddWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) AddWallet(name string, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - if _, ok := wm.wallets[name]; ok { - return errors.New("wallet already exists") - } - store := NewEphemeralStore() - wm.wallets[name] = &managedEphemeralWallet{store, info, false} - return nil -} - -// DeleteWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) DeleteWallet(name string) error { - wm.mu.Lock() - defer wm.mu.Unlock() - delete(wm.wallets, name) - return nil -} - -// Wallets implements api.WalletManager. -func (wm *EphemeralWalletManager) Wallets() map[string]json.RawMessage { - wm.mu.Lock() - defer wm.mu.Unlock() - ws := make(map[string]json.RawMessage, len(wm.wallets)) - for name, w := range wm.wallets { - ws[name] = w.info - } - return ws -} - -// AddAddress implements api.WalletManager. -func (wm *EphemeralWalletManager) AddAddress(name string, addr types.Address, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.AddAddress(addr, info) -} - -// RemoveAddress implements api.WalletManager. -func (wm *EphemeralWalletManager) RemoveAddress(name string, addr types.Address) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.RemoveAddress(addr) -} - -// Addresses implements api.WalletManager. -func (wm *EphemeralWalletManager) Addresses(name string) (map[types.Address]json.RawMessage, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Addresses() -} - -// Events implements api.WalletManager. -func (wm *EphemeralWalletManager) Events(name string, offset, limit int) ([]wallet.Event, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Events(offset, limit) -} - -// Annotate implements api.WalletManager. -func (wm *EphemeralWalletManager) Annotate(name string, txns []types.Transaction) ([]wallet.PoolTransaction, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Annotate(txns), nil -} - -// UnspentOutputs implements api.WalletManager. -func (wm *EphemeralWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, nil, errNoWallet - } - return mw.w.UnspentOutputs() -} - -// SubscribeWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) SubscribeWallet(name string, startHeight uint64) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } else if mw.subscribed { - return errors.New("already subscribed") - } - // AddSubscriber applies each block *after* index, but we want to *include* - // the block at startHeight, so subtract one. - // - // NOTE: if subscribing from height 0, we must pass an empty index in order - // to receive the genesis block. - var index types.ChainIndex - if startHeight > 0 { - if index, ok = wm.cm.BestIndex(startHeight - 1); !ok { - return errors.New("invalid height") - } - } - if err := wm.cm.AddSubscriber(mw.w, index); err != nil { - return err - } - mw.subscribed = true - return nil -} - -// NewEphemeralWalletManager returns a new EphemeralWalletManager. -func NewEphemeralWalletManager(cm ChainManager) *EphemeralWalletManager { - return &EphemeralWalletManager{ - cm: cm, - wallets: make(map[string]*managedEphemeralWallet), - } -} - -type managedJSONWallet struct { - w *JSONStore - info json.RawMessage - subscribed bool -} - -type managerPersistData struct { - Wallets []managerPersistWallet `json:"wallets"` -} - -type managerPersistWallet struct { - Name string `json:"name"` - Info json.RawMessage `json:"info"` - Subscribed bool `json:"subscribed"` -} - -// A JSONWalletManager manages multiple JSON wallet stores. -type JSONWalletManager struct { - dir string - cm ChainManager - mu sync.Mutex - wallets map[string]*managedJSONWallet -} - -func (wm *JSONWalletManager) save() error { - var p managerPersistData - for name, mw := range wm.wallets { - p.Wallets = append(p.Wallets, managerPersistWallet{name, mw.info, mw.subscribed}) - } - js, err := json.MarshalIndent(p, "", " ") - if err != nil { - return err - } - dst := filepath.Join(wm.dir, "wallets.json") - f, err := os.OpenFile(dst+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(dst+"_tmp", dst); err != nil { - return err - } - return nil -} - -func (wm *JSONWalletManager) load() error { - dst := filepath.Join(wm.dir, "wallets.json") - f, err := os.Open(dst) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p managerPersistData - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - for _, pw := range p.Wallets { - wm.wallets[pw.Name] = &managedJSONWallet{nil, pw.Info, pw.Subscribed} - } - return nil -} - -// AddWallet implements api.WalletManager. -func (wm *JSONWalletManager) AddWallet(name string, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - if mw, ok := wm.wallets[name]; ok { - // update existing wallet - mw.info = info - return wm.save() - } else if _, err := os.Stat(filepath.Join(wm.dir, "wallets", name+".json")); err == nil { - // shouldn't happen in normal conditions - return errors.New("a wallet with that name already exists, but is absent from wallets.json") - } - store, _, err := NewJSONStore(filepath.Join(wm.dir, "wallets", name+".json")) - if err != nil { - return err - } - wm.wallets[name] = &managedJSONWallet{store, info, false} - return wm.save() -} - -// DeleteWallet implements api.WalletManager. -func (wm *JSONWalletManager) DeleteWallet(name string) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil - } - wm.cm.RemoveSubscriber(mw.w) - delete(wm.wallets, name) - return os.RemoveAll(filepath.Join(wm.dir, "wallets", name+".json")) -} - -// Wallets implements api.WalletManager. -func (wm *JSONWalletManager) Wallets() map[string]json.RawMessage { - wm.mu.Lock() - defer wm.mu.Unlock() - ws := make(map[string]json.RawMessage, len(wm.wallets)) - for name, w := range wm.wallets { - ws[name] = w.info - } - return ws -} - -// AddAddress implements api.WalletManager. -func (wm *JSONWalletManager) AddAddress(name string, addr types.Address, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.AddAddress(addr, info) -} - -// RemoveAddress implements api.WalletManager. -func (wm *JSONWalletManager) RemoveAddress(name string, addr types.Address) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.RemoveAddress(addr) -} - -// Addresses implements api.WalletManager. -func (wm *JSONWalletManager) Addresses(name string) (map[types.Address]json.RawMessage, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Addresses() -} - -// Events implements api.WalletManager. -func (wm *JSONWalletManager) Events(name string, offset, limit int) ([]wallet.Event, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Events(offset, limit) -} - -// Annotate implements api.WalletManager. -func (wm *JSONWalletManager) Annotate(name string, txns []types.Transaction) ([]wallet.PoolTransaction, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Annotate(txns), nil -} - -// UnspentOutputs implements api.WalletManager. -func (wm *JSONWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, nil, errNoWallet - } - return mw.w.UnspentOutputs() -} - -// SubscribeWallet implements api.WalletManager. -func (wm *JSONWalletManager) SubscribeWallet(name string, startHeight uint64) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } else if mw.subscribed { - return errors.New("already subscribed") - } - // AddSubscriber applies each block *after* index, but we want to *include* - // the block at startHeight, so subtract one. - // - // NOTE: if subscribing from height 0, we must pass an empty index in order - // to receive the genesis block. - var index types.ChainIndex - if startHeight > 0 { - if index, ok = wm.cm.BestIndex(startHeight - 1); !ok { - return errors.New("invalid height") - } - } - if err := wm.cm.AddSubscriber(mw.w, index); err != nil { - return err - } - mw.subscribed = true - return wm.save() -} - -// NewJSONWalletManager returns a wallet manager that stores wallets in the -// specified directory. -func NewJSONWalletManager(dir string, cm ChainManager) (*JSONWalletManager, error) { - wm := &JSONWalletManager{ - dir: dir, - cm: cm, - wallets: make(map[string]*managedJSONWallet), - } - if err := os.MkdirAll(filepath.Join(dir, "wallets"), 0700); err != nil { - return nil, err - } else if err := wm.load(); err != nil { - return nil, err - } - for name, mw := range wm.wallets { - store, tip, err := NewJSONStore(filepath.Join(dir, "wallets", name+".json")) - if err != nil { - return nil, err - } - if mw.subscribed { - if err := cm.AddSubscriber(store, tip); err != nil { - return nil, err - } - } - mw.w = store - } - return wm, nil -} diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go deleted file mode 100644 index 8a2ffef..0000000 --- a/internal/walletutil/store.go +++ /dev/null @@ -1,402 +0,0 @@ -package walletutil - -import ( - "encoding/json" - "fmt" - "os" - "sync" - - "go.sia.tech/coreutils/chain" - "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" -) - -// An EphemeralStore stores wallet state in memory. -type EphemeralStore struct { - tip types.ChainIndex - addrs map[types.Address]json.RawMessage - sces map[types.SiacoinOutputID]types.SiacoinElement - sfes map[types.SiafundOutputID]types.SiafundElement - events []wallet.Event - mu sync.Mutex -} - -func (s *EphemeralStore) ownsAddress(addr types.Address) bool { - _, ok := s.addrs[addr] - return ok -} - -// Events implements api.Wallet. -func (s *EphemeralStore) Events(offset, limit int) (events []wallet.Event, err error) { - s.mu.Lock() - defer s.mu.Unlock() - if limit == -1 { - limit = len(s.events) - } - if offset > len(s.events) { - offset = len(s.events) - } - if offset+limit > len(s.events) { - limit = len(s.events) - offset - } - // reverse - es := make([]wallet.Event, limit) - for i := range es { - es[i] = s.events[len(s.events)-offset-i-1] - } - return es, nil -} - -// Annotate implements api.Wallet. -func (s *EphemeralStore) Annotate(txns []types.Transaction) (ptxns []wallet.PoolTransaction) { - s.mu.Lock() - defer s.mu.Unlock() - for _, txn := range txns { - ptxn := wallet.Annotate(txn, s.ownsAddress) - if ptxn.Type != "unrelated" { - ptxns = append(ptxns, ptxn) - } - } - return -} - -// UnspentOutputs implements api.Wallet. -func (s *EphemeralStore) UnspentOutputs() (sces []types.SiacoinElement, sfes []types.SiafundElement, err error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, sco := range s.sces { - sces = append(sces, sco) - } - for _, sfo := range s.sfes { - sfes = append(sfes, sfo) - } - return -} - -// Addresses implements api.Wallet. -func (s *EphemeralStore) Addresses() (map[types.Address]json.RawMessage, error) { - s.mu.Lock() - defer s.mu.Unlock() - addrs := make(map[types.Address]json.RawMessage, len(s.addrs)) - for addr, info := range s.addrs { - addrs[addr] = info - } - return addrs, nil -} - -// AddAddress implements api.Wallet. -func (s *EphemeralStore) AddAddress(addr types.Address, info json.RawMessage) error { - s.mu.Lock() - defer s.mu.Unlock() - s.addrs[addr] = info - return nil -} - -// RemoveAddress implements api.Wallet. -func (s *EphemeralStore) RemoveAddress(addr types.Address) error { - s.mu.Lock() - defer s.mu.Unlock() - if _, ok := s.addrs[addr]; !ok { - return nil - } - delete(s.addrs, addr) - - // filter outputs - for scoid, sce := range s.sces { - if sce.SiacoinOutput.Address == addr { - delete(s.sces, scoid) - } - } - for sfoid, sfe := range s.sfes { - if sfe.SiafundOutput.Address == addr { - delete(s.sfes, sfoid) - } - } - - // filter events - relevantContract := func(fc types.FileContract) bool { - for _, sco := range fc.ValidProofOutputs { - if s.ownsAddress(sco.Address) { - return true - } - } - for _, sco := range fc.MissedProofOutputs { - if s.ownsAddress(sco.Address) { - return true - } - } - return false - } - relevantV2Contract := func(fc types.V2FileContract) bool { - return s.ownsAddress(fc.RenterOutput.Address) || s.ownsAddress(fc.HostOutput.Address) - } - relevantEvent := func(e wallet.Event) bool { - switch e := e.Val.(type) { - case *wallet.EventTransaction: - for _, sce := range e.SiacoinInputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - for _, sce := range e.SiacoinOutputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - for _, sfe := range e.SiafundInputs { - if s.ownsAddress(sfe.SiafundElement.SiafundOutput.Address) || - s.ownsAddress(sfe.ClaimElement.SiacoinOutput.Address) { - return true - } - } - for _, sfe := range e.SiafundOutputs { - if s.ownsAddress(sfe.SiafundOutput.Address) { - return true - } - } - for _, fc := range e.FileContracts { - if relevantContract(fc.FileContract.FileContract) || (fc.Revision != nil && relevantContract(*fc.Revision)) { - return true - } - } - for _, fc := range e.V2FileContracts { - if relevantV2Contract(fc.FileContract.V2FileContract) || (fc.Revision != nil && relevantV2Contract(*fc.Revision)) { - return true - } - if fc.Resolution != nil { - switch r := fc.Resolution.(type) { - case *types.V2FileContractFinalization: - if relevantV2Contract(types.V2FileContract(*r)) { - return true - } - case *types.V2FileContractRenewal: - if relevantV2Contract(r.FinalRevision) || relevantV2Contract(r.InitialRevision) { - return true - } - } - } - } - return false - case *wallet.EventMinerPayout: - return s.ownsAddress(e.SiacoinOutput.SiacoinOutput.Address) - case *wallet.EventMissedFileContract: - for _, sce := range e.MissedOutputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - return false - default: - panic(fmt.Sprintf("unhandled event type %T", e)) - } - } - - rem := s.events[:0] - for _, e := range s.events { - if relevantEvent(e) { - rem = append(rem, e) - } - } - s.events = rem - return nil -} - -// ProcessChainApplyUpdate implements chain.Subscriber. -func (s *EphemeralStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, _ bool) error { - s.mu.Lock() - defer s.mu.Unlock() - - events := wallet.AppliedEvents(cau.State, cau.Block, cau, s.ownsAddress) - s.events = append(s.events, events...) - - // add/remove outputs - cau.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if s.ownsAddress(sce.SiacoinOutput.Address) { - if spent { - delete(s.sces, types.SiacoinOutputID(sce.ID)) - } else { - sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) - s.sces[types.SiacoinOutputID(sce.ID)] = sce - } - } - }) - cau.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if s.ownsAddress(sfe.SiafundOutput.Address) { - if spent { - delete(s.sfes, types.SiafundOutputID(sfe.ID)) - } else { - sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) - s.sfes[types.SiafundOutputID(sfe.ID)] = sfe - } - } - }) - - // update proofs - for id, sce := range s.sces { - cau.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cau.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - - s.tip = cau.State.Index - return nil -} - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - s.mu.Lock() - defer s.mu.Unlock() - - // terribly inefficient, but not a big deal because reverts are infrequent - numEvents := len(wallet.AppliedEvents(cru.State, cru.Block, cru, s.ownsAddress)) - s.events = s.events[:len(s.events)-numEvents] - - cru.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if s.ownsAddress(sce.SiacoinOutput.Address) { - if !spent { - delete(s.sces, types.SiacoinOutputID(sce.ID)) - } else { - sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) - s.sces[types.SiacoinOutputID(sce.ID)] = sce - } - } - }) - cru.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if s.ownsAddress(sfe.SiafundOutput.Address) { - if !spent { - delete(s.sfes, types.SiafundOutputID(sfe.ID)) - } else { - sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) - s.sfes[types.SiafundOutputID(sfe.ID)] = sfe - } - } - }) - - // update proofs - for id, sce := range s.sces { - cru.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cru.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - - s.tip = cru.State.Index - return nil -} - -// NewEphemeralStore returns a new EphemeralStore. -func NewEphemeralStore() *EphemeralStore { - return &EphemeralStore{ - addrs: make(map[types.Address]json.RawMessage), - sces: make(map[types.SiacoinOutputID]types.SiacoinElement), - sfes: make(map[types.SiafundOutputID]types.SiafundElement), - } -} - -// A JSONStore stores wallet state in memory, backed by a JSON file. -type JSONStore struct { - *EphemeralStore - path string -} - -type persistData struct { - Tip types.ChainIndex - Addresses map[types.Address]json.RawMessage - SiacoinElements map[types.SiacoinOutputID]types.SiacoinElement - SiafundElements map[types.SiafundOutputID]types.SiafundElement - Events []wallet.Event -} - -func (s *JSONStore) save() error { - js, err := json.MarshalIndent(persistData{ - Tip: s.tip, - Addresses: s.addrs, - SiacoinElements: s.sces, - SiafundElements: s.sfes, - Events: s.events, - }, "", " ") - if err != nil { - return err - } - - f, err := os.OpenFile(s.path+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(s.path+"_tmp", s.path); err != nil { - return err - } - return nil -} - -func (s *JSONStore) load() error { - f, err := os.Open(s.path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p persistData - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - s.tip = p.Tip - s.addrs = p.Addresses - s.sces = p.SiacoinElements - s.sfes = p.SiafundElements - s.events = p.Events - return nil -} - -// ProcessChainApplyUpdate implements chain.Subscriber. -func (s *JSONStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { - err := s.EphemeralStore.ProcessChainApplyUpdate(cau, mayCommit) - if err == nil && mayCommit { - err = s.save() - } - return err -} - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (s *JSONStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - return s.EphemeralStore.ProcessChainRevertUpdate(cru) -} - -// AddAddress implements api.Wallet. -func (s *JSONStore) AddAddress(addr types.Address, info json.RawMessage) error { - if err := s.EphemeralStore.AddAddress(addr, info); err != nil { - return err - } - return s.save() -} - -// RemoveAddress implements api.Wallet. -func (s *JSONStore) RemoveAddress(addr types.Address) error { - if err := s.EphemeralStore.RemoveAddress(addr); err != nil { - return err - } - return s.save() -} - -// NewJSONStore returns a new JSONStore. -func NewJSONStore(path string) (*JSONStore, types.ChainIndex, error) { - s := &JSONStore{ - EphemeralStore: NewEphemeralStore(), - path: path, - } - err := s.load() - return s, s.tip, err -} From ffed07b17970672c9efe4beef798621a73c7274e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 9 Jan 2024 16:00:47 -0800 Subject: [PATCH 059/630] cmd: create data directory if it doesn't exist --- cmd/walletd/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index ae4f3ec..477344c 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -159,6 +159,11 @@ func main() { cmd.Usage() return } + + if err := os.MkdirAll(dir, 0700); err != nil { + log.Fatal(err) + } + apiPassword := getAPIPassword() l, err := net.Listen("tcp", apiAddr) if err != nil { From fb4466f8402f0439bc432b65a65af76d3719035b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 9 Jan 2024 16:04:27 -0800 Subject: [PATCH 060/630] sqlite: fix tip encoding --- persist/sqlite/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 03f8f0a..1d33ab7 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -248,7 +248,7 @@ func applySiafundOutputs(tx txn, added map[types.Hash256]types.SiafundElement) e } func updateLastIndexedTip(tx txn, tip types.ChainIndex) error { - _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(tip.ID)) + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(tip)) return err } From 6c45c1f0549c0c5f82dcbe2defd8e557cac4e86b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 11:07:15 -0800 Subject: [PATCH 061/630] cmd: gracefully close stores on shutdown --- cmd/walletd/node.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index bd6119b..a3e3c34 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -84,13 +84,23 @@ var anagamiBootstrap = []string{ } type node struct { - cm *chain.Manager - s *syncer.Syncer - wm *wallet.Manager + chainStore *boltDB + cm *chain.Manager + + s *syncer.Syncer + + walletStore *sqlite.Store + wm *wallet.Manager Start func() (stop func()) } +// Close shuts down the node and closes its database. +func (n *node) Close() error { + n.chainStore.Close() + return n.walletStore.Close() +} + func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logger) (*node, error) { var network *consensus.Network var genesisBlock types.Block @@ -178,9 +188,11 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logge } return &node{ - cm: cm, - s: s, - wm: wm, + chainStore: db, + cm: cm, + s: s, + walletStore: walletDB, + wm: wm, Start: func() func() { ch := make(chan struct{}) go func() { From f78017189e72fd7b475f13fbd177f8036a6c96a4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 11:17:07 -0800 Subject: [PATCH 062/630] ci: fix lint errors --- .github/actions/test/action.yml | 8 +- .golangci.yml | 161 ++++++++++++++++++++++++++++++++ persist/sqlite/consensus.go | 3 + persist/sqlite/types.go | 1 + persist/sqlite/wallet.go | 12 +++ wallet/manager.go | 2 + wallet/state.go | 83 ---------------- wallet/wallet.go | 16 +++- 8 files changed, 196 insertions(+), 90 deletions(-) create mode 100644 .golangci.yml delete mode 100644 wallet/state.go diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index de33d2c..5754f0e 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -7,10 +7,10 @@ runs: - name: Configure git # required for golangci-lint on Windows shell: bash run: git config --global core.autocrlf false -# - name: Lint -# uses: golangci/golangci-lint-action@v3 -# with: -# skip-cache: true + - name: Lint + uses: golangci/golangci-lint-action@v3 + with: + skip-cache: true # - name: Analyze # uses: SiaFoundation/action-golang-analysis@HEAD # with: diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..ca4188f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,161 @@ +# Based off of the example file at https://github.com/golangci/golangci-lint + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 600s + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: [] + + # which dirs to skip: issues from them won't be reported; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but default dirs are skipped independently + # from this option's value (see skip-dirs-use-default). + skip-dirs: + - cover + + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: [] + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +# all available settings of specific linters +linters-settings: + ## Enabled linters: + govet: + # report about shadowed variables + check-shadowing: false + disable-all: false + + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + + + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + enabled-checks: + - argOrder # Diagnostic options + - badCond + - caseOrder + - dupArg + - dupBranchBody + - dupCase + - dupSubExpr + - nilValReturn + - offBy1 + - weakCond + - boolExprSimplify # Style options here and below. + - builtinShadow + - emptyFallthrough + - hexLiteral + - underef + - equalFold + revive: + ignore-generated-header: true + rules: + - name: blank-imports + disabled: false + - name: bool-literal-in-expr + disabled: false + - name: confusing-results + disabled: false + - name: constant-logical-expr + disabled: false + - name: context-as-argument + disabled: false + - name: exported + disabled: false + - name: errorf + disabled: false + - name: if-return + disabled: false + - name: indent-error-flow + disabled: false + - name: increment-decrement + disabled: false + - name: modifies-value-receiver + disabled: false + - name: optimize-operands-order + disabled: false + - name: range-val-in-closure + disabled: false + - name: struct-tag + disabled: false + - name: superfluous-else + disabled: false + - name: time-equal + disabled: false + - name: unexported-naming + disabled: false + - name: unexported-return + disabled: false + - name: unnecessary-stmt + disabled: false + - name: unreachable-code + disabled: false + - name: package-comments + disabled: true + +linters: + disable-all: true + fast: false + enable: + - tagliatelle + - gocritic + - gofmt + - revive + - govet + - misspell + - typecheck + - whitespace + +issues: + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: [] + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false \ No newline at end of file diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 1d33ab7..5edb89e 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -314,6 +314,7 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { return nil } +// applyChainUpdates applies the given chain updates to the database. func applyChainUpdates(tx txn, updates []*chain.ApplyUpdate) error { stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { @@ -399,6 +400,7 @@ func applyChainUpdates(tx txn, updates []*chain.ApplyUpdate) error { return nil } +// ProcessChainApplyUpdate implements chain.Subscriber func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { s.updates = append(s.updates, cau) @@ -414,6 +416,7 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) return nil } +// ProcessChainRevertUpdate implements chain.Subscriber func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { // update hasn't been committed yet if len(s.updates) > 0 && s.updates[len(s.updates)-1].Block.ID() == cru.Block.ID() { diff --git a/persist/sqlite/types.go b/persist/sqlite/types.go index 44f1f10..31083c6 100644 --- a/persist/sqlite/types.go +++ b/persist/sqlite/types.go @@ -115,6 +115,7 @@ type decodable[T types.DecoderFrom] struct { n int64 } +// Scan implements the sql.Scanner interface. func (d *decodable[T]) Scan(src any) error { switch src := src.(type) { case []byte: diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 56ba3fd..8306e1c 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -19,6 +19,7 @@ RETURNING id` return } +// WalletEvents returns the events relevant to a wallet, sorted by height descending. func (s *Store) WalletEvents(walletID string, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx txn) error { const query = `SELECT ev.id, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data @@ -76,6 +77,7 @@ LIMIT $2 OFFSET $3` return } +// AddWallet adds a wallet to the database. func (s *Store) AddWallet(name string, info json.RawMessage) error { return s.transaction(func(tx txn) error { const query = `INSERT INTO wallets (id, extra_data) VALUES ($1, $2)` @@ -88,6 +90,8 @@ func (s *Store) AddWallet(name string, info json.RawMessage) error { }) } +// DeleteWallet deletes a wallet from the database. This does not stop tracking +// addresses that were previously associated with the wallet. func (s *Store) DeleteWallet(name string) error { return s.transaction(func(tx txn) error { _, err := tx.Exec(`DELETE FROM wallets WHERE id=$1`, name) @@ -95,6 +99,7 @@ func (s *Store) DeleteWallet(name string) error { }) } +// Wallets returns a map of wallet names to wallet extra data. func (s *Store) Wallets() (map[string]json.RawMessage, error) { wallets := make(map[string]json.RawMessage) err := s.transaction(func(tx txn) error { @@ -119,6 +124,7 @@ func (s *Store) Wallets() (map[string]json.RawMessage, error) { return wallets, err } +// AddAddress adds an address to a wallet. func (s *Store) AddAddress(walletID string, address types.Address, info json.RawMessage) error { return s.transaction(func(tx txn) error { addressID, err := insertAddress(tx, address) @@ -130,6 +136,8 @@ func (s *Store) AddAddress(walletID string, address types.Address, info json.Raw }) } +// RemoveAddress removes an address from a wallet. This does not stop tracking +// the address. func (s *Store) RemoveAddress(walletID string, address types.Address) error { return s.transaction(func(tx txn) error { const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2)` @@ -138,6 +146,7 @@ func (s *Store) RemoveAddress(walletID string, address types.Address) error { }) } +// Addresses returns a map of addresses to their extra data for a wallet. func (s *Store) Addresses(walletID string) (map[types.Address]json.RawMessage, error) { addresses := make(map[types.Address]json.RawMessage) err := s.transaction(func(tx txn) error { @@ -165,6 +174,7 @@ WHERE wa.wallet_id=$1` return addresses, err } +// UnspentSiacoinOutputs returns the unspent siacoin outputs for a wallet. func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx txn) error { const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height @@ -193,6 +203,7 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin return } +// UnspentSiafundOutputs returns the unspent siafund outputs for a wallet. func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.SiafundElement, err error) { err = s.transaction(func(tx txn) error { const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address @@ -257,6 +268,7 @@ func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uin return } +// Annotate annotates a list of transactions using the wallet's addresses. func (s *Store) Annotate(walletID string, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { err = s.transaction(func(tx txn) error { stmt, err := tx.Prepare(`SELECT sia_address FROM wallet_addresses WHERE wallet_id=$1 AND sia_address=$2 LIMIT 1`) diff --git a/wallet/manager.go b/wallet/manager.go index 3e982e5..f1b0e62 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -13,6 +13,7 @@ import ( ) type ( + // A ChainManager manages the consensus state ChainManager interface { AddSubscriber(chain.Subscriber, types.ChainIndex) error RemoveSubscriber(chain.Subscriber) @@ -20,6 +21,7 @@ type ( BestIndex(height uint64) (types.ChainIndex, bool) } + // A Store is a persistent store of wallet data. Store interface { chain.Subscriber diff --git a/wallet/state.go b/wallet/state.go deleted file mode 100644 index 219a7f2..0000000 --- a/wallet/state.go +++ /dev/null @@ -1,83 +0,0 @@ -package wallet - -import ( - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" -) - -// A Midstate is a snapshot of unapplied consensus changes. -type Midstate struct { - SpentSiacoinOutputs map[types.Hash256]bool - SpentSiafundOutputs map[types.Hash256]bool - - NewSiacoinOutputs map[types.Hash256]types.SiacoinElement - NewSiafundOutputs map[types.Hash256]types.SiafundElement - - Events []Event -} - -func (ms *Midstate) Apply(cau *chain.ApplyUpdate, ownsAddress func(types.Address) bool) { - events := AppliedEvents(cau.State, cau.Block, cau, ownsAddress) - ms.Events = append(ms.Events, events...) - - cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if !ownsAddress(se.SiacoinOutput.Address) { - return - } - - if spent { - ms.SpentSiacoinOutputs[se.ID] = true - delete(ms.NewSiacoinOutputs, se.ID) - } else { - ms.NewSiacoinOutputs[se.ID] = se - } - }) - - cau.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { - if !ownsAddress(sf.SiafundOutput.Address) { - return - } - - if spent { - ms.SpentSiafundOutputs[sf.ID] = true - delete(ms.NewSiafundOutputs, sf.ID) - } else { - ms.NewSiafundOutputs[sf.ID] = sf - } - }) -} - -func (ms *Midstate) Revert(cru *chain.RevertUpdate, ownsAddress func(types.Address) bool) { - revertedBlockID := cru.Block.ID() - for i := len(ms.Events) - 1; i >= 0; i-- { - // working backwards, revert all events until the block ID no longer - // matches. - if ms.Events[i].Index.ID != revertedBlockID { - break - } - ms.Events = ms.Events[:i] - } - - cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if !ownsAddress(se.SiacoinOutput.Address) { - return - } - - if !spent { - delete(ms.SpentSiacoinOutputs, se.ID) - } - }) - - cru.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { - if !ownsAddress(sf.SiafundOutput.Address) { - return - } - - if spent { - ms.SpentSiafundOutputs[sf.ID] = true - delete(ms.NewSiafundOutputs, sf.ID) - } else { - ms.NewSiafundOutputs[sf.ID] = sf - } - }) -} diff --git a/wallet/wallet.go b/wallet/wallet.go index d1a458c..7a806b1 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -9,8 +9,8 @@ import ( "go.sia.tech/core/types" ) +// event type constants const ( - // transactions EventTypeTransaction = "transaction" EventTypeMinerPayout = "miner payout" EventTypeMissedFileContract = "missed file contract" @@ -153,8 +153,13 @@ type Event struct { Val interface{ EventType() string } } -func (*EventTransaction) EventType() string { return EventTypeTransaction } -func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } +// EventType implements Event. +func (*EventTransaction) EventType() string { return EventTypeTransaction } + +// EventType implements Event. +func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } + +// EventType implements Event. func (*EventMissedFileContract) EventType() string { return EventTypeMissedFileContract } // MarshalJSON implements json.Marshaler. @@ -235,6 +240,7 @@ type V2FileContract struct { Outputs []types.SiacoinElement `json:"outputs,omitempty"` } +// An EventTransaction represents a transaction that affects the wallet. type EventTransaction struct { ID types.TransactionID `json:"id"` SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` @@ -247,15 +253,19 @@ type EventTransaction struct { Fee types.Currency `json:"fee"` } +// An EventMinerPayout represents a miner payout from a block. type EventMinerPayout struct { SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` } +// An EventMissedFileContract represents a file contract that has expired +// without a storage proof type EventMissedFileContract struct { FileContract types.FileContractElement `json:"fileContract"` MissedOutputs []types.SiacoinElement `json:"missedOutputs"` } +// A ChainUpdate is a set of changes to the consensus state. type ChainUpdate interface { ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) From e6f17174008f37f08024458b70d23c54b6bf0b49 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 11:18:43 -0800 Subject: [PATCH 063/630] ci: disable jape analyzer --- .golangci.yml | 2 -- persist/sqlite/consensus.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ca4188f..041664e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -101,8 +101,6 @@ linters-settings: disabled: false - name: if-return disabled: false - - name: indent-error-flow - disabled: false - name: increment-decrement disabled: false - name: modifies-value-receiver diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 5edb89e..dc3d82a 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "log" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" @@ -60,7 +59,6 @@ func applyEvents(tx txn, events []wallet.Event) error { } else if _, err := addRelevantAddrStmt.Exec(eventID, addressID, event.Index.Height); err != nil { return fmt.Errorf("failed to add relevant address: %w", err) } - log.Println("added relevant address", eventID, addr) } } return nil From 7a3f173f4404d295e7eb927483bd9c21d9d8f1be Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 14:15:38 -0800 Subject: [PATCH 064/630] sqlite: fix siacoin element arg order --- persist/sqlite/consensus.go | 8 ++++---- persist/sqlite/wallet.go | 9 +++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index dc3d82a..c720675 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -144,7 +144,7 @@ func applySiacoinOutputs(tx txn, added map[types.Hash256]types.SiacoinElement) e } // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.MaturityHeight, se.LeafIndex) + _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) if err != nil { return fmt.Errorf("failed to insert output %q: %w", se.ID, err) } @@ -261,7 +261,7 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { } defer stmt.Close() - updateStmt, err := tx.Prepare(`UPDATE ` + table + ` SET merkle_proof=$1, leaf_index=$2 WHERE id=$3`) + updateStmt, err := tx.Prepare(`UPDATE ` + table + ` SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } @@ -298,7 +298,8 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { } for _, se := range updated { - _, err := updateStmt.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) + var dummy types.Hash256 + err := updateStmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy, 32)) if err != nil { return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) } @@ -308,7 +309,6 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { break } } - return nil } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 8306e1c..e142a0b 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -190,12 +190,11 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin for rows.Next() { var siacoin types.SiacoinElement - var proof []byte - - err := rows.Scan(decode(&siacoin.ID, 32), &siacoin.LeafIndex, &proof, (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address, 32), &siacoin.MaturityHeight) + err := rows.Scan(decode(&siacoin.ID, 32), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof, 32*1000), (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address, 32), &siacoin.MaturityHeight) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } + siacoins = append(siacoins, siacoin) } return nil @@ -219,9 +218,7 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund for rows.Next() { var siafund types.SiafundElement - var proof []byte - - err := rows.Scan(decode(&siafund.ID, 32), &siafund.LeafIndex, &proof, &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address, 32)) + err := rows.Scan(decode(&siafund.ID, 32), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof, 32*1000), &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address, 32)) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } From 77ec0611ecb829e7be3c1dcc1a3f438db899c7b8 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 14:36:54 -0800 Subject: [PATCH 065/630] ci: enable cgo, bump go versions for test --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e59abc..863f1a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,9 @@ on: branches: - master +env: + CGO_ENABLED: 1 + jobs: test: runs-on: ${{ matrix.os }} @@ -14,7 +17,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.19', '1.20' ] + go-version: [ '1.20', '1.21' ] steps: - name: Configure git run: git config --global core.autocrlf false # required on Windows From 986347158b4e15adf9804350bff1446c634367ca Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 16:14:14 -0800 Subject: [PATCH 066/630] sqlite: use NewBufDecoder --- persist/sqlite/consensus.go | 10 +++++----- persist/sqlite/types.go | 22 ++++++---------------- persist/sqlite/wallet.go | 8 ++++---- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index c720675..8722547 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -100,7 +100,7 @@ func deleteSiacoinOutputs(tx txn, spent []types.SiacoinElement) error { } var dummy types.Hash256 - err = deleteStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy, 32)) + err = deleteStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to delete output %q: %w", se.ID, err) } @@ -191,7 +191,7 @@ func deleteSiafundOutputs(tx txn, spent []types.SiafundElement) error { } var dummy types.Hash256 - err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy, 32)) + err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to delete output %q: %w", se.ID, err) } @@ -284,7 +284,7 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { more = true var se types.StateElement - err := rows.Scan(decode(&se.ID, 32), decodeSlice(&se.MerkleProof, 32*1000), &se.LeafIndex) + err := rows.Scan(decode(&se.ID), decodeSlice(&se.MerkleProof), &se.LeafIndex) if err != nil { return false, fmt.Errorf("failed to scan state element: %w", err) } @@ -299,7 +299,7 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { for _, se := range updated { var dummy types.Hash256 - err := updateStmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy, 32)) + err := updateStmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) } @@ -506,6 +506,6 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { // LastCommittedIndex returns the last chain index that was committed. func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { - err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index, 40)) + err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index)) return } diff --git a/persist/sqlite/types.go b/persist/sqlite/types.go index 31083c6..f7f0973 100644 --- a/persist/sqlite/types.go +++ b/persist/sqlite/types.go @@ -6,7 +6,6 @@ import ( "database/sql/driver" "encoding/binary" "fmt" - "io" "time" "go.sia.tech/core/types" @@ -78,16 +77,12 @@ func encodeSlice[T types.EncoderTo](v []T) []byte { type decodableSlice[T any] struct { v *[]T - n int64 } func (d *decodableSlice[T]) Scan(src any) error { switch src := src.(type) { case []byte: - dec := types.NewDecoder(io.LimitedReader{ - R: bytes.NewReader(src), - N: d.n, - }) + dec := types.NewBufDecoder(src) s := make([]T, dec.ReadPrefix()) for i := range s { dv, ok := any(&s[i]).(types.DecoderFrom) @@ -106,24 +101,19 @@ func (d *decodableSlice[T]) Scan(src any) error { } } -func decodeSlice[T any](v *[]T, maxLen int64) sql.Scanner { - return &decodableSlice[T]{v: v, n: maxLen} +func decodeSlice[T any](v *[]T) sql.Scanner { + return &decodableSlice[T]{v: v} } type decodable[T types.DecoderFrom] struct { v T - n int64 } // Scan implements the sql.Scanner interface. func (d *decodable[T]) Scan(src any) error { switch src := src.(type) { case []byte: - dec := types.NewDecoder(io.LimitedReader{ - R: bytes.NewReader(src), - N: d.n, - }) - + dec := types.NewBufDecoder(src) d.v.DecodeFrom(dec) return dec.Err() default: @@ -131,6 +121,6 @@ func (d *decodable[T]) Scan(src any) error { } } -func decode[T types.DecoderFrom](v T, maxLen int64) sql.Scanner { - return &decodable[T]{v, maxLen} +func decode[T types.DecoderFrom](v T) sql.Scanner { + return &decodable[T]{v} } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index e142a0b..5d9f37c 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -41,7 +41,7 @@ LIMIT $2 OFFSET $3` var eventType string var eventBuf []byte - err := rows.Scan(&eventID, (*sqlTime)(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID, 32), &eventType, &eventBuf) + err := rows.Scan(&eventID, (*sqlTime)(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) if err != nil { return fmt.Errorf("failed to scan event: %w", err) } @@ -164,7 +164,7 @@ WHERE wa.wallet_id=$1` for rows.Next() { var address types.Address var extraData json.RawMessage - if err := rows.Scan(decode(&address, 32), &extraData); err != nil { + if err := rows.Scan(decode(&address), &extraData); err != nil { return fmt.Errorf("failed to scan address: %w", err) } addresses[address] = extraData @@ -190,7 +190,7 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin for rows.Next() { var siacoin types.SiacoinElement - err := rows.Scan(decode(&siacoin.ID, 32), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof, 32*1000), (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address, 32), &siacoin.MaturityHeight) + err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } @@ -218,7 +218,7 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund for rows.Next() { var siafund types.SiafundElement - err := rows.Scan(decode(&siafund.ID, 32), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof, 32*1000), &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address, 32)) + err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } From 37c4daaf85bd1c0becd4c08bd82e7f8b1428132b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 16:14:21 -0800 Subject: [PATCH 067/630] api: fix client docstring --- api/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client.go b/api/client.go index 3258d9e..8365495 100644 --- a/api/client.go +++ b/api/client.go @@ -106,7 +106,7 @@ func (c *Client) Wallet(name string) *WalletClient { } // Resubscribe subscribes the wallet to consensus updates, starting at the -// specified height. This can only be done once. +// specified height. func (c *Client) Resubscribe(height uint64) (err error) { err = c.c.POST("/resubscribe", height, nil) return From ef6583049592fea6a2dd47db5b7c3f6971feb9bb Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 16:22:40 -0800 Subject: [PATCH 068/630] api,cmd,sqlite,syncer: remove ephemeral store, add sqlite peer store --- api/api_test.go | 23 ++-- api/server.go | 3 +- cmd/walletd/node.go | 39 +++---- cmd/walletd/testnet.go | 3 +- internal/syncerutil/store.go | 208 ----------------------------------- persist/sqlite/init.sql | 15 +++ persist/sqlite/peers.go | 188 +++++++++++++++++++++++++++++++ 7 files changed, 235 insertions(+), 244 deletions(-) delete mode 100644 internal/syncerutil/store.go create mode 100644 persist/sqlite/peers.go diff --git a/api/api_test.go b/api/api_test.go index 9ffa789..afcb5a5 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -14,7 +14,6 @@ import ( "go.sia.tech/coreutils/syncer" "go.sia.tech/jape" "go.sia.tech/walletd/api" - "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" "go.uber.org/zap/zaptest" @@ -394,7 +393,7 @@ func TestV2(t *testing.T) { } func TestP2P(t *testing.T) { - log := zaptest.NewLogger(t) + logger := zaptest.NewLogger(t) n, genesisBlock := testNetwork() // gift primary wallet some coins primaryPrivateKey := types.GeneratePrivateKey() @@ -409,13 +408,14 @@ func TestP2P(t *testing.T) { if err != nil { t.Fatal(err) } + log1 := logger.Named("one") cm1 := chain.NewManager(dbstore1, tipState) - ws1, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + store1, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log1.Named("sqlite3")) if err != nil { t.Fatal(err) } - defer ws1.Close() - wm1, err := wallet.NewManager(cm1, ws1, log.Named("wallet")) + defer store1.Close() + wm1, err := wallet.NewManager(cm1, store1, log1.Named("wallet")) if err != nil { t.Fatal(err) } @@ -424,7 +424,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer l1.Close() - s1 := syncer.New(l1, cm1, syncerutil.NewEphemeralPeerStore(), gateway.Header{ + s1 := syncer.New(l1, cm1, store1, gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l1.Addr().String(), @@ -447,13 +447,14 @@ func TestP2P(t *testing.T) { if err != nil { t.Fatal(err) } + log2 := logger.Named("two") cm2 := chain.NewManager(dbstore2, tipState) - ws2, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + store2, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log2.Named("sqlite3")) if err != nil { t.Fatal(err) } - defer ws2.Close() - wm2, err := wallet.NewManager(cm2, ws2, log.Named("wallet")) + defer store2.Close() + wm2, err := wallet.NewManager(cm2, store2, log2.Named("wallet")) if err != nil { t.Fatal(err) } @@ -462,11 +463,11 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer l2.Close() - s2 := syncer.New(l2, cm2, syncerutil.NewEphemeralPeerStore(), gateway.Header{ + s2 := syncer.New(l2, cm2, store2, gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l2.Addr().String(), - }) + }, syncer.WithLogger(zaptest.NewLogger(t))) go s2.Run() c2, shutdown2 := runServer(cm2, s2, wm2) defer shutdown2() diff --git a/api/server.go b/api/server.go index f7c87d7..5030592 100644 --- a/api/server.go +++ b/api/server.go @@ -92,7 +92,8 @@ func (s *server) syncerPeersHandler(jc jape.Context) { for _, p := range s.s.Peers() { info, ok := s.s.PeerInfo(p.Addr()) if !ok { - continue + jc.Error(errors.New("peer not found"), http.StatusNotFound) + return } peers = append(peers, GatewayPeer{ Addr: p.Addr(), diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index a3e3c34..b5a6808 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -15,7 +15,6 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" - "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" "go.uber.org/zap" @@ -84,13 +83,12 @@ var anagamiBootstrap = []string{ } type node struct { - chainStore *boltDB + chainStore *coreutils.BoltChainDB cm *chain.Manager - s *syncer.Syncer - - walletStore *sqlite.Store - wm *wallet.Manager + store *sqlite.Store + s *syncer.Syncer + wm *wallet.Manager Start func() (stop func()) } @@ -98,7 +96,7 @@ type node struct { // Close shuts down the node and closes its database. func (n *node) Close() error { n.chainStore.Close() - return n.walletStore.Close() + return n.store.Close() } func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logger) (*node, error) { @@ -163,36 +161,31 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logge syncerAddr = net.JoinHostPort("127.0.0.1", port) } - ps, err := syncerutil.NewJSONPeerStore(filepath.Join(dir, "peers.json")) + store, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { - return nil, fmt.Errorf("failed to open peer store: %w", err) + return nil, fmt.Errorf("failed to open wallet database: %w", err) } + for _, peer := range bootstrapPeers { - ps.AddPeer(peer) + store.AddPeer(peer) } header := gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) - - walletDB, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - return nil, fmt.Errorf("failed to open wallet database: %w", err) - } - - wm, err := wallet.NewManager(cm, walletDB, log.Named("wallet")) + s := syncer.New(l, cm, store, header, syncer.WithLogger(log.Named("syncer"))) + wm, err := wallet.NewManager(cm, store, log.Named("wallet")) if err != nil { return nil, fmt.Errorf("failed to create wallet manager: %w", err) } return &node{ - chainStore: db, - cm: cm, - s: s, - walletStore: walletDB, - wm: wm, + chainStore: bdb, + cm: cm, + store: store, + s: s, + wm: wm, Start: func() func() { ch := make(chan struct{}) go func() { diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 254d7a0..b87ad21 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -115,12 +115,13 @@ func initTestnetClient(addr string, network string, seed wallet.Seed) *api.Clien } else if err := wc.AddAddress(ourAddr, nil); err != nil { fmt.Println() log.Fatal(err) - } else if err := wc.Subscribe(0); err != nil { + } else if err := c.Resubscribe(0); err != nil { fmt.Println() log.Fatal(err) } fmt.Println("done.") } + return c } diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go deleted file mode 100644 index 6456c6b..0000000 --- a/internal/syncerutil/store.go +++ /dev/null @@ -1,208 +0,0 @@ -package syncerutil - -import ( - "encoding/json" - "net" - "os" - "sync" - "time" - - "go.sia.tech/coreutils/syncer" -) - -type peerBan struct { - Expiry time.Time `json:"expiry"` - Reason string `json:"reason"` -} - -// EphemeralPeerStore implements PeerStore with an in-memory map. -type EphemeralPeerStore struct { - peers map[string]syncer.PeerInfo - bans map[string]peerBan - mu sync.Mutex -} - -func (eps *EphemeralPeerStore) banned(peer string) bool { - host, _, err := net.SplitHostPort(peer) - if err != nil { - return false // shouldn't happen - } - for _, s := range []string{ - peer, // 1.2.3.4:5678 - syncer.Subnet(host, "/32"), // 1.2.3.4:* - syncer.Subnet(host, "/24"), // 1.2.3.* - syncer.Subnet(host, "/16"), // 1.2.* - syncer.Subnet(host, "/8"), // 1.* - } { - if b, ok := eps.bans[s]; ok { - if time.Until(b.Expiry) <= 0 { - delete(eps.bans, s) - } else { - return true - } - } - } - return false -} - -// AddPeer implements PeerStore. -func (eps *EphemeralPeerStore) AddPeer(peer string) { - eps.mu.Lock() - defer eps.mu.Unlock() - if _, ok := eps.peers[peer]; !ok { - eps.peers[peer] = syncer.PeerInfo{FirstSeen: time.Now()} - } -} - -// Peers implements PeerStore. -func (eps *EphemeralPeerStore) Peers() []string { - eps.mu.Lock() - defer eps.mu.Unlock() - var peers []string - for p := range eps.peers { - if !eps.banned(p) { - peers = append(peers, p) - } - } - return peers -} - -// UpdatePeerInfo implements PeerStore. -func (eps *EphemeralPeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - eps.mu.Lock() - defer eps.mu.Unlock() - info, ok := eps.peers[peer] - if !ok { - return - } - fn(&info) - eps.peers[peer] = info -} - -// PeerInfo implements PeerStore. -func (eps *EphemeralPeerStore) PeerInfo(peer string) (syncer.PeerInfo, bool) { - eps.mu.Lock() - defer eps.mu.Unlock() - info, ok := eps.peers[peer] - return info, ok -} - -// Ban implements PeerStore. -func (eps *EphemeralPeerStore) Ban(peer string, duration time.Duration, reason string) { - eps.mu.Lock() - defer eps.mu.Unlock() - // canonicalize - if _, ipnet, err := net.ParseCIDR(peer); err == nil { - peer = ipnet.String() - } - eps.bans[peer] = peerBan{Expiry: time.Now().Add(duration), Reason: reason} -} - -// Banned implements PeerStore. -func (eps *EphemeralPeerStore) Banned(peer string) bool { - eps.mu.Lock() - defer eps.mu.Unlock() - return eps.banned(peer) -} - -// NewEphemeralPeerStore initializes an EphemeralPeerStore. -func NewEphemeralPeerStore() *EphemeralPeerStore { - return &EphemeralPeerStore{ - peers: make(map[string]syncer.PeerInfo), - bans: make(map[string]peerBan), - } -} - -type jsonPersist struct { - Peers map[string]syncer.PeerInfo `json:"peers"` - Bans map[string]peerBan `json:"bans"` -} - -// JSONPeerStore implements PeerStore with a JSON file on disk. -type JSONPeerStore struct { - *EphemeralPeerStore - path string - lastSave time.Time -} - -func (jps *JSONPeerStore) load() error { - f, err := os.Open(jps.path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p jsonPersist - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - jps.EphemeralPeerStore.peers = p.Peers - jps.EphemeralPeerStore.bans = p.Bans - return nil -} - -func (jps *JSONPeerStore) save() error { - jps.EphemeralPeerStore.mu.Lock() - defer jps.EphemeralPeerStore.mu.Unlock() - if time.Since(jps.lastSave) < 5*time.Second { - return nil - } - defer func() { jps.lastSave = time.Now() }() - // clear out expired bans - for peer, b := range jps.EphemeralPeerStore.bans { - if time.Until(b.Expiry) <= 0 { - delete(jps.EphemeralPeerStore.bans, peer) - } - } - p := jsonPersist{ - Peers: jps.EphemeralPeerStore.peers, - Bans: jps.EphemeralPeerStore.bans, - } - js, err := json.MarshalIndent(p, "", " ") - if err != nil { - return err - } - f, err := os.OpenFile(jps.path+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(jps.path+"_tmp", jps.path); err != nil { - return err - } - return nil -} - -// AddPeer implements PeerStore. -func (jps *JSONPeerStore) AddPeer(peer string) { - jps.EphemeralPeerStore.AddPeer(peer) - jps.save() -} - -// UpdatePeerInfo implements PeerStore. -func (jps *JSONPeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - jps.EphemeralPeerStore.UpdatePeerInfo(peer, fn) - jps.save() -} - -// Ban implements PeerStore. -func (jps *JSONPeerStore) Ban(peer string, duration time.Duration, reason string) { - jps.EphemeralPeerStore.Ban(peer, duration, reason) - jps.save() -} - -// NewJSONPeerStore returns a JSONPeerStore backed by the specified file. -func NewJSONPeerStore(path string) (*JSONPeerStore, error) { - jps := &JSONPeerStore{ - EphemeralPeerStore: NewEphemeralPeerStore(), - path: path, - } - return jps, jps.load() -} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index d6a8619..504e3b5 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -63,6 +63,21 @@ CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); CREATE INDEX event_addresses_event_id_address_id_block_height ON event_addresses(event_id, address_id, block_height DESC); +CREATE TABLE syncer_peers ( + peer_address TEXT PRIMARY KEY NOT NULL, + first_seen INTEGER NOT NULL, + last_connect INTEGER NOT NULL, + synced_blocks INTEGER NOT NULL, + sync_duration INTEGER NOT NULL +); + +CREATE TABLE syncer_bans ( + net_cidr TEXT PRIMARY KEY NOT NULL, + expiration INTEGER NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); + CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row db_version INTEGER NOT NULL, -- used for migrations diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go new file mode 100644 index 0000000..206822b --- /dev/null +++ b/persist/sqlite/peers.go @@ -0,0 +1,188 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "go.sia.tech/coreutils/syncer" + "go.uber.org/zap" +) + +func getPeerInfo(tx txn, peer string) (syncer.PeerInfo, error) { + const query = `SELECT first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` + var info syncer.PeerInfo + err := tx.QueryRow(query, peer).Scan((*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), &info.SyncedBlocks, &info.SyncDuration) + return info, err +} + +func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error { + const query = `UPDATE syncer_peers SET first_seen=$2, last_connect=$3, synced_blocks=$4, sync_duration=$5 WHERE peer_address=$1` + _, err := tx.Exec(query, peer, (*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), info.SyncedBlocks, info.SyncDuration) + return err +} + +// AddPeer adds the given peer to the store. +func (s *Store) AddPeer(peer string) { + err := s.transaction(func(tx txn) error { + const query = `INSERT INTO syncer_peers (peer_address, first_seen, last_connect, synced_blocks, sync_duration) VALUES ($1, $2, 0, 0, 0) ON CONFLICT (peer_address) DO NOTHING` + _, err := tx.Exec(query, peer, sqlTime(time.Now())) + return err + }) + if err != nil { + s.log.Error("failed to add peer", zap.Error(err)) + } +} + +// Peers returns the addresses of all known peers. +func (s *Store) Peers() (peers []string) { + err := s.transaction(func(tx txn) error { + const query = `SELECT peer_address FROM syncer_peers` + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var peer string + if err := rows.Scan(&peer); err != nil { + return err + } + peers = append(peers, peer) + } + return nil + }) + if err != nil { + panic(err) // 😔 + } + return +} + +// UpdatePeerInfo updates the info for the given peer. +func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { + err := s.transaction(func(tx txn) error { + info, err := getPeerInfo(tx, peer) + if err != nil { + return err + } + fn(&info) + return s.updatePeerInfo(tx, peer, info) + }) + if err != nil { + panic(err) // 😔 + } +} + +// PeerInfo returns the info for the given peer. +func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, bool) { + var info syncer.PeerInfo + var err error + err = s.transaction(func(tx txn) error { + info, err = getPeerInfo(tx, peer) + return err + }) + if errors.Is(err, sql.ErrNoRows) { + return info, false + } else if err != nil { + panic(err) // 😔 + } + return info, true +} + +// normalizePeer normalizes a peer address to a CIDR subnet. +func normalizePeer(peer string) (string, error) { + host, _, err := net.SplitHostPort(peer) + if err != nil { + host = peer + } + if strings.IndexByte(host, '/') != -1 { + _, subnet, err := net.ParseCIDR(host) + if err != nil { + return "", fmt.Errorf("failed to parse CIDR: %w", err) + } + return subnet.String(), nil + } + + ip := net.ParseIP(host) + if ip == nil { + return "", errors.New("invalid IP address") + } + + var maskLen int + if ip.To4() != nil { + maskLen = 32 + } else { + maskLen = 128 + } + + _, normalized, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), maskLen)) + if err != nil { + panic("failed to parse CIDR") + } + return normalized.String(), nil +} + +// Ban temporarily bans one or more IPs. The addr should either be a single +// IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). +func (s *Store) Ban(peer string, duration time.Duration, reason string) { + address, err := normalizePeer(peer) + if err != nil { + s.log.Error("failed to normalize peer", zap.Error(err)) + return + } + err = s.transaction(func(tx txn) error { + const query = `INSERT INTO syncer_bans (net_cidr, expiration, reason) VALUES ($1, $2, $3) ON CONFLICT (net_cidr) DO UPDATE SET expiration=EXCLUDED.expiration, reason=EXCLUDED.reason` + _, err := tx.Exec(query, address, sqlTime(time.Now().Add(duration)), reason) + return err + }) + if err != nil { + s.log.Error("failed to ban peer", zap.Error(err)) + } +} + +// Banned returns true if the peer is banned. +func (s *Store) Banned(peer string) (banned bool) { + // normalize the peer into a CIDR subnet + peer, err := normalizePeer(peer) + if err != nil { + s.log.Error("failed to normalize peer", zap.Error(err)) + return false + } + + _, subnet, err := net.ParseCIDR(peer) + if err != nil { + s.log.Error("failed to parse CIDR", zap.Error(err)) + return false + } + + // check all subnets from the given subnet to the max subnet length + var maxMaskLen int + if subnet.IP.To4() != nil { + maxMaskLen = 32 + } else { + maxMaskLen = 128 + } + + checkSubnets := make([]string, 0, maxMaskLen) + for i := maxMaskLen; i > 0; i-- { + check := subnet.IP.String() + "/" + strconv.Itoa(i) + checkSubnets = append(checkSubnets, check) + } + + err = s.transaction(func(tx txn) error { + query := `SELECT net_cidr, expiration FROM syncer_bans WHERE net_cidr IN (` + queryPlaceHolders(len(checkSubnets)) + `) ORDER BY expiration DESC LIMIT 1` + + var expiration time.Time + err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan((*sqlTime)(&expiration)) + banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows + return err + }) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + s.log.Error("failed to check ban status", zap.Error(err)) + } + return +} From 0dcc110b9e1f998262474b122d859cd1242cc0aa Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 16:43:11 -0800 Subject: [PATCH 069/630] sqlite: add peer tests --- persist/sqlite/peers.go | 17 ++++-- persist/sqlite/peers_test.go | 101 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 persist/sqlite/peers_test.go diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 206822b..7046f73 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -21,8 +21,8 @@ func getPeerInfo(tx txn, peer string) (syncer.PeerInfo, error) { } func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error { - const query = `UPDATE syncer_peers SET first_seen=$2, last_connect=$3, synced_blocks=$4, sync_duration=$5 WHERE peer_address=$1` - _, err := tx.Exec(query, peer, (*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), info.SyncedBlocks, info.SyncDuration) + const query = `UPDATE syncer_peers SET first_seen=$1, last_connect=$2, synced_blocks=$3, sync_duration=$4 WHERE peer_address=$5 RETURNING peer_address` + err := tx.QueryRow(query, (*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), info.SyncedBlocks, info.SyncDuration, peer).Scan(&peer) return err } @@ -169,16 +169,23 @@ func (s *Store) Banned(peer string) (banned bool) { checkSubnets := make([]string, 0, maxMaskLen) for i := maxMaskLen; i > 0; i-- { - check := subnet.IP.String() + "/" + strconv.Itoa(i) - checkSubnets = append(checkSubnets, check) + _, subnet, err := net.ParseCIDR(subnet.IP.String() + "/" + strconv.Itoa(i)) + if err != nil { + panic("failed to parse CIDR") + } + checkSubnets = append(checkSubnets, subnet.String()) } err = s.transaction(func(tx txn) error { query := `SELECT net_cidr, expiration FROM syncer_bans WHERE net_cidr IN (` + queryPlaceHolders(len(checkSubnets)) + `) ORDER BY expiration DESC LIMIT 1` + var subnet string var expiration time.Time - err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan((*sqlTime)(&expiration)) + err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan(&subnet, (*sqlTime)(&expiration)) banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows + if err == nil && banned { + s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) + } return err }) if err != nil && !errors.Is(err, sql.ErrNoRows) { diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go new file mode 100644 index 0000000..2de6d2f --- /dev/null +++ b/persist/sqlite/peers_test.go @@ -0,0 +1,101 @@ +package sqlite + +import ( + "net" + "path/filepath" + "testing" + "time" + + "go.sia.tech/walletd/syncer" + "go.uber.org/zap/zaptest" +) + +func TestAddPeer(t *testing.T) { + log := zaptest.NewLogger(t) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + const peer = "1.2.3.4:9981" + + if err := db.AddPeer(peer); err != nil { + t.Fatal(err) + } + + lastConnect := time.Now().Truncate(time.Second) // stored as unix milliseconds + syncedBlocks := uint64(15) + syncDuration := 5 * time.Second + + err = db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { + info.LastConnect = lastConnect + info.SyncedBlocks = syncedBlocks + info.SyncDuration = syncDuration + }) + if err != nil { + t.Fatal(err) + } + + info, err := db.PeerInfo(peer) + if err != nil { + t.Fatal(err) + } + + if !info.LastConnect.Equal(lastConnect) { + t.Errorf("expected LastConnect = %v; got %v", lastConnect, info.LastConnect) + } + if info.SyncedBlocks != syncedBlocks { + t.Errorf("expected SyncedBlocks = %d; got %d", syncedBlocks, info.SyncedBlocks) + } + if info.SyncDuration != 5*time.Second { + t.Errorf("expected SyncDuration = %s; got %s", syncDuration, info.SyncDuration) + } +} + +func TestBanPeer(t *testing.T) { + log := zaptest.NewLogger(t) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + const peer = "1.2.3.4" + + if db.Banned(peer) { + t.Fatal("expected peer to not be banned") + } + + // ban the peer + if err := db.Ban(peer, time.Second, "test"); err != nil { + t.Fatal(err) + } + + if !db.Banned(peer) { + t.Fatal("expected peer to be banned") + } + + // wait for the ban to expire + time.Sleep(time.Second) + + if db.Banned(peer) { + t.Fatal("expected peer to not be banned") + } + + // ban a subnet + _, subnet, err := net.ParseCIDR(peer + "/24") + if err != nil { + t.Fatal(err) + } + + t.Log("banning", subnet) + + if err := db.Ban(subnet.String(), time.Second, "test"); err != nil { + t.Fatal(err) + } + + if !db.Banned(peer) { + t.Fatal("expected peer to be banned") + } +} From ac351b5510d78a87ebb98af776a62871f1e15f13 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 10 Jan 2024 16:49:18 -0800 Subject: [PATCH 070/630] sqlite: better update proof logic --- persist/sqlite/consensus.go | 77 +++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 8722547..3424e49 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -250,10 +250,38 @@ func updateLastIndexedTip(tx txn, tip types.ChainIndex) error { return err } +func getStateElementBatch(stmt *loggedStmt, offset, limit int) ([]types.StateElement, error) { + rows, err := stmt.Query(limit, offset) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var updated []types.StateElement + for rows.Next() { + var se types.StateElement + err := rows.Scan(decode(&se.ID), decodeSlice(&se.MerkleProof), &se.LeafIndex) + if err != nil { + return nil, fmt.Errorf("failed to scan state element: %w", err) + } + updated = append(updated, se) + } + return updated, nil +} + +func updateStateElement(stmt *loggedStmt, se types.StateElement) error { + res, err := stmt.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row to be affected, got %d", n) + } + return nil +} + // how slow is this going to be 😬? -// -// todo: determine if it's feasible for exchange mode to keep everything in -// memory. func updateElementProofs(tx txn, table string, updater proofUpdater) error { stmt, err := tx.Prepare(`SELECT id, merkle_proof, leaf_index FROM ` + table + ` LIMIT $1 OFFSET $2`) if err != nil { @@ -267,47 +295,20 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { } defer updateStmt.Close() - var updated []types.StateElement for offset := 0; ; offset += updateProofBatchSize { - updated = updated[:0] - - more, err := func(n int) (bool, error) { - rows, err := stmt.Query(updateProofBatchSize, n) - if err != nil { - return false, fmt.Errorf("failed to query siacoin elements: %w", err) - } - defer rows.Close() - - var more bool - for rows.Next() { - // if we get here, there may be more rows to process - more = true - - var se types.StateElement - err := rows.Scan(decode(&se.ID), decodeSlice(&se.MerkleProof), &se.LeafIndex) - if err != nil { - return false, fmt.Errorf("failed to scan state element: %w", err) - } - updater.UpdateElementProof(&se) - updated = append(updated, se) - } - return more, nil - }(offset) + elements, err := getStateElementBatch(stmt, offset, updateProofBatchSize) if err != nil { - return err + return fmt.Errorf("failed to get state element batch: %w", err) + } else if len(elements) == 0 { + break } - for _, se := range updated { - var dummy types.Hash256 - err := updateStmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) - if err != nil { - return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) + for _, se := range elements { + updater.UpdateElementProof(&se) + if err := updateStateElement(updateStmt, se); err != nil { + return fmt.Errorf("failed to update state element: %w", err) } } - - if !more { - break - } } return nil } From d470946aa9b4a0687f0410e58802fc849fd4b2d4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 11 Jan 2024 07:24:28 -0800 Subject: [PATCH 071/630] sqlite: fix ownsAddress --- persist/sqlite/consensus.go | 2 +- persist/sqlite/wallet.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 3424e49..525cce4 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -425,7 +425,7 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { // update has been committed, revert it return s.transaction(func(tx txn) error { - stmt, err := tx.Prepare(`SELECT sia_address FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) + stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 5d9f37c..cbd3de7 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -268,7 +268,10 @@ func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uin // Annotate annotates a list of transactions using the wallet's addresses. func (s *Store) Annotate(walletID string, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { err = s.transaction(func(tx txn) error { - stmt, err := tx.Prepare(`SELECT sia_address FROM wallet_addresses WHERE wallet_id=$1 AND sia_address=$2 LIMIT 1`) + const query = `SELECT sa.id FROM sia_addresses sa +INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) +WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` + stmt, err := tx.Prepare(query) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } From d066588ad82a9248416b95c5a7f0f0c0079b95e8 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jan 2024 11:10:22 -0800 Subject: [PATCH 072/630] sqlite: better encoding --- persist/sqlite/consensus.go | 14 ++-- persist/sqlite/encoding.go | 116 +++++++++++++++++++++++++++++++++ persist/sqlite/peers.go | 12 ++-- persist/sqlite/types.go | 126 ------------------------------------ persist/sqlite/wallet.go | 12 ++-- 5 files changed, 135 insertions(+), 145 deletions(-) create mode 100644 persist/sqlite/encoding.go delete mode 100644 persist/sqlite/types.go diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 525cce4..ff9c1b4 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -47,7 +47,7 @@ func applyEvents(tx txn, events []wallet.Event) error { } var eventID int64 - err = stmt.QueryRow(sqlTime(event.Timestamp), id, event.Val.EventType(), buf).Scan(&eventID) + err = stmt.QueryRow(encode(event.Timestamp), id, event.Val.EventType(), buf).Scan(&eventID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -87,14 +87,14 @@ func deleteSiacoinOutputs(tx txn, spent []types.SiacoinElement) error { // query the address database ID and balance var addressID int64 var balance types.Currency - err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, (*sqlCurrency)(&balance)) + err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, decode(&balance)) if err != nil { return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) } // update the balance balance = balance.Sub(se.SiacoinOutput.Value) - _, err = updateBalanceStmt.Exec((*sqlCurrency)(&balance), addressID) + _, err = updateBalanceStmt.Exec(encode(balance), addressID) if err != nil { return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) } @@ -131,20 +131,20 @@ func applySiacoinOutputs(tx txn, added map[types.Hash256]types.SiacoinElement) e // query the address database ID and balance var addressID int64 var balance types.Currency - err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, (*sqlCurrency)(&balance)) + err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, decode(&balance)) if err != nil { return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) } // update the balance balance = balance.Add(se.SiacoinOutput.Value) - _, err = updateBalanceStmt.Exec((*sqlCurrency)(&balance), addressID) + _, err = updateBalanceStmt.Exec(encode(balance), addressID) if err != nil { return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) } // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) + _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) if err != nil { return fmt.Errorf("failed to insert output %q: %w", se.ID, err) } @@ -237,7 +237,7 @@ func applySiafundOutputs(tx txn, added map[types.Hash256]types.SiafundElement) e return fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) } - _, err = addStmt.Exec(encode(se.ID), addressID, sqlCurrency(se.ClaimStart), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex) + _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.ClaimStart), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex) if err != nil { return fmt.Errorf("failed to insert output %q: %w", se.ID, err) } diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go new file mode 100644 index 0000000..f011b23 --- /dev/null +++ b/persist/sqlite/encoding.go @@ -0,0 +1,116 @@ +package sqlite + +import ( + "bytes" + "database/sql" + "encoding/binary" + "errors" + "fmt" + "time" + + "go.sia.tech/core/types" +) + +func encode(obj any) any { + switch obj := obj.(type) { + case types.EncoderTo: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + obj.EncodeTo(e) + e.Flush() + return buf.Bytes() + case uint64: + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, obj) + return b + case time.Time: + return obj.Unix() + default: + panic(fmt.Sprintf("dbEncode: unsupported type %T", obj)) + } +} + +type decodable struct { + v any +} + +// Scan implements the sql.Scanner interface. +func (d *decodable) Scan(src any) error { + if src == nil { + return errors.New("cannot scan nil into decodable") + } + + switch src := src.(type) { + case []byte: + switch v := d.v.(type) { + case types.DecoderFrom: + dec := types.NewBufDecoder(src) + v.DecodeFrom(dec) + return dec.Err() + case *uint64: + *v = binary.LittleEndian.Uint64(src) + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } + return nil + case int64: + switch v := d.v.(type) { + case *uint64: + *v = uint64(src) + case *time.Time: + *v = time.Unix(src, 0).UTC() + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } + return nil + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } +} + +func decode(obj any) sql.Scanner { + return &decodable{obj} +} + +type decodableSlice[T any] struct { + v *[]T +} + +func (d *decodableSlice[T]) Scan(src any) error { + switch src := src.(type) { + case []byte: + dec := types.NewBufDecoder(src) + s := make([]T, dec.ReadPrefix()) + for i := range s { + dv, ok := any(&s[i]).(types.DecoderFrom) + if !ok { + panic(fmt.Errorf("cannot decode %T", s[i])) + } + dv.DecodeFrom(dec) + } + if err := dec.Err(); err != nil { + return err + } + *d.v = s + return nil + default: + return fmt.Errorf("cannot scan %T to []byte", src) + } +} + +func decodeSlice[T any](v *[]T) sql.Scanner { + return &decodableSlice[T]{v: v} +} + +func encodeSlice[T types.EncoderTo](v []T) []byte { + var buf bytes.Buffer + enc := types.NewEncoder(&buf) + enc.WritePrefix(len(v)) + for _, e := range v { + e.EncodeTo(enc) + } + if err := enc.Flush(); err != nil { + panic(err) + } + return buf.Bytes() +} diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 7046f73..3b59936 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -16,13 +16,13 @@ import ( func getPeerInfo(tx txn, peer string) (syncer.PeerInfo, error) { const query = `SELECT first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` var info syncer.PeerInfo - err := tx.QueryRow(query, peer).Scan((*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), &info.SyncedBlocks, &info.SyncDuration) + err := tx.QueryRow(query, peer).Scan(decode(&info.FirstSeen), decode(&info.LastConnect), &info.SyncedBlocks, &info.SyncDuration) return info, err } func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error { const query = `UPDATE syncer_peers SET first_seen=$1, last_connect=$2, synced_blocks=$3, sync_duration=$4 WHERE peer_address=$5 RETURNING peer_address` - err := tx.QueryRow(query, (*sqlTime)(&info.FirstSeen), (*sqlTime)(&info.LastConnect), info.SyncedBlocks, info.SyncDuration, peer).Scan(&peer) + err := tx.QueryRow(query, encode(info.FirstSeen), encode(info.LastConnect), info.SyncedBlocks, info.SyncDuration, peer).Scan(&peer) return err } @@ -30,7 +30,7 @@ func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error func (s *Store) AddPeer(peer string) { err := s.transaction(func(tx txn) error { const query = `INSERT INTO syncer_peers (peer_address, first_seen, last_connect, synced_blocks, sync_duration) VALUES ($1, $2, 0, 0, 0) ON CONFLICT (peer_address) DO NOTHING` - _, err := tx.Exec(query, peer, sqlTime(time.Now())) + _, err := tx.Exec(query, peer, encode(time.Now())) return err }) if err != nil { @@ -67,7 +67,7 @@ func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { err := s.transaction(func(tx txn) error { info, err := getPeerInfo(tx, peer) if err != nil { - return err + return fmt.Errorf("failed to get peer info: %w", err) } fn(&info) return s.updatePeerInfo(tx, peer, info) @@ -136,7 +136,7 @@ func (s *Store) Ban(peer string, duration time.Duration, reason string) { } err = s.transaction(func(tx txn) error { const query = `INSERT INTO syncer_bans (net_cidr, expiration, reason) VALUES ($1, $2, $3) ON CONFLICT (net_cidr) DO UPDATE SET expiration=EXCLUDED.expiration, reason=EXCLUDED.reason` - _, err := tx.Exec(query, address, sqlTime(time.Now().Add(duration)), reason) + _, err := tx.Exec(query, address, encode(time.Now().Add(duration)), reason) return err }) if err != nil { @@ -181,7 +181,7 @@ func (s *Store) Banned(peer string) (banned bool) { var subnet string var expiration time.Time - err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan(&subnet, (*sqlTime)(&expiration)) + err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan(&subnet, decode(&expiration)) banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows if err == nil && banned { s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) diff --git a/persist/sqlite/types.go b/persist/sqlite/types.go deleted file mode 100644 index f7f0973..0000000 --- a/persist/sqlite/types.go +++ /dev/null @@ -1,126 +0,0 @@ -package sqlite - -import ( - "bytes" - "database/sql" - "database/sql/driver" - "encoding/binary" - "fmt" - "time" - - "go.sia.tech/core/types" -) - -type ( - sqlCurrency types.Currency - sqlTime time.Time -) - -// Scan implements the sql.Scanner interface. -func (sc *sqlCurrency) Scan(src any) error { - buf, ok := src.([]byte) - if !ok { - return fmt.Errorf("cannot scan %T to Currency", src) - } else if len(buf) != 16 { - return fmt.Errorf("cannot scan %d bytes to Currency", len(buf)) - } - - sc.Lo = binary.LittleEndian.Uint64(buf[:8]) - sc.Hi = binary.LittleEndian.Uint64(buf[8:]) - return nil -} - -// Value implements the driver.Valuer interface. -func (sc sqlCurrency) Value() (driver.Value, error) { - buf := make([]byte, 16) - binary.LittleEndian.PutUint64(buf[:8], sc.Lo) - binary.LittleEndian.PutUint64(buf[8:], sc.Hi) - return buf, nil -} - -func (st *sqlTime) Scan(src any) error { - switch src := src.(type) { - case int64: - *st = sqlTime(time.Unix(src, 0)) - return nil - default: - return fmt.Errorf("cannot scan %T to Time", src) - } -} - -func (st sqlTime) Value() (driver.Value, error) { - return time.Time(st).Unix(), nil -} - -func encode[T types.EncoderTo](v T) []byte { - var buf bytes.Buffer - enc := types.NewEncoder(&buf) - v.EncodeTo(enc) - if err := enc.Flush(); err != nil { - panic(err) - } - return buf.Bytes() -} - -func encodeSlice[T types.EncoderTo](v []T) []byte { - var buf bytes.Buffer - enc := types.NewEncoder(&buf) - enc.WritePrefix(len(v)) - for _, e := range v { - e.EncodeTo(enc) - } - if err := enc.Flush(); err != nil { - panic(err) - } - return buf.Bytes() -} - -type decodableSlice[T any] struct { - v *[]T -} - -func (d *decodableSlice[T]) Scan(src any) error { - switch src := src.(type) { - case []byte: - dec := types.NewBufDecoder(src) - s := make([]T, dec.ReadPrefix()) - for i := range s { - dv, ok := any(&s[i]).(types.DecoderFrom) - if !ok { - panic(fmt.Errorf("cannot decode %T", s[i])) - } - dv.DecodeFrom(dec) - } - if err := dec.Err(); err != nil { - return err - } - *d.v = s - return nil - default: - return fmt.Errorf("cannot scan %T to []byte", src) - } -} - -func decodeSlice[T any](v *[]T) sql.Scanner { - return &decodableSlice[T]{v: v} -} - -type decodable[T types.DecoderFrom] struct { - v T -} - -// Scan implements the sql.Scanner interface. -func (d *decodable[T]) Scan(src any) error { - switch src := src.(type) { - case []byte: - dec := types.NewBufDecoder(src) - d.v.DecodeFrom(dec) - return dec.Err() - default: - return fmt.Errorf("cannot scan %T to []byte", src) - } -} - -func decode[T types.DecoderFrom](v T) sql.Scanner { - return &decodable[T]{v} -} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index cbd3de7..0fc0a05 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -15,7 +15,7 @@ func insertAddress(tx txn, addr types.Address) (id int64, err error) { VALUES ($1, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id` - err = tx.QueryRow(query, encode(addr), (*sqlCurrency)(&types.ZeroCurrency)).Scan(&id) + err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency)).Scan(&id) return } @@ -41,7 +41,7 @@ LIMIT $2 OFFSET $3` var eventType string var eventBuf []byte - err := rows.Scan(&eventID, (*sqlTime)(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) + err := rows.Scan(&eventID, decode(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) if err != nil { return fmt.Errorf("failed to scan event: %w", err) } @@ -190,7 +190,7 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin for rows.Next() { var siacoin types.SiacoinElement - err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), (*sqlCurrency)(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) + err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), decode(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } @@ -218,7 +218,7 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund for rows.Next() { var siafund types.SiafundElement - err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, (*sqlCurrency)(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) + err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } @@ -245,7 +245,7 @@ func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, er var siacoin types.Currency var siafund uint64 - if err := rows.Scan((*sqlCurrency)(&siacoin), &siafund); err != nil { + if err := rows.Scan(decode(&siacoin), &siafund); err != nil { return fmt.Errorf("failed to scan address balance: %w", err) } sc = sc.Add(siacoin) @@ -260,7 +260,7 @@ func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, er func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { err = s.transaction(func(tx txn) error { const query = `SELECT siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan((*sqlCurrency)(&sc), &sf) + return tx.QueryRow(query, encode(address)).Scan(decode(&sc), &sf) }) return } From db194563a3871e9a4b83f4c2f3597be31c589ef1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 23 Jan 2024 15:11:41 -0800 Subject: [PATCH 073/630] sqlite: fix tests --- persist/sqlite/peers_test.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 2de6d2f..4f3e26e 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "go.sia.tech/walletd/syncer" + "go.sia.tech/coreutils/syncer" "go.uber.org/zap/zaptest" ) @@ -20,15 +20,13 @@ func TestAddPeer(t *testing.T) { const peer = "1.2.3.4:9981" - if err := db.AddPeer(peer); err != nil { - t.Fatal(err) - } + db.AddPeer(peer) lastConnect := time.Now().Truncate(time.Second) // stored as unix milliseconds syncedBlocks := uint64(15) syncDuration := 5 * time.Second - err = db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { + db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { info.LastConnect = lastConnect info.SyncedBlocks = syncedBlocks info.SyncDuration = syncDuration @@ -37,9 +35,9 @@ func TestAddPeer(t *testing.T) { t.Fatal(err) } - info, err := db.PeerInfo(peer) - if err != nil { - t.Fatal(err) + info, ok := db.PeerInfo(peer) + if !ok { + t.Fatal("expected peer to be in database") } if !info.LastConnect.Equal(lastConnect) { @@ -68,9 +66,7 @@ func TestBanPeer(t *testing.T) { } // ban the peer - if err := db.Ban(peer, time.Second, "test"); err != nil { - t.Fatal(err) - } + db.Ban(peer, time.Second, "test") if !db.Banned(peer) { t.Fatal("expected peer to be banned") @@ -90,11 +86,7 @@ func TestBanPeer(t *testing.T) { } t.Log("banning", subnet) - - if err := db.Ban(subnet.String(), time.Second, "test"); err != nil { - t.Fatal(err) - } - + db.Ban(subnet.String(), time.Second, "test") if !db.Banned(peer) { t.Fatal("expected peer to be banned") } From 21dd5141508bd024b37460b832ab7e74ab89999a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 23 Jan 2024 18:04:54 -0800 Subject: [PATCH 074/630] sqlite: remove txn interface --- persist/sqlite/consensus.go | 30 ++++----- persist/sqlite/init.go | 6 +- persist/sqlite/migrations.go | 2 +- persist/sqlite/peers.go | 16 ++--- persist/sqlite/sql.go | 123 +++++++++++++++-------------------- persist/sqlite/store.go | 17 +++-- persist/sqlite/wallet.go | 26 ++++---- 7 files changed, 104 insertions(+), 116 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index ff9c1b4..9079fc8 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -17,12 +17,12 @@ type proofUpdater interface { UpdateElementProof(*types.StateElement) } -func insertChainIndex(tx txn, index types.ChainIndex) (id int64, err error) { +func insertChainIndex(tx *txn, index types.ChainIndex) (id int64, err error) { err = tx.QueryRow(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`, index.Height, encode(index.ID)).Scan(&id) return } -func applyEvents(tx txn, events []wallet.Event) error { +func applyEvents(tx *txn, events []wallet.Event) error { stmt, err := tx.Prepare(`INSERT INTO events (date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4) RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) @@ -64,7 +64,7 @@ func applyEvents(tx txn, events []wallet.Event) error { return nil } -func deleteSiacoinOutputs(tx txn, spent []types.SiacoinElement) error { +func deleteSiacoinOutputs(tx *txn, spent []types.SiacoinElement) error { addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare lookup statement: %w", err) @@ -108,7 +108,7 @@ func deleteSiacoinOutputs(tx txn, spent []types.SiacoinElement) error { return nil } -func applySiacoinOutputs(tx txn, added map[types.Hash256]types.SiacoinElement) error { +func applySiacoinOutputs(tx *txn, added map[types.Hash256]types.SiacoinElement) error { addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare lookup statement: %w", err) @@ -152,7 +152,7 @@ func applySiacoinOutputs(tx txn, added map[types.Hash256]types.SiacoinElement) e return nil } -func deleteSiafundOutputs(tx txn, spent []types.SiafundElement) error { +func deleteSiafundOutputs(tx *txn, spent []types.SiafundElement) error { addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare lookup statement: %w", err) @@ -199,7 +199,7 @@ func deleteSiafundOutputs(tx txn, spent []types.SiafundElement) error { return nil } -func applySiafundOutputs(tx txn, added map[types.Hash256]types.SiafundElement) error { +func applySiafundOutputs(tx *txn, added map[types.Hash256]types.SiafundElement) error { addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare lookup statement: %w", err) @@ -245,13 +245,13 @@ func applySiafundOutputs(tx txn, added map[types.Hash256]types.SiafundElement) e return nil } -func updateLastIndexedTip(tx txn, tip types.ChainIndex) error { +func updateLastIndexedTip(tx *txn, tip types.ChainIndex) error { _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(tip)) return err } -func getStateElementBatch(stmt *loggedStmt, offset, limit int) ([]types.StateElement, error) { - rows, err := stmt.Query(limit, offset) +func getStateElementBatch(s *stmt, offset, limit int) ([]types.StateElement, error) { + rows, err := s.Query(limit, offset) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } @@ -269,8 +269,8 @@ func getStateElementBatch(stmt *loggedStmt, offset, limit int) ([]types.StateEle return updated, nil } -func updateStateElement(stmt *loggedStmt, se types.StateElement) error { - res, err := stmt.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) +func updateStateElement(s *stmt, se types.StateElement) error { + res, err := s.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) if err != nil { return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) } else if n, err := res.RowsAffected(); err != nil { @@ -282,7 +282,7 @@ func updateStateElement(stmt *loggedStmt, se types.StateElement) error { } // how slow is this going to be 😬? -func updateElementProofs(tx txn, table string, updater proofUpdater) error { +func updateElementProofs(tx *txn, table string, updater proofUpdater) error { stmt, err := tx.Prepare(`SELECT id, merkle_proof, leaf_index FROM ` + table + ` LIMIT $1 OFFSET $2`) if err != nil { return fmt.Errorf("failed to prepare batch statement: %w", err) @@ -314,7 +314,7 @@ func updateElementProofs(tx txn, table string, updater proofUpdater) error { } // applyChainUpdates applies the given chain updates to the database. -func applyChainUpdates(tx txn, updates []*chain.ApplyUpdate) error { +func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate) error { stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) @@ -404,7 +404,7 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) s.updates = append(s.updates, cau) if mayCommit { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { if err := applyChainUpdates(tx, s.updates); err != nil { return err } @@ -424,7 +424,7 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { } // update has been committed, revert it - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index bc6c1a4..24ae378 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -17,13 +17,13 @@ import ( //go:embed init.sql var initDatabase string -func initializeSettings(tx txn, target int64) error { +func initializeSettings(tx *txn, target int64) error { _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_tip) VALUES (0, ?, ?)`, target, encode(types.ChainIndex{})) return err } func (s *Store) initNewDatabase(target int64) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { if _, err := tx.Exec(initDatabase); err != nil { return fmt.Errorf("failed to initialize database: %w", err) } else if err := initializeSettings(tx, target); err != nil { @@ -48,7 +48,7 @@ func (s *Store) upgradeDatabase(current, target int64) error { } }() - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { for _, fn := range migrations[current-1:] { current++ start := time.Now() diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 7aa3692..99d01e1 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,4 +7,4 @@ import ( // migrations is a list of functions that are run to migrate the database from // one version to the next. Migrations are used to update existing databases to // match the schema in init.sql. -var migrations = []func(tx txn, log *zap.Logger) error{} +var migrations = []func(tx *txn, log *zap.Logger) error{} diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 3b59936..4d8de8d 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -13,14 +13,14 @@ import ( "go.uber.org/zap" ) -func getPeerInfo(tx txn, peer string) (syncer.PeerInfo, error) { +func getPeerInfo(tx *txn, peer string) (syncer.PeerInfo, error) { const query = `SELECT first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` var info syncer.PeerInfo err := tx.QueryRow(query, peer).Scan(decode(&info.FirstSeen), decode(&info.LastConnect), &info.SyncedBlocks, &info.SyncDuration) return info, err } -func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error { +func (s *Store) updatePeerInfo(tx *txn, peer string, info syncer.PeerInfo) error { const query = `UPDATE syncer_peers SET first_seen=$1, last_connect=$2, synced_blocks=$3, sync_duration=$4 WHERE peer_address=$5 RETURNING peer_address` err := tx.QueryRow(query, encode(info.FirstSeen), encode(info.LastConnect), info.SyncedBlocks, info.SyncDuration, peer).Scan(&peer) return err @@ -28,7 +28,7 @@ func (s *Store) updatePeerInfo(tx txn, peer string, info syncer.PeerInfo) error // AddPeer adds the given peer to the store. func (s *Store) AddPeer(peer string) { - err := s.transaction(func(tx txn) error { + err := s.transaction(func(tx *txn) error { const query = `INSERT INTO syncer_peers (peer_address, first_seen, last_connect, synced_blocks, sync_duration) VALUES ($1, $2, 0, 0, 0) ON CONFLICT (peer_address) DO NOTHING` _, err := tx.Exec(query, peer, encode(time.Now())) return err @@ -40,7 +40,7 @@ func (s *Store) AddPeer(peer string) { // Peers returns the addresses of all known peers. func (s *Store) Peers() (peers []string) { - err := s.transaction(func(tx txn) error { + err := s.transaction(func(tx *txn) error { const query = `SELECT peer_address FROM syncer_peers` rows, err := tx.Query(query) if err != nil { @@ -64,7 +64,7 @@ func (s *Store) Peers() (peers []string) { // UpdatePeerInfo updates the info for the given peer. func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - err := s.transaction(func(tx txn) error { + err := s.transaction(func(tx *txn) error { info, err := getPeerInfo(tx, peer) if err != nil { return fmt.Errorf("failed to get peer info: %w", err) @@ -81,7 +81,7 @@ func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, bool) { var info syncer.PeerInfo var err error - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { info, err = getPeerInfo(tx, peer) return err }) @@ -134,7 +134,7 @@ func (s *Store) Ban(peer string, duration time.Duration, reason string) { s.log.Error("failed to normalize peer", zap.Error(err)) return } - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `INSERT INTO syncer_bans (net_cidr, expiration, reason) VALUES ($1, $2, $3) ON CONFLICT (net_cidr) DO UPDATE SET expiration=EXCLUDED.expiration, reason=EXCLUDED.reason` _, err := tx.Exec(query, address, encode(time.Now().Add(duration)), reason) return err @@ -176,7 +176,7 @@ func (s *Store) Banned(peer string) (banned bool) { checkSubnets = append(checkSubnets, subnet.String()) } - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { query := `SELECT net_cidr, expiration FROM syncer_bans WHERE net_cidr IN (` + queryPlaceHolders(len(checkSubnets)) + `) ORDER BY expiration DESC LIMIT 1` var subnet string diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index 6ea715f..fb253d1 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -22,122 +22,107 @@ type ( Scan(dest ...any) error } - // A txn is an interface for executing queries within a transaction. - txn interface { - // Exec executes a query without returning any rows. The args are for - // any placeholder parameters in the query. - Exec(query string, args ...any) (sql.Result, error) - // Prepare creates a prepared statement for later queries or executions. - // Multiple queries or executions may be run concurrently from the - // returned statement. The caller must call the statement's Close method - // when the statement is no longer needed. - Prepare(query string) (*loggedStmt, error) - // Query executes a query that returns rows, typically a SELECT. The - // args are for any placeholder parameters in the query. - Query(query string, args ...any) (*loggedRows, error) - // QueryRow executes a query that is expected to return at most one row. - // QueryRow always returns a non-nil value. Errors are deferred until - // Row's Scan method is called. If the query selects no rows, the *Row's - // Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the - // first selected row and discards the rest. - QueryRow(query string, args ...any) *loggedRow - } - - loggedStmt struct { + // A stmt wraps a *sql.Stmt, logging slow queries. + stmt struct { *sql.Stmt query string - log *zap.Logger + + log *zap.Logger } - loggedTxn struct { + // A txn wraps a *sql.Tx, logging slow queries. + txn struct { *sql.Tx log *zap.Logger } - loggedRow struct { + // A row wraps a *sql.Row, logging slow queries. + row struct { *sql.Row log *zap.Logger } - loggedRows struct { + // rows wraps a *sql.Rows, logging slow queries. + rows struct { *sql.Rows + log *zap.Logger } ) -func (lr *loggedRows) Next() bool { +func (r *rows) Next() bool { start := time.Now() - next := lr.Rows.Next() + next := r.Rows.Next() if dur := time.Since(start); dur > longQueryDuration { - lr.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) } return next } -func (lr *loggedRows) Scan(dest ...any) error { +func (r *rows) Scan(dest ...any) error { start := time.Now() - err := lr.Rows.Scan(dest...) + err := r.Rows.Scan(dest...) if dur := time.Since(start); dur > longQueryDuration { - lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) } return err } -func (lr *loggedRow) Scan(dest ...any) error { +func (r *row) Scan(dest ...any) error { start := time.Now() - err := lr.Row.Scan(dest...) + err := r.Row.Scan(dest...) if dur := time.Since(start); dur > longQueryDuration { - lr.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) } return err } -func (ls *loggedStmt) Exec(args ...any) (sql.Result, error) { - return ls.ExecContext(context.Background(), args...) +func (s *stmt) Exec(args ...any) (sql.Result, error) { + return s.ExecContext(context.Background(), args...) } -func (ls *loggedStmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { +func (s *stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { start := time.Now() - result, err := ls.Stmt.ExecContext(ctx, args...) + result, err := s.Stmt.ExecContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow exec", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow exec", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return result, err } -func (ls *loggedStmt) Query(args ...any) (*sql.Rows, error) { - return ls.QueryContext(context.Background(), args...) +func (s *stmt) Query(args ...any) (*sql.Rows, error) { + return s.QueryContext(context.Background(), args...) } -func (ls *loggedStmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { +func (s *stmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { start := time.Now() - rows, err := ls.Stmt.QueryContext(ctx, args...) + rows, err := s.Stmt.QueryContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow query", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow query", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return rows, err } -func (ls *loggedStmt) QueryRow(args ...any) *loggedRow { - return ls.QueryRowContext(context.Background(), args...) +func (s *stmt) QueryRow(args ...any) *row { + return s.QueryRowContext(context.Background(), args...) } -func (ls *loggedStmt) QueryRowContext(ctx context.Context, args ...any) *loggedRow { +func (s *stmt) QueryRowContext(ctx context.Context, args ...any) *row { start := time.Now() - row := ls.Stmt.QueryRowContext(ctx, args...) + r := s.Stmt.QueryRowContext(ctx, args...) if dur := time.Since(start); dur > longQueryDuration { - ls.log.Debug("slow query row", zap.String("query", ls.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + s.log.Debug("slow query row", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRow{row, ls.log.Named("row")} + return &row{r, s.log.Named("row")} } // Exec executes a query without returning any rows. The args are for // any placeholder parameters in the query. -func (lt *loggedTxn) Exec(query string, args ...any) (sql.Result, error) { +func (tx *txn) Exec(query string, args ...any) (sql.Result, error) { start := time.Now() - result, err := lt.Tx.Exec(query, args...) + result, err := tx.Tx.Exec(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } return result, err } @@ -146,30 +131,30 @@ func (lt *loggedTxn) Exec(query string, args ...any) (sql.Result, error) { // Multiple queries or executions may be run concurrently from the // returned statement. The caller must call the statement's Close method // when the statement is no longer needed. -func (lt *loggedTxn) Prepare(query string) (*loggedStmt, error) { +func (tx *txn) Prepare(query string) (*stmt, error) { start := time.Now() - stmt, err := lt.Tx.Prepare(query) + s, err := tx.Tx.Prepare(query) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } else if err != nil { return nil, err } - return &loggedStmt{ - Stmt: stmt, + return &stmt{ + Stmt: s, query: query, - log: lt.log.Named("statement"), + log: tx.log.Named("statement"), }, nil } // Query executes a query that returns rows, typically a SELECT. The // args are for any placeholder parameters in the query. -func (lt *loggedTxn) Query(query string, args ...any) (*loggedRows, error) { +func (tx *txn) Query(query string, args ...any) (*rows, error) { start := time.Now() - rows, err := lt.Tx.Query(query, args...) + r, err := tx.Tx.Query(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRows{rows, lt.log.Named("rows")}, err + return &rows{r, tx.log.Named("rows")}, err } // QueryRow executes a query that is expected to return at most one row. @@ -177,13 +162,13 @@ func (lt *loggedTxn) Query(query string, args ...any) (*loggedRows, error) { // Row's Scan method is called. If the query selects no rows, the *Row's // Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the // first selected row and discards the rest. -func (lt *loggedTxn) QueryRow(query string, args ...any) *loggedRow { +func (tx *txn) QueryRow(query string, args ...any) *row { start := time.Now() - row := lt.Tx.QueryRow(query, args...) + r := tx.Tx.QueryRow(query, args...) if dur := time.Since(start); dur > longQueryDuration { - lt.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + tx.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) } - return &loggedRow{row, lt.log.Named("row")} + return &row{r, tx.log.Named("row")} } func queryPlaceHolders(n int) string { @@ -220,7 +205,7 @@ func getDBVersion(db *sql.DB) (version int64) { } // setDBVersion sets the current version of the database. -func setDBVersion(tx txn, version int64) error { +func setDBVersion(tx *txn, version int64) error { const query = `UPDATE global_settings SET db_version=$1 RETURNING id;` var dbID int64 return tx.QueryRow(query, version).Scan(&dbID) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 0db301e..e50fda5 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -3,6 +3,7 @@ package sqlite import ( "database/sql" "encoding/hex" + "errors" "fmt" "math" "strings" @@ -27,7 +28,7 @@ type ( // function returns an error, the transaction is rolled back. Otherwise, the // transaction is committed. If the transaction fails due to a busy error, it is // retried up to 10 times before returning. -func (s *Store) transaction(fn func(txn) error) error { +func (s *Store) transaction(fn func(*txn) error) error { var err error txnID := hex.EncodeToString(frand.Bytes(4)) log := s.log.Named("transaction").With(zap.String("id", txnID)) @@ -76,25 +77,27 @@ func sqliteFilepath(fp string) string { // doTransaction is a helper function to execute a function within a transaction. If fn returns // an error, the transaction is rolled back. Otherwise, the transaction is // committed. -func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx txn) error) error { +func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { start := time.Now() - tx, err := db.Begin() + dbtx, err := db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - defer tx.Rollback() defer func() { + if err := dbtx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + log.Error("failed to rollback transaction", zap.Error(err)) + } // log the transaction if it took longer than txn duration if time.Since(start) > longTxnDuration { log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) } }() - ltx := &loggedTxn{ - Tx: tx, + tx := &txn{ + Tx: dbtx, log: log, } - if err = fn(ltx); err != nil { + if err = fn(tx); err != nil { return err } else if err = tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 0fc0a05..f5f081d 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -10,7 +10,7 @@ import ( "go.sia.tech/walletd/wallet" ) -func insertAddress(tx txn, addr types.Address) (id int64, err error) { +func insertAddress(tx *txn, addr types.Address) (id int64, err error) { const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) VALUES ($1, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id` @@ -21,7 +21,7 @@ RETURNING id` // WalletEvents returns the events relevant to a wallet, sorted by height descending. func (s *Store) WalletEvents(walletID string, offset, limit int) (events []wallet.Event, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT ev.id, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INNER JOIN chain_indices ci ON (ev.index_id = ci.id) @@ -79,7 +79,7 @@ LIMIT $2 OFFSET $3` // AddWallet adds a wallet to the database. func (s *Store) AddWallet(name string, info json.RawMessage) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { const query = `INSERT INTO wallets (id, extra_data) VALUES ($1, $2)` _, err := tx.Exec(query, name, info) @@ -93,7 +93,7 @@ func (s *Store) AddWallet(name string, info json.RawMessage) error { // DeleteWallet deletes a wallet from the database. This does not stop tracking // addresses that were previously associated with the wallet. func (s *Store) DeleteWallet(name string) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { _, err := tx.Exec(`DELETE FROM wallets WHERE id=$1`, name) return err }) @@ -102,7 +102,7 @@ func (s *Store) DeleteWallet(name string) error { // Wallets returns a map of wallet names to wallet extra data. func (s *Store) Wallets() (map[string]json.RawMessage, error) { wallets := make(map[string]json.RawMessage) - err := s.transaction(func(tx txn) error { + err := s.transaction(func(tx *txn) error { const query = `SELECT id, extra_data FROM wallets` rows, err := tx.Query(query) @@ -126,7 +126,7 @@ func (s *Store) Wallets() (map[string]json.RawMessage, error) { // AddAddress adds an address to a wallet. func (s *Store) AddAddress(walletID string, address types.Address, info json.RawMessage) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { addressID, err := insertAddress(tx, address) if err != nil { return fmt.Errorf("failed to insert address: %w", err) @@ -139,7 +139,7 @@ func (s *Store) AddAddress(walletID string, address types.Address, info json.Raw // RemoveAddress removes an address from a wallet. This does not stop tracking // the address. func (s *Store) RemoveAddress(walletID string, address types.Address) error { - return s.transaction(func(tx txn) error { + return s.transaction(func(tx *txn) error { const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2)` _, err := tx.Exec(query, walletID, encode(address)) return err @@ -149,7 +149,7 @@ func (s *Store) RemoveAddress(walletID string, address types.Address) error { // Addresses returns a map of addresses to their extra data for a wallet. func (s *Store) Addresses(walletID string) (map[types.Address]json.RawMessage, error) { addresses := make(map[types.Address]json.RawMessage) - err := s.transaction(func(tx txn) error { + err := s.transaction(func(tx *txn) error { const query = `SELECT sa.sia_address, wa.extra_data FROM wallet_addresses wa INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) @@ -176,7 +176,7 @@ WHERE wa.wallet_id=$1` // UnspentSiacoinOutputs returns the unspent siacoin outputs for a wallet. func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.SiacoinElement, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) @@ -204,7 +204,7 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin // UnspentSiafundOutputs returns the unspent siafund outputs for a wallet. func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.SiafundElement, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) @@ -231,7 +231,7 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund // WalletBalance returns the total balance of a wallet. func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, siafund_balance FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` @@ -258,7 +258,7 @@ func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, er // AddressBalance returns the balance of a single address. func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` return tx.QueryRow(query, encode(address)).Scan(decode(&sc), &sf) }) @@ -267,7 +267,7 @@ func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uin // Annotate annotates a list of transactions using the wallet's addresses. func (s *Store) Annotate(walletID string, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { - err = s.transaction(func(tx txn) error { + err = s.transaction(func(tx *txn) error { const query = `SELECT sa.id FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` From 44e471d74f541e6f6b8573964d872d49ccefabdf Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 26 Jan 2024 11:00:24 -0800 Subject: [PATCH 075/630] sqlite: fix address balance tracking --- persist/sqlite/consensus.go | 376 +++++++++++++++--------------------- 1 file changed, 161 insertions(+), 215 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9079fc8..889da40 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -9,12 +9,16 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" ) const updateProofBatchSize = 1000 -type proofUpdater interface { +type chainUpdate interface { UpdateElementProof(*types.StateElement) + ForEachTreeNode(func(row, col uint64, h types.Hash256)) + ForEachSiacoinElement(func(types.SiacoinElement, bool)) + ForEachSiafundElement(func(types.SiafundElement, bool)) } func insertChainIndex(tx *txn, index types.ChainIndex) (id int64, err error) { @@ -64,12 +68,14 @@ func applyEvents(tx *txn, events []wallet.Event) error { return nil } -func deleteSiacoinOutputs(tx *txn, spent []types.SiacoinElement) error { - addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) +func applySiacoinElements(tx *txn, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { + addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) VALUES ($1, $2, 0) +ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address +RETURNING id, siacoin_balance`) if err != nil { - return fmt.Errorf("failed to prepare lookup statement: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer addrStmt.Close() + defer addrStatement.Close() updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) if err != nil { @@ -77,172 +83,183 @@ func deleteSiacoinOutputs(tx *txn, spent []types.SiacoinElement) error { } defer updateBalanceStmt.Close() - deleteStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) + addStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, address_id, siacoin_value, merkle_proof, leaf_index, maturity_height) VALUES ($1, $2, $3, $4, $5, $6)`) + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer addStmt.Close() + + spendStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer deleteStmt.Close() + defer spendStmt.Close() + + // using ForEachSiacoinElement creates an interesting problem. The + // ForEachSiacoinElement function is only called once for each element. So + // if a siacoin element is spent and created in the same block, the element + // will not exist in the database. + // + // This creates a problem with balance tracking since it subtracts the + // element value from the balance. However, since the element value was + // never added to the balance in the first place, the balance will be + // incorrect. The solution is to check if the UTXO is in the database before + // decrementing the balance. + // + // This is an important implementation detail since the store must assume + // the chain manager is correct and can't check the integrity of the database + // without reimplementing some of the consensus logic. + cu.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + // sticky error + if err != nil { + return + } else if !relevantAddress(se.SiacoinOutput.Address) { + return + } - for _, se := range spent { // query the address database ID and balance var addressID int64 var balance types.Currency - err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, decode(&balance)) + err = addrStatement.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, decode(&balance)) if err != nil { - return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) + err = fmt.Errorf("failed to query address %q: %w", se.SiacoinOutput.Address, err) + return } - // update the balance - balance = balance.Sub(se.SiacoinOutput.Value) - _, err = updateBalanceStmt.Exec(encode(balance), addressID) - if err != nil { - return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) - } + if spent { + var dummy types.Hash256 + err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) + if errors.Is(err, sql.ErrNoRows) { + // spent output not found, most likely an ephemeral output. ignore + err = nil + return + } else if err != nil { + err = fmt.Errorf("failed to delete output %q: %w", se.ID, err) + return + } - var dummy types.Hash256 - err = deleteStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) - if err != nil { - return fmt.Errorf("failed to delete output %q: %w", se.ID, err) + // update the balance after making sure the utxo was in the database + // and not an ephemeral output + updated, underflow := balance.SubWithUnderflow(se.SiacoinOutput.Value) + if underflow { + log.Panic("balance is negative", zap.Stringer("address", se.SiacoinOutput.Address), zap.String("balance", balance.ExactString()), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString())) + } + _, err = updateBalanceStmt.Exec(encode(updated), addressID) + if err != nil { + err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) + return + } + + log.Debug("removed utxo", zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) + } else { + balance = balance.Add(se.SiacoinOutput.Value) + + // update the balance + _, err = updateBalanceStmt.Exec(encode(balance), addressID) + if err != nil { + err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) + return + } + + // insert the created utxo + _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) + if err != nil { + err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) + return + } + log.Debug("added utxo", zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) } - } - return nil + }) + return err } -func applySiacoinOutputs(tx *txn, added map[types.Hash256]types.SiacoinElement) error { - addrStmt, err := tx.Prepare(`SELECT id, siacoin_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) +func applySiafundElements(tx *txn, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { + addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) VALUES ($1, $2, 0) +ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address +RETURNING id, siafund_balance`) if err != nil { - return fmt.Errorf("failed to prepare lookup statement: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer addrStmt.Close() + defer addrStatement.Close() - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } defer updateBalanceStmt.Close() - addStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, address_id, siacoin_value, merkle_proof, leaf_index, maturity_height) VALUES ($1, $2, $3, $4, $5, $6)`) + addStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, address_id, claim_start, merkle_proof, leaf_index, siafund_value) VALUES ($1, $2, $3, $4, $5, $6)`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer addStmt.Close() - for _, se := range added { - // query the address database ID and balance - var addressID int64 - var balance types.Currency - err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address)).Scan(&addressID, decode(&balance)) - if err != nil { - return fmt.Errorf("failed to lookup address %q: %w", se.SiacoinOutput.Address, err) - } - - // update the balance - balance = balance.Add(se.SiacoinOutput.Value) - _, err = updateBalanceStmt.Exec(encode(balance), addressID) - if err != nil { - return fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) - } - - // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) - if err != nil { - return fmt.Errorf("failed to insert output %q: %w", se.ID, err) - } - } - return nil -} - -func deleteSiafundOutputs(tx *txn, spent []types.SiafundElement) error { - addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) - if err != nil { - return fmt.Errorf("failed to prepare lookup statement: %w", err) - } - defer addrStmt.Close() - - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) - } - defer updateBalanceStmt.Close() - - spendStmt, err := tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) + spendStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer spendStmt.Close() - for _, se := range spent { - // query the address database ID and balance - var addressID int64 - var balance uint64 - err := addrStmt.QueryRow(encode(se.SiafundOutput.Address)).Scan(&addressID, balance) + cu.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + // sticky error if err != nil { - return fmt.Errorf("failed to lookup address %q: %w", se.SiafundOutput.Address, err) + return + } else if !relevantAddress(se.SiafundOutput.Address) { + return } - // update the balance - if balance < se.SiafundOutput.Value { - panic("siafund balance is negative") // developer error - } - balance -= se.SiafundOutput.Value - _, err = updateBalanceStmt.Exec(balance, addressID) - if err != nil { - return fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) - } - - var dummy types.Hash256 - err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) - if err != nil { - return fmt.Errorf("failed to delete output %q: %w", se.ID, err) - } - } - return nil -} - -func applySiafundOutputs(tx *txn, added map[types.Hash256]types.SiafundElement) error { - addrStmt, err := tx.Prepare(`SELECT id, siafund_balance FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) - if err != nil { - return fmt.Errorf("failed to prepare lookup statement: %w", err) - } - defer addrStmt.Close() - - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) - } - defer updateBalanceStmt.Close() - - addStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, address_id, claim_start, siafund_value, merkle_proof, leaf_index) VALUES ($1, $2, $3, $4, $5, $6)`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer addStmt.Close() - - for _, se := range added { // query the address database ID and balance var addressID int64 var balance uint64 - err := addrStmt.QueryRow(encode(se.SiafundOutput.Address)).Scan(&addressID, balance) + err = addrStatement.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, &balance) if err != nil { - return fmt.Errorf("failed to lookup address %q: %w", se.SiafundOutput.Address, err) + err = fmt.Errorf("failed to query address %q: %w", se.SiafundOutput.Address, err) + return } // update the balance - if balance < se.SiafundOutput.Value { - panic("siafund balance is negative") // developer error - } - balance -= se.SiafundOutput.Value - _, err = updateBalanceStmt.Exec(balance, addressID) - if err != nil { - return fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) - } + if spent { + var dummy types.Hash256 + err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) + if errors.Is(err, sql.ErrNoRows) { + // spent output not found, most likely an ephemeral output. + // ignore + err = nil + return + } else if err != nil { + err = fmt.Errorf("failed to delete output %q: %w", se.ID, err) + return + } - _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.ClaimStart), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex) - if err != nil { - return fmt.Errorf("failed to insert output %q: %w", se.ID, err) + // update the balance only if the utxo was successfully deleted + if se.SiafundOutput.Value > balance { + log.Panic("balance is negative", zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("balance", se.SiafundOutput.Value), zap.Stringer("outputID", se.ID), zap.Uint64("value", se.SiafundOutput.Value)) + } + + balance -= se.SiafundOutput.Value + _, err = updateBalanceStmt.Exec(encode(balance), addressID) + if err != nil { + err = fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) + return + } + } else { + balance += se.SiafundOutput.Value + // update the balance + _, err = updateBalanceStmt.Exec(balance, addressID) + if err != nil { + err = fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) + return + } + + // insert the created utxo + _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.ClaimStart), encodeSlice(se.MerkleProof), se.LeafIndex, se.SiafundOutput.Value) + if err != nil { + err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) + return + } } - } - return nil + }) + return err } func updateLastIndexedTip(tx *txn, tip types.ChainIndex) error { @@ -282,7 +299,7 @@ func updateStateElement(s *stmt, se types.StateElement) error { } // how slow is this going to be 😬? -func updateElementProofs(tx *txn, table string, updater proofUpdater) error { +func updateElementProofs(tx *txn, table string, cu chainUpdate) error { stmt, err := tx.Prepare(`SELECT id, merkle_proof, leaf_index FROM ` + table + ` LIMIT $1 OFFSET $2`) if err != nil { return fmt.Errorf("failed to prepare batch statement: %w", err) @@ -304,7 +321,7 @@ func updateElementProofs(tx *txn, table string, updater proofUpdater) error { } for _, se := range elements { - updater.UpdateElementProof(&se) + cu.UpdateElementProof(&se) if err := updateStateElement(updateStmt, se); err != nil { return fmt.Errorf("failed to update state element: %w", err) } @@ -314,7 +331,7 @@ func updateElementProofs(tx *txn, table string, updater proofUpdater) error { } // applyChainUpdates applies the given chain updates to the database. -func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate) error { +func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate, log *zap.Logger) error { stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) @@ -327,7 +344,7 @@ func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate) error { // address. Monitor performance and consider changing this in the // future. From a memory perspective, it would be fine to lazy load all // addresses into memory. - ownsAddress := func(address types.Address) bool { + relevantAddress := func(address types.Address) bool { var dbID int64 err := stmt.QueryRow(encode(address)).Scan(&dbID) if err != nil && !errors.Is(err, sql.ErrNoRows) { @@ -337,51 +354,15 @@ func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate) error { } for _, update := range updates { - events := wallet.AppliedEvents(update.State, update.Block, update, ownsAddress) + events := wallet.AppliedEvents(update.State, update.Block, update, relevantAddress) if err := applyEvents(tx, events); err != nil { return fmt.Errorf("failed to apply events: %w", err) } - var spentSiacoinOutputs []types.SiacoinElement - newSiacoinOutputs := make(map[types.Hash256]types.SiacoinElement) - update.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if !ownsAddress(se.SiacoinOutput.Address) { - return - } - - if spent { - spentSiacoinOutputs = append(spentSiacoinOutputs, se) - delete(newSiacoinOutputs, se.ID) - } else { - newSiacoinOutputs[se.ID] = se - } - }) - - if err := deleteSiacoinOutputs(tx, spentSiacoinOutputs); err != nil { - return fmt.Errorf("failed to delete siacoin outputs: %w", err) - } else if err := applySiacoinOutputs(tx, newSiacoinOutputs); err != nil { - return fmt.Errorf("failed to apply siacoin outputs: %w", err) - } - - var spentSiafundOutputs []types.SiafundElement - newSiafundOutputs := make(map[types.Hash256]types.SiafundElement) - update.ForEachSiafundElement(func(sf types.SiafundElement, spent bool) { - if !ownsAddress(sf.SiafundOutput.Address) { - return - } - - if spent { - spentSiafundOutputs = append(spentSiafundOutputs, sf) - delete(newSiafundOutputs, sf.ID) - } else { - newSiafundOutputs[sf.ID] = sf - } - }) - - if err := deleteSiafundOutputs(tx, spentSiafundOutputs); err != nil { - return fmt.Errorf("failed to delete siafund outputs: %w", err) - } else if err := applySiafundOutputs(tx, newSiafundOutputs); err != nil { - return fmt.Errorf("failed to apply siafund outputs: %w", err) + if err := applySiacoinElements(tx, update, relevantAddress, log.Named("siacoins")); err != nil { + return fmt.Errorf("failed to apply siacoin elements: %w", err) + } else if err := applySiafundElements(tx, update, relevantAddress, log.Named("siafunds")); err != nil { + return fmt.Errorf("failed to apply siafund elements: %w", err) } // update proofs @@ -405,7 +386,7 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) if mayCommit { return s.transaction(func(tx *txn) error { - if err := applyChainUpdates(tx, s.updates); err != nil { + if err := applyChainUpdates(tx, s.updates, s.log.Named("apply")); err != nil { return err } s.updates = nil @@ -417,6 +398,8 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) // ProcessChainRevertUpdate implements chain.Subscriber func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { + log := s.log.Named("revert") + // update hasn't been committed yet if len(s.updates) > 0 && s.updates[len(s.updates)-1].Block.ID() == cru.Block.ID() { s.updates = s.updates[:len(s.updates)-1] @@ -437,7 +420,7 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { // address. Monitor performance and consider changing this in the // future. From a memory perspective, it would be fine to lazy load all // addresses into memory. - ownsAddress := func(address types.Address) bool { + relevantAddress := func(address types.Address) bool { var dbID int64 err := stmt.QueryRow(encode(address)).Scan(&dbID) if err != nil && !errors.Is(err, sql.ErrNoRows) { @@ -446,47 +429,10 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { return err == nil } - var spentSiacoinOutputs []types.SiacoinElement - var spentSiafundOutputs []types.SiafundElement - addedSiacoinOutputs := make(map[types.Hash256]types.SiacoinElement) - addedSiafundOutputs := make(map[types.Hash256]types.SiafundElement) - - cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if !ownsAddress(se.SiacoinOutput.Address) { - return - } - - if !spent { - spentSiacoinOutputs = append(spentSiacoinOutputs, se) - } else { - addedSiacoinOutputs[se.ID] = se - } - }) - - cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if !ownsAddress(se.SiafundOutput.Address) { - return - } - - if !spent { - spentSiafundOutputs = append(spentSiafundOutputs, se) - } else { - addedSiafundOutputs[se.ID] = se - } - }) - - // revert siacoin outputs - if err := deleteSiacoinOutputs(tx, spentSiacoinOutputs); err != nil { - return fmt.Errorf("failed to delete siacoin outputs: %w", err) - } else if err := applySiacoinOutputs(tx, addedSiacoinOutputs); err != nil { - return fmt.Errorf("failed to apply siacoin outputs: %w", err) - } - - // revert siafund outputs - if err := deleteSiafundOutputs(tx, spentSiafundOutputs); err != nil { - return fmt.Errorf("failed to delete siafund outputs: %w", err) - } else if err := applySiafundOutputs(tx, addedSiafundOutputs); err != nil { - return fmt.Errorf("failed to apply siafund outputs: %w", err) + if err := applySiacoinElements(tx, cru, relevantAddress, log.Named("siacoins")); err != nil { + return fmt.Errorf("failed to apply siacoin elements: %w", err) + } else if err := applySiafundElements(tx, cru, relevantAddress, log.Named("siafunds")); err != nil { + return fmt.Errorf("failed to apply siafund elements: %w", err) } // revert events From b7ef75436140b0a952342065e6763fb2d6d5d661 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 26 Jan 2024 11:39:59 -0800 Subject: [PATCH 076/630] api,sqlite,wallet: support immature siacoin balance --- api/api_test.go | 56 +++++++++++++++- api/server.go | 6 +- persist/sqlite/consensus.go | 128 +++++++++++++++++++++++++++++------- persist/sqlite/init.sql | 1 + persist/sqlite/wallet.go | 20 +++--- wallet/manager.go | 4 +- 6 files changed, 176 insertions(+), 39 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index afcb5a5..71f79cf 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -90,7 +90,7 @@ func TestWallet(t *testing.T) { balance, err := wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.IsZero() || balance.Siafunds != 0 { + } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { t.Fatal("balance should be 0") } @@ -163,6 +163,8 @@ func TestWallet(t *testing.T) { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } // transaction should appear in history @@ -179,6 +181,58 @@ func TestWallet(t *testing.T) { } else if len(outputs) != 2 { t.Error("should have two UTXOs, got", len(outputs)) } + + // mine a block to add an immature balance + cs = cm.TipState() + b = types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: addr, Value: cs.BlockReward()}}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // get new balance + balance, err = wc.Balance() + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { + t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + } + + // mine enough blocks for the miner payout to mature + expectedBalance := types.Siacoins(1).Add(b.MinerPayouts[0].Value) + target := cs.MaturityHeight() + for cs.Index.Height < target { + cs = cm.TipState() + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + // get new balance + balance, err = wc.Balance() + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedBalance) { + t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + } } func TestV2(t *testing.T) { diff --git a/api/server.go b/api/server.go index 5030592..6949fba 100644 --- a/api/server.go +++ b/api/server.go @@ -57,7 +57,7 @@ type ( Events(name string, offset, limit int) ([]wallet.Event, error) UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) - WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) + WalletBalance(walletID string) (sc, immatureSC types.Currency, sf uint64, err error) Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) Reserve(ids []types.Hash256, duration time.Duration) error @@ -245,13 +245,13 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { return } - sc, sf, err := s.wm.WalletBalance(name) + sc, isc, sf, err := s.wm.WalletBalance(name) if jc.Check("couldn't load balance", err) != nil { return } jc.Encode(WalletBalanceResponse{ Siacoins: sc, - ImmatureSiacoins: types.ZeroCurrency, + ImmatureSiacoins: isc, Siafunds: sf, }) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 889da40..33fa61d 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -68,16 +68,16 @@ func applyEvents(tx *txn, events []wallet.Event) error { return nil } -func applySiacoinElements(tx *txn, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { - addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) VALUES ($1, $2, 0) +func applySiacoinElements(tx *txn, index types.ChainIndex, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { + addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address -RETURNING id, siacoin_balance`) +RETURNING id, siacoin_balance, immature_siacoin_balance`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer addrStatement.Close() - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) if err != nil { return fmt.Errorf("failed to prepare update statement: %w", err) } @@ -119,8 +119,8 @@ RETURNING id, siacoin_balance`) // query the address database ID and balance var addressID int64 - var balance types.Currency - err = addrStatement.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, decode(&balance)) + var balance, immatureBalance types.Currency + err = addrStatement.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, decode(&balance), decode(&immatureBalance)) if err != nil { err = fmt.Errorf("failed to query address %q: %w", se.SiacoinOutput.Address, err) return @@ -138,13 +138,13 @@ RETURNING id, siacoin_balance`) return } - // update the balance after making sure the utxo was in the database - // and not an ephemeral output - updated, underflow := balance.SubWithUnderflow(se.SiacoinOutput.Value) - if underflow { - log.Panic("balance is negative", zap.Stringer("address", se.SiacoinOutput.Address), zap.String("balance", balance.ExactString()), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString())) + if se.MaturityHeight > index.Height { + immatureBalance = immatureBalance.Sub(se.SiacoinOutput.Value) + } else { + balance = balance.Sub(se.SiacoinOutput.Value) } - _, err = updateBalanceStmt.Exec(encode(updated), addressID) + + _, err = updateBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) if err != nil { err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) return @@ -152,29 +152,36 @@ RETURNING id, siacoin_balance`) log.Debug("removed utxo", zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) } else { - balance = balance.Add(se.SiacoinOutput.Value) - - // update the balance - _, err = updateBalanceStmt.Exec(encode(balance), addressID) + // insert the created utxo + _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) if err != nil { - err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) + err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) return } - // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) + if se.MaturityHeight > index.Height { + immatureBalance = immatureBalance.Add(se.SiacoinOutput.Value) + log.Debug("adding immature balance") + } else { + balance = balance.Add(se.SiacoinOutput.Value) + log.Debug("adding balance") + } + + // update the balance + _, err = updateBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) if err != nil { - err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) + err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) return } - log.Debug("added utxo", zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) + log.Debug("added utxo", zap.Uint64("maturityHeight", se.MaturityHeight), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) } }) return err } func applySiafundElements(tx *txn, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { - addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) VALUES ($1, $2, 0) + // create the address if it doesn't exist + addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siafund_balance`) if err != nil { @@ -211,6 +218,7 @@ RETURNING id, siafund_balance`) // query the address database ID and balance var addressID int64 var balance uint64 + // get the address ID err = addrStatement.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, &balance) if err != nil { err = fmt.Errorf("failed to query address %q: %w", se.SiafundOutput.Address, err) @@ -330,6 +338,67 @@ func updateElementProofs(tx *txn, table string, cu chainUpdate) error { return nil } +func getMaturedValue(tx *txn, index types.ChainIndex) (matured map[int64]types.Currency, err error) { + rows, err := tx.Query(`SELECT address_id, siacoin_value FROM siacoin_elements WHERE maturity_height=$1`, index.Height) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + matured = make(map[int64]types.Currency) + for rows.Next() { + var addressID int64 + var value types.Currency + err := rows.Scan(&addressID, decode(&value)) + if err != nil { + return nil, fmt.Errorf("failed to scan matured balance: %w", err) + } + matured[addressID] = matured[addressID].Add(value) + } + return +} + +func updateImmatureBalance(tx *txn, index types.ChainIndex, revert bool) error { + balanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer balanceStmt.Close() + + updateStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateStmt.Close() + + delta, err := getMaturedValue(tx, index) + if err != nil { + return fmt.Errorf("failed to get matured utxos: %w", err) + } + + for addressID, value := range delta { + var balance, immatureBalance types.Currency + err := balanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) + if err != nil { + return fmt.Errorf("failed to query address %d: %w", addressID, err) + } + + if revert { + balance = balance.Sub(value) + immatureBalance = immatureBalance.Add(value) + } else { + balance = balance.Add(value) + immatureBalance = immatureBalance.Sub(value) + } + + _, err = updateStmt.Exec(encode(balance), encode(immatureBalance), addressID) + if err != nil { + return fmt.Errorf("failed to update address %d: %w", addressID, err) + } + } + return nil +} + // applyChainUpdates applies the given chain updates to the database. func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate, log *zap.Logger) error { stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) @@ -354,12 +423,18 @@ func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate, log *zap.Logger) e } for _, update := range updates { + // mature the immature balance first + if err := updateImmatureBalance(tx, update.State.Index, false); err != nil { + return fmt.Errorf("failed to update immature balance: %w", err) + } + // apply new events events := wallet.AppliedEvents(update.State, update.Block, update, relevantAddress) if err := applyEvents(tx, events); err != nil { return fmt.Errorf("failed to apply events: %w", err) } - if err := applySiacoinElements(tx, update, relevantAddress, log.Named("siacoins")); err != nil { + // apply new elements + if err := applySiacoinElements(tx, update.State.Index, update, relevantAddress, log.Named("siacoins")); err != nil { return fmt.Errorf("failed to apply siacoin elements: %w", err) } else if err := applySiafundElements(tx, update, relevantAddress, log.Named("siafunds")); err != nil { return fmt.Errorf("failed to apply siafund elements: %w", err) @@ -429,7 +504,7 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { return err == nil } - if err := applySiacoinElements(tx, cru, relevantAddress, log.Named("siacoins")); err != nil { + if err := applySiacoinElements(tx, cru.State.Index, cru, relevantAddress, log.Named("siacoins")); err != nil { return fmt.Errorf("failed to apply siacoin elements: %w", err) } else if err := applySiafundElements(tx, cru, relevantAddress, log.Named("siafunds")); err != nil { return fmt.Errorf("failed to apply siafund elements: %w", err) @@ -441,6 +516,11 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { return fmt.Errorf("failed to delete chain index: %w", err) } + // revert immature balance + if err := updateImmatureBalance(tx, cru.State.Index, true); err != nil { + return fmt.Errorf("failed to update immature balance: %w", err) + } + // update proofs if err := updateElementProofs(tx, "siacoin_elements", cru); err != nil { return fmt.Errorf("failed to update siacoin element proofs: %w", err) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 504e3b5..d9d4cff 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -8,6 +8,7 @@ CREATE TABLE sia_addresses ( id INTEGER PRIMARY KEY, sia_address BLOB UNIQUE NOT NULL, siacoin_balance BLOB NOT NULL, + immature_siacoin_balance BLOB NOT NULL, siafund_balance INTEGER NOT NULL ); diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index f5f081d..88f1248 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -11,8 +11,8 @@ import ( ) func insertAddress(tx *txn, addr types.Address) (id int64, err error) { - const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, siafund_balance) -VALUES ($1, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address + const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) +VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id` err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency)).Scan(&id) @@ -230,9 +230,9 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) { +func (s *Store) WalletBalance(walletID string) (sc, immatureSC types.Currency, sf uint64, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT siacoin_balance, siafund_balance FROM sia_addresses sa + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` @@ -242,14 +242,16 @@ func (s *Store) WalletBalance(walletID string) (sc types.Currency, sf uint64, er } for rows.Next() { - var siacoin types.Currency - var siafund uint64 + var addressSC types.Currency + var addressISC types.Currency + var addressSF uint64 - if err := rows.Scan(decode(&siacoin), &siafund); err != nil { + if err := rows.Scan(decode(&addressSC), decode(&addressISC), decode(&addressSF)); err != nil { return fmt.Errorf("failed to scan address balance: %w", err) } - sc = sc.Add(siacoin) - sf += siafund + sc = sc.Add(addressSC) + immatureSC = immatureSC.Add(addressISC) + sf += addressSF } return nil }) diff --git a/wallet/manager.go b/wallet/manager.go index f1b0e62..74039ed 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -36,7 +36,7 @@ type ( UnspentSiacoinOutputs(walletID string) ([]types.SiacoinElement, error) UnspentSiafundOutputs(walletID string) ([]types.SiafundElement, error) Annotate(walletID string, txns []types.Transaction) ([]PoolTransaction, error) - WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) + WalletBalance(walletID string) (sc, immature types.Currency, sf uint64, err error) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) @@ -105,7 +105,7 @@ func (m *Manager) Annotate(name string, pool []types.Transaction) ([]PoolTransac } // WalletBalance returns the balance of the given wallet. -func (m *Manager) WalletBalance(walletID string) (sc types.Currency, sf uint64, err error) { +func (m *Manager) WalletBalance(walletID string) (sc, immature types.Currency, sf uint64, err error) { return m.store.WalletBalance(walletID) } From 75812b1caf2dd6498666a81c3bf566538bb5b356 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 1 Feb 2024 10:56:02 -0400 Subject: [PATCH 077/630] remove duplicate api startup --- cmd/walletd/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 477344c..2b69192 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -197,8 +197,6 @@ func main() { stop := n.Start() log.Println("api: Listening on", l.Addr()) go startWeb(l, n, apiPassword) - log.Println("api: Listening on", l.Addr()) - go startWeb(l, n, apiPassword) signalCh := make(chan os.Signal, 1) signal.Notify(signalCh, os.Interrupt) <-signalCh From 0df580e02db066eb5f1f19d605c1839402084b18 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 1 Feb 2024 11:12:53 -0400 Subject: [PATCH 078/630] sqlite: fix currency encoding --- go.mod | 2 +- go.sum | 2 ++ persist/sqlite/encoding.go | 11 +++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b50ae10..084ce0b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/mattn/go-sqlite3 v1.14.21 - go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc + go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.16.0 diff --git a/go.sum b/go.sum index e25fb79..f29db2e 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc h1:oUCCTOatQIwYkJ2FUWRvJtgU+i/BwlzmzCxoSvmmJVQ= go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= +go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 h1:G2l6fRzAdNZy2z7+FhoG2y8ARtFpR6PkXXTB5tkdfZ8= go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7/go.mod h1:3Mb206QDd3NtRiaHZ2kN87/HKXhcBF6lHVatS7PkViY= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index f011b23..8f98230 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -13,6 +13,11 @@ import ( func encode(obj any) any { switch obj := obj.(type) { + case types.Currency: + buf := make([]byte, 16) + binary.LittleEndian.PutUint64(buf, obj.Lo) + binary.LittleEndian.PutUint64(buf[8:], obj.Hi) + return buf case types.EncoderTo: var buf bytes.Buffer e := types.NewEncoder(&buf) @@ -43,6 +48,12 @@ func (d *decodable) Scan(src any) error { switch src := src.(type) { case []byte: switch v := d.v.(type) { + case *types.Currency: + if len(src) != 16 { + return fmt.Errorf("cannot scan %d bytes into Currency", len(src)) + } + v.Lo = binary.LittleEndian.Uint64(src) + v.Hi = binary.LittleEndian.Uint64(src[8:]) case types.DecoderFrom: dec := types.NewBufDecoder(src) v.DecodeFrom(dec) From 9cb2eea3f8fc80dcfb5f7b6b875f795bbe858941 Mon Sep 17 00:00:00 2001 From: mike76-dev Date: Tue, 6 Feb 2024 13:21:43 +0100 Subject: [PATCH 079/630] Fix typo --- persist/sqlite/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 33fa61d..f194652 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -241,7 +241,7 @@ RETURNING id, siafund_balance`) // update the balance only if the utxo was successfully deleted if se.SiafundOutput.Value > balance { - log.Panic("balance is negative", zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("balance", se.SiafundOutput.Value), zap.Stringer("outputID", se.ID), zap.Uint64("value", se.SiafundOutput.Value)) + log.Panic("balance is negative", zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("balance", balance), zap.Stringer("outputID", se.ID), zap.Uint64("value", se.SiafundOutput.Value)) } balance -= se.SiafundOutput.Value From a3dea349b9726055087f8ccef116766d5e7e4311 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Feb 2024 10:55:30 -0800 Subject: [PATCH 080/630] api,cmd,wallet: return Balance type --- api/api.go | 9 +++------ api/api_test.go | 42 ++++++++++++++++++++-------------------- api/client.go | 2 +- api/server.go | 12 ++++-------- cmd/walletd/main.go | 6 +++--- persist/sqlite/wallet.go | 14 +++++++------- wallet/manager.go | 8 ++++---- wallet/wallet.go | 8 ++++++++ 8 files changed, 51 insertions(+), 50 deletions(-) diff --git a/api/api.go b/api/api.go index f5cb51b..28c20d4 100644 --- a/api/api.go +++ b/api/api.go @@ -4,6 +4,7 @@ import ( "time" "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" ) // A GatewayPeer is a currently-connected peer. @@ -30,12 +31,8 @@ type TxpoolTransactionsResponse struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } -// WalletBalanceResponse is the response type for /wallets/:name/balance. -type WalletBalanceResponse struct { - Siacoins types.Currency `json:"siacoins"` - ImmatureSiacoins types.Currency `json:"immatureSiacoins"` - Siafunds uint64 `json:"siafunds"` -} +// BalanceResponse is the response type for /wallets/:name/balance. +type BalanceResponse wallet.Balance // WalletOutputsResponse is the response type for /wallets/:name/outputs. type WalletOutputsResponse struct { diff --git a/api/api_test.go b/api/api_test.go index 71f79cf..a23960d 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -90,7 +90,7 @@ func TestWallet(t *testing.T) { balance, err := wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { + } else if !balance.Siacoin.IsZero() || !balance.Immature.IsZero() || balance.Siafund != 0 { t.Fatal("balance should be 0") } @@ -161,10 +161,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) - } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + } else if !balance.Siacoin.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoin) + } else if !balance.Immature.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.Immature) } // transaction should appear in history @@ -200,10 +200,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) - } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { - t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + } else if !balance.Siacoin.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoin) + } else if !balance.Immature.Equals(b.MinerPayouts[0].Value) { + t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.Immature) } // mine enough blocks for the miner payout to mature @@ -228,10 +228,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.Equals(expectedBalance) { - t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) - } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + } else if !balance.Siacoin.Equals(expectedBalance) { + t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoin) + } else if !balance.Immature.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.Immature) } } @@ -307,13 +307,13 @@ func TestV2(t *testing.T) { t.Helper() if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) + } else if !primaryBalance.Siacoin.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoin) } if secondaryBalance, err := secondary.Balance(); err != nil { t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) + } else if !secondaryBalance.Siacoin.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoin) } } sendV1 := func() error { @@ -588,13 +588,13 @@ func TestP2P(t *testing.T) { t.Helper() if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) + } else if !primaryBalance.Siacoin.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoin) } if secondaryBalance, err := secondary.Balance(); err != nil { t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) + } else if !secondaryBalance.Siacoin.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoin) } } sendV1 := func() error { diff --git a/api/client.go b/api/client.go index 8365495..8729863 100644 --- a/api/client.go +++ b/api/client.go @@ -139,7 +139,7 @@ func (c *WalletClient) Addresses() (resp map[types.Address]json.RawMessage, err } // Balance returns the current wallet balance. -func (c *WalletClient) Balance() (resp WalletBalanceResponse, err error) { +func (c *WalletClient) Balance() (resp BalanceResponse, err error) { err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.name), &resp) return } diff --git a/api/server.go b/api/server.go index 6949fba..81f63db 100644 --- a/api/server.go +++ b/api/server.go @@ -57,11 +57,11 @@ type ( Events(name string, offset, limit int) ([]wallet.Event, error) UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) - WalletBalance(walletID string) (sc, immatureSC types.Currency, sf uint64, err error) + WalletBalance(walletID string) (wallet.Balance, error) Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) Reserve(ids []types.Hash256, duration time.Duration) error - AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) + AddressBalance(address types.Address) (wallet.Balance, error) } ) @@ -245,15 +245,11 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { return } - sc, isc, sf, err := s.wm.WalletBalance(name) + b, err := s.wm.WalletBalance(name) if jc.Check("couldn't load balance", err) != nil { return } - jc.Encode(WalletBalanceResponse{ - Siacoins: sc, - ImmatureSiacoins: isc, - Siafunds: sf, - }) + jc.Encode(BalanceResponse(b)) } func (s *server) walletsEventsHandler(jc jape.Context) { diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 2b69192..2fc911a 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -240,9 +240,9 @@ func main() { c := initTestnetClient(apiAddr, network, seed) b, err := c.Wallet("primary").Balance() check("Couldn't get balance:", err) - out := fmt.Sprint(b.Siacoins) - if !b.ImmatureSiacoins.IsZero() { - out += fmt.Sprintf(" + %v immature", b.ImmatureSiacoins) + out := fmt.Sprint(b.Siacoin) + if !b.Immature.IsZero() { + out += fmt.Sprintf(" + %v immature", b.Immature) } poolGained, poolLost := testnetTxpoolBalance(c, seed) if !poolGained.IsZero() || !poolLost.IsZero() { diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 88f1248..179df70 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -230,7 +230,7 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(walletID string) (sc, immatureSC types.Currency, sf uint64, err error) { +func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) @@ -249,9 +249,9 @@ func (s *Store) WalletBalance(walletID string) (sc, immatureSC types.Currency, s if err := rows.Scan(decode(&addressSC), decode(&addressISC), decode(&addressSF)); err != nil { return fmt.Errorf("failed to scan address balance: %w", err) } - sc = sc.Add(addressSC) - immatureSC = immatureSC.Add(addressISC) - sf += addressSF + balance.Siacoin = balance.Siacoin.Add(addressSC) + balance.Immature = balance.Immature.Add(addressISC) + balance.Siafund += addressSF } return nil }) @@ -259,10 +259,10 @@ func (s *Store) WalletBalance(walletID string) (sc, immatureSC types.Currency, s } // AddressBalance returns the balance of a single address. -func (s *Store) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { +func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan(decode(&sc), &sf) + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` + return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoin), decode(&balance.Immature), &balance.Siafund) }) return } diff --git a/wallet/manager.go b/wallet/manager.go index 74039ed..8ecb5b3 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -36,9 +36,9 @@ type ( UnspentSiacoinOutputs(walletID string) ([]types.SiacoinElement, error) UnspentSiafundOutputs(walletID string) ([]types.SiafundElement, error) Annotate(walletID string, txns []types.Transaction) ([]PoolTransaction, error) - WalletBalance(walletID string) (sc, immature types.Currency, sf uint64, err error) + WalletBalance(walletID string) (Balance, error) - AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) + AddressBalance(address types.Address) (Balance, error) LastCommittedIndex() (types.ChainIndex, error) } @@ -105,12 +105,12 @@ func (m *Manager) Annotate(name string, pool []types.Transaction) ([]PoolTransac } // WalletBalance returns the balance of the given wallet. -func (m *Manager) WalletBalance(walletID string) (sc, immature types.Currency, sf uint64, err error) { +func (m *Manager) WalletBalance(walletID string) (Balance, error) { return m.store.WalletBalance(walletID) } // AddressBalance returns the balance of the given address. -func (m *Manager) AddressBalance(address types.Address) (sc types.Currency, sf uint64, err error) { +func (m *Manager) AddressBalance(address types.Address) (Balance, error) { return m.store.AddressBalance(address) } diff --git a/wallet/wallet.go b/wallet/wallet.go index 7a806b1..df31f89 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -16,6 +16,14 @@ const ( EventTypeMissedFileContract = "missed file contract" ) +type ( + Balance struct { + Siacoin types.Currency + Immature types.Currency + Siafund uint64 + } +) + // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. From f0fd0cf88dc0b82d7ce080904d7705c29ec86b74 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 13:16:56 -0800 Subject: [PATCH 081/630] cmd,sqlite,wallet: move wallet logic out of sqlite package --- cmd/walletd/testnet.go | 2 +- persist/sqlite/consensus.go | 597 ++++++++++++------------------- persist/sqlite/consensus_test.go | 132 +++++++ persist/sqlite/encoding.go | 9 +- persist/sqlite/init.sql | 7 +- persist/sqlite/wallet.go | 147 +++++--- wallet/update.go | 392 ++++++++++++++++++++ wallet/wallet.go | 125 ++++--- 8 files changed, 939 insertions(+), 472 deletions(-) create mode 100644 persist/sqlite/consensus_test.go create mode 100644 wallet/update.go diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index b87ad21..9a66bd0 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -324,7 +324,7 @@ func printTestnetEvents(c *api.Client, seed wallet.Seed) { check("Couldn't get events:", err) for i := range events { e := events[len(events)-1-i] - switch t := e.Val.(type) { + switch t := e.Data.(type) { case *wallet.EventTransaction: if len(t.SiacoinInputs) == 0 || len(t.SiacoinOutputs) == 0 { continue diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index f194652..99f565f 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1,6 +1,7 @@ package sqlite import ( + "bytes" "database/sql" "encoding/json" "errors" @@ -12,456 +13,339 @@ import ( "go.uber.org/zap" ) -const updateProofBatchSize = 1000 +type updateTx struct { + tx *txn -type chainUpdate interface { - UpdateElementProof(*types.StateElement) - ForEachTreeNode(func(row, col uint64, h types.Hash256)) - ForEachSiacoinElement(func(types.SiacoinElement, bool)) - ForEachSiafundElement(func(types.SiafundElement, bool)) + relevantAddresses map[types.Address]bool } -func insertChainIndex(tx *txn, index types.ChainIndex) (id int64, err error) { - err = tx.QueryRow(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`, index.Height, encode(index.ID)).Scan(&id) +func scanStateElement(s scanner) (se types.StateElement, err error) { + err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) return } -func applyEvents(tx *txn, events []wallet.Event) error { - stmt, err := tx.Prepare(`INSERT INTO events (date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4) RETURNING id`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() +func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + return +} - addRelevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id, block_height) VALUES ($1, $2, $3)`) +func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` + rows, err := ut.tx.Query(query) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } - defer addRelevantAddrStmt.Close() + defer rows.Close() - for _, event := range events { - id, err := insertChainIndex(tx, event.Index) + var elements []types.StateElement + for rows.Next() { + se, err := scanStateElement(rows) if err != nil { - return fmt.Errorf("failed to create chain index: %w", err) + return nil, fmt.Errorf("failed to scan state element: %w", err) } + elements = append(elements, se) + } + return elements, nil +} - buf, err := json.Marshal(event.Val) - if err != nil { - return fmt.Errorf("failed to marshal event: %w", err) - } +func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) error { + const query = `UPDATE siacoin_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` + stmt, err := ut.tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() - var eventID int64 - err = stmt.QueryRow(encode(event.Timestamp), id, event.Val.EventType(), buf).Scan(&eventID) + for _, se := range elements { + var dummy types.Hash256 + err := stmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } - - for _, addr := range event.Relevant { - addressID, err := insertAddress(tx, addr) - if err != nil { - return fmt.Errorf("failed to insert address: %w", err) - } else if _, err := addRelevantAddrStmt.Exec(eventID, addressID, event.Index.Height); err != nil { - return fmt.Errorf("failed to add relevant address: %w", err) - } - } } return nil } -func applySiacoinElements(tx *txn, index types.ChainIndex, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { - addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) -ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address -RETURNING id, siacoin_balance, immature_siacoin_balance`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer addrStatement.Close() - - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) +func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` + rows, err := ut.tx.Query(query) if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } - defer updateBalanceStmt.Close() + defer rows.Close() - addStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, address_id, siacoin_value, merkle_proof, leaf_index, maturity_height) VALUES ($1, $2, $3, $4, $5, $6)`) - if err != nil { - return fmt.Errorf("failed to prepare insert statement: %w", err) + var elements []types.StateElement + for rows.Next() { + se, err := scanStateElement(rows) + if err != nil { + return nil, fmt.Errorf("failed to scan state element: %w", err) + } + elements = append(elements, se) } - defer addStmt.Close() + return elements, nil +} - spendStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) +func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) error { + const query = `UPDATE siafund_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` + stmt, err := ut.tx.Prepare(query) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer spendStmt.Close() - - // using ForEachSiacoinElement creates an interesting problem. The - // ForEachSiacoinElement function is only called once for each element. So - // if a siacoin element is spent and created in the same block, the element - // will not exist in the database. - // - // This creates a problem with balance tracking since it subtracts the - // element value from the balance. However, since the element value was - // never added to the balance in the first place, the balance will be - // incorrect. The solution is to check if the UTXO is in the database before - // decrementing the balance. - // - // This is an important implementation detail since the store must assume - // the chain manager is correct and can't check the integrity of the database - // without reimplementing some of the consensus logic. - cu.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - // sticky error - if err != nil { - return - } else if !relevantAddress(se.SiacoinOutput.Address) { - return - } + defer stmt.Close() - // query the address database ID and balance - var addressID int64 - var balance, immatureBalance types.Currency - err = addrStatement.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, decode(&balance), decode(&immatureBalance)) + for _, se := range elements { + var dummy types.Hash256 + err := stmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) if err != nil { - err = fmt.Errorf("failed to query address %q: %w", se.SiacoinOutput.Address, err) - return + return fmt.Errorf("failed to execute statement: %w", err) } + } + return nil +} - if spent { - var dummy types.Hash256 - err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) - if errors.Is(err, sql.ErrNoRows) { - // spent output not found, most likely an ephemeral output. ignore - err = nil - return - } else if err != nil { - err = fmt.Errorf("failed to delete output %q: %w", se.ID, err) - return - } - - if se.MaturityHeight > index.Height { - immatureBalance = immatureBalance.Sub(se.SiacoinOutput.Value) - } else { - balance = balance.Sub(se.SiacoinOutput.Value) - } +func (ut *updateTx) AddressRelevant(addr types.Address) (bool, error) { + if relevant, ok := ut.relevantAddresses[addr]; ok { + return relevant, nil + } - _, err = updateBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) - if err != nil { - err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) - return - } + var id int64 + err := ut.tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + ut.relevantAddresses[addr] = false + return false, nil + } else if err != nil { + return false, fmt.Errorf("failed to query address: %w", err) + } + ut.relevantAddresses[addr] = true + return ut.relevantAddresses[addr], nil +} - log.Debug("removed utxo", zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) - } else { - // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight) - if err != nil { - err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) - return - } +func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, err error) { + err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoin), decode(&balance.Immature), &balance.Siafund) + return +} - if se.MaturityHeight > index.Height { - immatureBalance = immatureBalance.Add(se.SiacoinOutput.Value) - log.Debug("adding immature balance") - } else { - balance = balance.Add(se.SiacoinOutput.Value) - log.Debug("adding balance") - } +func (ut *updateTx) UpdateBalances(balances []wallet.AddressBalance) error { + const query = `UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE sia_address=$4` + stmt, err := ut.tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() - // update the balance - _, err = updateBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) - if err != nil { - err = fmt.Errorf("failed to update address %q balance: %w", se.SiacoinOutput.Address, err) - return - } - log.Debug("added utxo", zap.Uint64("maturityHeight", se.MaturityHeight), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("outputID", se.ID), zap.String("value", se.SiacoinOutput.Value.ExactString()), zap.Int64("addressID", addressID)) + for _, ab := range balances { + _, err := stmt.Exec(encode(ab.Balance.Siacoin), encode(ab.Balance.Immature), ab.Balance.Siafund, encode(ab.Address)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) } - }) - return err + } + return nil } -func applySiafundElements(tx *txn, cu chainUpdate, relevantAddress func(types.Address) bool, log *zap.Logger) error { - // create the address if it doesn't exist - addrStatement, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) -ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address -RETURNING id, siafund_balance`) +func (ut *updateTx) MaturedSiacoinElements(index types.ChainIndex) (elements []types.SiacoinElement, err error) { + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, a.sia_address +FROM siacoin_elements se +INNER JOIN sia_addresses a ON (se.address_id=a.id) +WHERE maturity_height=$1` + rows, err := ut.tx.Query(query, index.Height) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } - defer addrStatement.Close() + defer rows.Close() - updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) - if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) + for rows.Next() { + element, err := scanSiacoinElement(rows) + if err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + elements = append(elements, element) } - defer updateBalanceStmt.Close() + return +} - addStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, address_id, claim_start, merkle_proof, leaf_index, siafund_value) VALUES ($1, $2, $3, $4, $5, $6)`) +func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement) error { + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { - return fmt.Errorf("failed to prepare insert statement: %w", err) + return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addStmt.Close() + defer addrStmt.Close() - spendStmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) + inserStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to prepare insert statement: %w", err) } - defer spendStmt.Close() + defer inserStmt.Close() - cu.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - // sticky error - if err != nil { - return - } else if !relevantAddress(se.SiafundOutput.Address) { - return - } - - // query the address database ID and balance + for _, se := range elements { var addressID int64 - var balance uint64 - // get the address ID - err = addrStatement.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency)).Scan(&addressID, &balance) + err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) if err != nil { - err = fmt.Errorf("failed to query address %q: %w", se.SiafundOutput.Address, err) - return + return fmt.Errorf("failed to query address: %w", err) } - // update the balance - if spent { - var dummy types.Hash256 - err = spendStmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) - if errors.Is(err, sql.ErrNoRows) { - // spent output not found, most likely an ephemeral output. - // ignore - err = nil - return - } else if err != nil { - err = fmt.Errorf("failed to delete output %q: %w", se.ID, err) - return - } - - // update the balance only if the utxo was successfully deleted - if se.SiafundOutput.Value > balance { - log.Panic("balance is negative", zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("balance", balance), zap.Stringer("outputID", se.ID), zap.Uint64("value", se.SiafundOutput.Value)) - } - - balance -= se.SiafundOutput.Value - _, err = updateBalanceStmt.Exec(encode(balance), addressID) - if err != nil { - err = fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) - return - } - } else { - balance += se.SiafundOutput.Value - // update the balance - _, err = updateBalanceStmt.Exec(balance, addressID) - if err != nil { - err = fmt.Errorf("failed to update address %q balance: %w", se.SiafundOutput.Address, err) - return - } - - // insert the created utxo - _, err = addStmt.Exec(encode(se.ID), addressID, encode(se.ClaimStart), encodeSlice(se.MerkleProof), se.LeafIndex, se.SiafundOutput.Value) - if err != nil { - err = fmt.Errorf("failed to insert output %q: %w", se.ID, err) - return - } + _, err = inserStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addressID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) } - }) - return err -} - -func updateLastIndexedTip(tx *txn, tip types.ChainIndex) error { - _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(tip)) - return err + } + return nil } -func getStateElementBatch(s *stmt, offset, limit int) ([]types.StateElement, error) { - rows, err := s.Query(limit, offset) +func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinOutputID) error { + stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - var updated []types.StateElement - for rows.Next() { - var se types.StateElement - err := rows.Scan(decode(&se.ID), decodeSlice(&se.MerkleProof), &se.LeafIndex) + for _, id := range elements { + var dummy types.Hash256 + err := stmt.QueryRow(encode(id)).Scan(decode(&dummy)) if err != nil { - return nil, fmt.Errorf("failed to scan state element: %w", err) + return fmt.Errorf("failed to delete element %q: %w", id, err) } - updated = append(updated, se) - } - return updated, nil -} - -func updateStateElement(s *stmt, se types.StateElement) error { - res, err := s.Exec(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)) - if err != nil { - return fmt.Errorf("failed to update siacoin element %q: %w", se.ID, err) - } else if n, err := res.RowsAffected(); err != nil { - return fmt.Errorf("failed to get rows affected: %w", err) - } else if n != 1 { - return fmt.Errorf("expected 1 row to be affected, got %d", n) } return nil } -// how slow is this going to be 😬? -func updateElementProofs(tx *txn, table string, cu chainUpdate) error { - stmt, err := tx.Prepare(`SELECT id, merkle_proof, leaf_index FROM ` + table + ` LIMIT $1 OFFSET $2`) +func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement) error { + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { - return fmt.Errorf("failed to prepare batch statement: %w", err) + return fmt.Errorf("failed to prepare address statement: %w", err) } - defer stmt.Close() + defer addrStmt.Close() - updateStmt, err := tx.Prepare(`UPDATE ` + table + ` SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id`) + inserStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) if err != nil { - return fmt.Errorf("failed to prepare update statement: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer updateStmt.Close() + defer inserStmt.Close() - for offset := 0; ; offset += updateProofBatchSize { - elements, err := getStateElementBatch(stmt, offset, updateProofBatchSize) + for _, se := range elements { + var addressID int64 + err := addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) if err != nil { - return fmt.Errorf("failed to get state element batch: %w", err) - } else if len(elements) == 0 { - break + return fmt.Errorf("failed to query address: %w", err) } - for _, se := range elements { - cu.UpdateElementProof(&se) - if err := updateStateElement(updateStmt, se); err != nil { - return fmt.Errorf("failed to update state element: %w", err) - } + _, err = inserStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addressID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) } } return nil } -func getMaturedValue(tx *txn, index types.ChainIndex) (matured map[int64]types.Currency, err error) { - rows, err := tx.Query(`SELECT address_id, siacoin_value FROM siacoin_elements WHERE maturity_height=$1`, index.Height) +func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundOutputID) error { + stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + return fmt.Errorf("failed to prepare statement: %w", err) } - defer rows.Close() + defer stmt.Close() - matured = make(map[int64]types.Currency) - for rows.Next() { - var addressID int64 - var value types.Currency - err := rows.Scan(&addressID, decode(&value)) + for _, id := range elements { + var dummy types.Hash256 + err := stmt.QueryRow(encode(id)).Scan(decode(&dummy)) if err != nil { - return nil, fmt.Errorf("failed to scan matured balance: %w", err) + return fmt.Errorf("failed to delete element %q: %w", id, err) } - matured[addressID] = matured[addressID].Add(value) } - return + return nil } -func updateImmatureBalance(tx *txn, index types.ChainIndex, revert bool) error { - balanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) +func (ut *updateTx) AddEvents(events []wallet.Event) error { + indexStmt, err := ut.tx.Prepare(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to prepare index statement: %w", err) } - defer balanceStmt.Close() + defer indexStmt.Close() - updateStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + eventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to prepare event statement: %w", err) + } + defer eventStmt.Close() + + addrStmt, err := insertAddressStatement(ut.tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) } - defer updateStmt.Close() + defer addrStmt.Close() - delta, err := getMaturedValue(tx, index) + relevantAddrStmt, err := ut.tx.Prepare(`INSERT INTO event_addresses (event_id, address_id) VALUES ($1, $2) ON CONFLICT (event_id, address_id) DO NOTHING`) if err != nil { - return fmt.Errorf("failed to get matured utxos: %w", err) + return fmt.Errorf("failed to prepare relevant address statement: %w", err) } + defer addrStmt.Close() - for addressID, value := range delta { - var balance, immatureBalance types.Currency - err := balanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for _, event := range events { + var chainIndexID int64 + err := indexStmt.QueryRow(event.Index.Height, encode(event.Index.ID)).Scan(&chainIndexID) if err != nil { - return fmt.Errorf("failed to query address %d: %w", addressID, err) + return fmt.Errorf("failed to execute statement: %w", err) } - if revert { - balance = balance.Sub(value) - immatureBalance = immatureBalance.Add(value) - } else { - balance = balance.Add(value) - immatureBalance = immatureBalance.Sub(value) + buf.Reset() + if err := enc.Encode(event.Data); err != nil { + return fmt.Errorf("failed to encode event: %w", err) } - _, err = updateStmt.Exec(encode(balance), encode(immatureBalance), addressID) + var eventID int64 + err = eventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), chainIndexID, event.Data.EventType(), buf.String()).Scan(&eventID) if err != nil { - return fmt.Errorf("failed to update address %d: %w", addressID, err) + return fmt.Errorf("failed to add event: %w", err) } - } - return nil -} -// applyChainUpdates applies the given chain updates to the database. -func applyChainUpdates(tx *txn, updates []*chain.ApplyUpdate, log *zap.Logger) error { - stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() + used := make(map[types.Address]bool) + for _, addr := range event.Relevant { + if used[addr] { + continue + } - // note: this would be more performant for small wallets to load all - // addresses into memory. However, for larger wallets (> 10K addresses), - // this is time consuming. Instead, the database is queried for each - // address. Monitor performance and consider changing this in the - // future. From a memory perspective, it would be fine to lazy load all - // addresses into memory. - relevantAddress := func(address types.Address) bool { - var dbID int64 - err := stmt.QueryRow(encode(address)).Scan(&dbID) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - panic(err) // database error - } - return err == nil - } + var addressID int64 + err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency), 0).Scan(&addressID) + if err != nil { + return fmt.Errorf("failed to get address: %w", err) + } - for _, update := range updates { - // mature the immature balance first - if err := updateImmatureBalance(tx, update.State.Index, false); err != nil { - return fmt.Errorf("failed to update immature balance: %w", err) - } - // apply new events - events := wallet.AppliedEvents(update.State, update.Block, update, relevantAddress) - if err := applyEvents(tx, events); err != nil { - return fmt.Errorf("failed to apply events: %w", err) - } + _, err = relevantAddrStmt.Exec(eventID, addressID) + if err != nil { + return fmt.Errorf("failed to add relevant address: %w", err) + } - // apply new elements - if err := applySiacoinElements(tx, update.State.Index, update, relevantAddress, log.Named("siacoins")); err != nil { - return fmt.Errorf("failed to apply siacoin elements: %w", err) - } else if err := applySiafundElements(tx, update, relevantAddress, log.Named("siafunds")); err != nil { - return fmt.Errorf("failed to apply siafund elements: %w", err) + used[addr] = true } - // update proofs - if err := updateElementProofs(tx, "siacoin_elements", update); err != nil { - return fmt.Errorf("failed to update siacoin element proofs: %w", err) - } else if err := updateElementProofs(tx, "siafund_elements", update); err != nil { - return fmt.Errorf("failed to update siafund element proofs: %w", err) - } } + return nil +} - lastTip := updates[len(updates)-1].State.Index - if err := updateLastIndexedTip(tx, lastTip); err != nil { - return fmt.Errorf("failed to update last indexed tip: %w", err) +// RevertEvents reverts the events that were added in the given block. +func (ut *updateTx) RevertEvents(blockID types.BlockID) error { + var id int64 + err := ut.tx.QueryRow(`DELETE FROM chain_indices WHERE block_id=$1 RETURNING id`, encode(blockID)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil } - return nil + return err } // ProcessChainApplyUpdate implements chain.Subscriber func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { s.updates = append(s.updates, cau) - + log := s.log.Named("ProcessChainApplyUpdate").With(zap.Stringer("index", cau.State.Index)) + log.Debug("received update") if mayCommit { + log.Debug("committing updates", zap.Int("n", len(s.updates))) return s.transaction(func(tx *txn) error { - if err := applyChainUpdates(tx, s.updates, s.log.Named("apply")); err != nil { + utx := &updateTx{ + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } + + if err := wallet.ApplyChainUpdates(utx, s.updates); err != nil { return err } s.updates = nil @@ -473,61 +357,24 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) // ProcessChainRevertUpdate implements chain.Subscriber func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - log := s.log.Named("revert") + log := s.log.Named("ProcessChainRevertUpdate").With(zap.Stringer("index", cru.State.Index)) // update hasn't been committed yet if len(s.updates) > 0 && s.updates[len(s.updates)-1].Block.ID() == cru.Block.ID() { + log.Debug("removed uncommitted update") s.updates = s.updates[:len(s.updates)-1] return nil } + log.Debug("reverting update") // update has been committed, revert it return s.transaction(func(tx *txn) error { - stmt, err := tx.Prepare(`SELECT id FROM sia_addresses WHERE sia_address=$1 LIMIT 1`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() - - // note: this would be more performant for small wallets to load all - // addresses into memory. However, for larger wallets (> 10K addresses), - // this is time consuming. Instead, the database is queried for each - // address. Monitor performance and consider changing this in the - // future. From a memory perspective, it would be fine to lazy load all - // addresses into memory. - relevantAddress := func(address types.Address) bool { - var dbID int64 - err := stmt.QueryRow(encode(address)).Scan(&dbID) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - panic(err) // database error - } - return err == nil + utx := &updateTx{ + tx: tx, + relevantAddresses: make(map[types.Address]bool), } - if err := applySiacoinElements(tx, cru.State.Index, cru, relevantAddress, log.Named("siacoins")); err != nil { - return fmt.Errorf("failed to apply siacoin elements: %w", err) - } else if err := applySiafundElements(tx, cru, relevantAddress, log.Named("siafunds")); err != nil { - return fmt.Errorf("failed to apply siafund elements: %w", err) - } - - // revert events - _, err = tx.Exec(`DELETE FROM chain_indices WHERE block_id=$1`, cru.Block.ID()) - if err != nil { - return fmt.Errorf("failed to delete chain index: %w", err) - } - - // revert immature balance - if err := updateImmatureBalance(tx, cru.State.Index, true); err != nil { - return fmt.Errorf("failed to update immature balance: %w", err) - } - - // update proofs - if err := updateElementProofs(tx, "siacoin_elements", cru); err != nil { - return fmt.Errorf("failed to update siacoin element proofs: %w", err) - } else if err := updateElementProofs(tx, "siafund_elements", cru); err != nil { - return fmt.Errorf("failed to update siafund element proofs: %w", err) - } - return nil + return wallet.RevertChainUpdate(utx, cru) }) } @@ -536,3 +383,7 @@ func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index)) return } + +func insertAddressStatement(tx *txn) (*stmt, error) { + return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) +} diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go new file mode 100644 index 0000000..679b478 --- /dev/null +++ b/persist/sqlite/consensus_test.go @@ -0,0 +1,132 @@ +package sqlite_test + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" +) + +func testNetwork() (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 5 + n.HardforkV2.RequireHeight = 10 + return n, genesisBlock +} + +func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + } + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func TestReorg(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testNetwork() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + cm := chain.NewManager(store, genesisState) + + if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + if err := db.AddWallet("test", nil); err != nil { + t.Fatal(err) + } else if err := db.AddAddress("test", addr, nil); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Immature.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Immature) + } + + // check that a payout event was recorded + events, err := db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height = state.Index.Height + 1 + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + // check that the payout was reverted + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Immature.IsZero() { + t.Fatalf("expected 0, got %v", balance.Immature) + } + + // check that the payout event was reverted + events, err = db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } +} diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 8f98230..966c6b0 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -14,9 +14,10 @@ import ( func encode(obj any) any { switch obj := obj.(type) { case types.Currency: + // Currency is encoded as two 64-bit big-endian integers for sorting buf := make([]byte, 16) - binary.LittleEndian.PutUint64(buf, obj.Lo) - binary.LittleEndian.PutUint64(buf[8:], obj.Hi) + binary.BigEndian.PutUint64(buf, obj.Hi) + binary.BigEndian.PutUint64(buf[8:], obj.Lo) return buf case types.EncoderTo: var buf bytes.Buffer @@ -52,8 +53,8 @@ func (d *decodable) Scan(src any) error { if len(src) != 16 { return fmt.Errorf("cannot scan %d bytes into Currency", len(src)) } - v.Lo = binary.LittleEndian.Uint64(src) - v.Hi = binary.LittleEndian.Uint64(src[8:]) + v.Hi = binary.BigEndian.Uint64(src) + v.Lo = binary.BigEndian.Uint64(src[8:]) case types.DecoderFrom: dec := types.NewBufDecoder(src) v.DecodeFrom(dec) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index d9d4cff..2366b64 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -47,6 +47,8 @@ CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); CREATE TABLE events ( id INTEGER PRIMARY KEY, + event_id BLOB NOT NULL, + maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, event_type TEXT NOT NULL, @@ -54,15 +56,12 @@ CREATE TABLE events ( ); CREATE TABLE event_addresses ( - id INTEGER PRIMARY KEY, event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - block_height INTEGER NOT NULL, /* prevents extra join when querying for events */ - UNIQUE (event_id, address_id) + PRIMARY KEY (event_id, address_id) ); CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); -CREATE INDEX event_addresses_event_id_address_id_block_height ON event_addresses(event_id, address_id, block_height DESC); CREATE TABLE syncer_peers ( peer_address TEXT PRIMARY KEY NOT NULL, diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 179df70..2b90f80 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -19,58 +19,105 @@ RETURNING id` return } +func getWalletEvents(tx *txn, walletID string, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + FROM events ev + INNER JOIN chain_indices ci ON (ev.index_id = ci.id) + WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) + ORDER BY ev.maturity_height DESC + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, walletID, limit, offset) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + for rows.Next() { + var eventID int64 + var event wallet.Event + var eventType string + var eventBuf []byte + + err := rows.Scan(&eventID, decode(&event.ID), &event.MaturityHeight, decode(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan event: %w", err) + } + + switch eventType { + case wallet.EventTypeTransaction: + var tx wallet.EventTransaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + event.Data = &tx + case wallet.EventTypeContractPayout: + var m wallet.EventContractPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) + } + event.Data = &m + case wallet.EventTypeMinerPayout: + var m wallet.EventMinerPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal payout event: %w", err) + } + event.Data = &m + case wallet.EventTypeFoundationSubsidy: + var m wallet.EventFoundationSubsidy + if err = json.Unmarshal(eventBuf, &m); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) + } + default: + return nil, nil, fmt.Errorf("unknown event type: %s", eventType) + } + + events = append(events, event) + eventIDs = append(eventIDs, eventID) + } + return +} + +func (s *Store) getWalletEventRelevantAddresses(tx *txn, walletID string, eventIDs []int64) (map[int64][]types.Address, error) { + query := `SELECT ea.event_id, sa.sia_address +FROM event_addresses ea +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=?)` + + rows, err := tx.Query(query, append(queryArgs(eventIDs), walletID)...) + if err != nil { + return nil, err + } + defer rows.Close() + + relevantAddresses := make(map[int64][]types.Address) + for rows.Next() { + var eventID int64 + var address types.Address + if err := rows.Scan(&eventID, decode(&address)); err != nil { + return nil, fmt.Errorf("failed to scan relevant address: %w", err) + } + relevantAddresses[eventID] = append(relevantAddresses[eventID], address) + } + return relevantAddresses, nil +} + // WalletEvents returns the events relevant to a wallet, sorted by height descending. func (s *Store) WalletEvents(walletID string, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT ev.id, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data -FROM events ev -INNER JOIN chain_indices ci ON (ev.index_id = ci.id) -WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) -ORDER BY ci.height DESC, ev.id ASC -LIMIT $2 OFFSET $3` - - rows, err := tx.Query(query, walletID, limit, offset) + var dbIDs []int64 + events, dbIDs, err = getWalletEvents(tx, walletID, offset, limit) if err != nil { - return err + return fmt.Errorf("failed to get wallet events: %w", err) } - defer rows.Close() - - for rows.Next() { - var eventID int64 - var event wallet.Event - var eventType string - var eventBuf []byte - err := rows.Scan(&eventID, decode(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) - if err != nil { - return fmt.Errorf("failed to scan event: %w", err) - } - - switch eventType { - case wallet.EventTypeTransaction: - var tx wallet.EventTransaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - event.Val = &tx - case wallet.EventTypeMissedFileContract: - var m wallet.EventMissedFileContract - if err = json.Unmarshal(eventBuf, &m); err != nil { - return fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - event.Val = &m - case wallet.EventTypeMinerPayout: - var m wallet.EventMinerPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return fmt.Errorf("failed to unmarshal payout event: %w", err) - } - event.Val = &m - default: - return fmt.Errorf("unknown event type: %s", eventType) - } + eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, walletID, dbIDs) + if err != nil { + return fmt.Errorf("failed to get relevant addresses: %w", err) + } - // event.Relevant = relevantAddresses[eventID] - events = append(events, event) + for i := range events { + events[i].Relevant = eventRelevantAddresses[dbIDs[i]] } return nil }) @@ -79,6 +126,9 @@ LIMIT $2 OFFSET $3` // AddWallet adds a wallet to the database. func (s *Store) AddWallet(name string, info json.RawMessage) error { + if info == nil { + info = json.RawMessage("{}") + } return s.transaction(func(tx *txn) error { const query = `INSERT INTO wallets (id, extra_data) VALUES ($1, $2)` @@ -126,6 +176,9 @@ func (s *Store) Wallets() (map[string]json.RawMessage, error) { // AddAddress adds an address to a wallet. func (s *Store) AddAddress(walletID string, address types.Address, info json.RawMessage) error { + if info == nil { + info = json.RawMessage("{}") + } return s.transaction(func(tx *txn) error { addressID, err := insertAddress(tx, address) if err != nil { @@ -246,7 +299,7 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro var addressISC types.Currency var addressSF uint64 - if err := rows.Scan(decode(&addressSC), decode(&addressISC), decode(&addressSF)); err != nil { + if err := rows.Scan(decode(&addressSC), decode(&addressISC), &addressSF); err != nil { return fmt.Errorf("failed to scan address balance: %w", err) } balance.Siacoin = balance.Siacoin.Add(addressSC) @@ -261,7 +314,7 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro // AddressBalance returns the balance of a single address. func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM address_balance WHERE sia_address=$1` + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoin), decode(&balance.Immature), &balance.Siafund) }) return diff --git a/wallet/update.go b/wallet/update.go new file mode 100644 index 0000000..27a8346 --- /dev/null +++ b/wallet/update.go @@ -0,0 +1,392 @@ +package wallet + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" +) + +type ( + AddressBalance struct { + Address types.Address `json:"address"` + Balance + } + + ApplyTx interface { + SiacoinStateElements() ([]types.StateElement, error) + UpdateSiacoinStateElements([]types.StateElement) error + + SiafundStateElements() ([]types.StateElement, error) + UpdateSiafundStateElements([]types.StateElement) error + + AddressRelevant(types.Address) (bool, error) + AddressBalance(types.Address) (Balance, error) + UpdateBalances([]AddressBalance) error + + MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) + AddSiacoinElements([]types.SiacoinElement) error + RemoveSiacoinElements([]types.SiacoinOutputID) error + + AddSiafundElements([]types.SiafundElement) error + RemoveSiafundElements([]types.SiafundOutputID) error + + AddEvents([]Event) error + } + + RevertTx interface { + RevertEvents(types.BlockID) error + + SiacoinStateElements() ([]types.StateElement, error) + UpdateSiacoinStateElements([]types.StateElement) error + + SiafundStateElements() ([]types.StateElement, error) + UpdateSiafundStateElements([]types.StateElement) error + + AddressRelevant(types.Address) (bool, error) + AddressBalance(types.Address) (Balance, error) + UpdateBalances([]AddressBalance) error + + MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) + AddSiacoinElements([]types.SiacoinElement) error + RemoveSiacoinElements([]types.SiacoinOutputID) error + } +) + +// ApplyChainUpdates atomically applies a set of chain updates to a store +func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { + var events []Event + balances := make(map[types.Address]Balance) + newSiacoinElements := make(map[types.SiacoinOutputID]types.SiacoinElement) + newSiafundElements := make(map[types.SiafundOutputID]types.SiafundElement) + spentSiacoinElements := make(map[types.SiacoinOutputID]bool) + spentSiafundElements := make(map[types.SiafundOutputID]bool) + + updateBalance := func(addr types.Address, fn func(b *Balance)) error { + balance, ok := balances[addr] + if !ok { + var err error + balance, err = tx.AddressBalance(addr) + if err != nil { + return fmt.Errorf("failed to get address balance: %w", err) + } + } + + fn(&balance) + balances[addr] = balance + return nil + } + + // fetch all siacoin and siafund state elements + siacoinStateElements, err := tx.SiacoinStateElements() + if err != nil { + return fmt.Errorf("failed to get siacoin state elements: %w", err) + } + siafundStateElements, err := tx.SiafundStateElements() + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } + + for _, cau := range updates { + // update the immature balance of each relevant address + matured, err := tx.MaturedSiacoinElements(cau.State.Index) + if err != nil { + return fmt.Errorf("failed to get matured siacoin elements: %w", err) + } + for _, se := range matured { + err := updateBalance(se.SiacoinOutput.Address, func(b *Balance) { + b.Immature = b.Immature.Sub(se.SiacoinOutput.Value) + b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + }) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } + } + + // add new siacoin elements to the store + var siacoinElementErr error + cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if siacoinElementErr != nil { + return + } + + if se.LeafIndex == types.EphemeralLeafIndex { + return + } + + relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) + if err != nil { + siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) + return + } else if !relevant { + return + } + + if spent { + delete(newSiacoinElements, types.SiacoinOutputID(se.ID)) + spentSiacoinElements[types.SiacoinOutputID(se.ID)] = true + } else { + newSiacoinElements[types.SiacoinOutputID(se.ID)] = se + } + + err = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { + switch { + case se.MaturityHeight > cau.State.Index.Height: + b.Immature = b.Immature.Add(se.SiacoinOutput.Value) + case spent: + b.Siacoin = b.Siacoin.Sub(se.SiacoinOutput.Value) + default: + b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + } + }) + if err != nil { + siacoinElementErr = fmt.Errorf("failed to update address balance: %w", err) + return + } + }) + if siacoinElementErr != nil { + return fmt.Errorf("failed to add siacoin elements: %w", siacoinElementErr) + } + + var siafundElementErr error + cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if siafundElementErr != nil { + return + } + + relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) + if err != nil { + siafundElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) + return + } else if !relevant { + return + } + + if spent { + delete(newSiafundElements, types.SiafundOutputID(se.ID)) + spentSiafundElements[types.SiafundOutputID(se.ID)] = true + } else { + newSiafundElements[types.SiafundOutputID(se.ID)] = se + } + + err = updateBalance(se.SiafundOutput.Address, func(b *Balance) { + if spent { + b.Siafund -= se.SiafundOutput.Value + } else { + b.Siafund += se.SiafundOutput.Value + } + }) + if err != nil { + siafundElementErr = fmt.Errorf("failed to update address balance: %w", err) + return + } + }) + + // add events + relevant := func(addr types.Address) bool { + relevant, err := tx.AddressRelevant(addr) + if err != nil { + panic(fmt.Errorf("failed to check if address is relevant: %w", err)) + } + return relevant + } + if err != nil { + return fmt.Errorf("failed to get applied events: %w", err) + } + events = append(events, AppliedEvents(cau.State, cau.Block, cau, relevant)...) + + // update siacoin element proofs + for id := range newSiacoinElements { + ele := newSiacoinElements[id] + cau.UpdateElementProof(&ele.StateElement) + newSiacoinElements[id] = ele + } + for i := range siacoinStateElements { + cau.UpdateElementProof(&siacoinStateElements[i]) + } + + // update siafund element proofs + for id := range newSiafundElements { + ele := newSiafundElements[id] + cau.UpdateElementProof(&ele.StateElement) + newSiafundElements[id] = ele + } + for i := range siafundStateElements { + cau.UpdateElementProof(&siafundStateElements[i]) + } + } + + // update the address balances + balanceChanges := make([]AddressBalance, 0, len(balances)) + for addr, balance := range balances { + balanceChanges = append(balanceChanges, AddressBalance{ + Address: addr, + Balance: balance, + }) + } + if err = tx.UpdateBalances(balanceChanges); err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } + + // add the new siacoin elements + siacoinElements := make([]types.SiacoinElement, 0, len(newSiacoinElements)) + for _, ele := range newSiacoinElements { + siacoinElements = append(siacoinElements, ele) + } + if err = tx.AddSiacoinElements(siacoinElements); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } + + // remove the spent siacoin elements + siacoinOutputIDs := make([]types.SiacoinOutputID, 0, len(spentSiacoinElements)) + for id := range spentSiacoinElements { + siacoinOutputIDs = append(siacoinOutputIDs, id) + } + if err = tx.RemoveSiacoinElements(siacoinOutputIDs); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) + } + + // add the new siafund elements + siafundElements := make([]types.SiafundElement, 0, len(newSiafundElements)) + for _, ele := range newSiafundElements { + siafundElements = append(siafundElements, ele) + } + if err = tx.AddSiafundElements(siafundElements); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } + + // remove the spent siafund elements + siafundOutputIDs := make([]types.SiafundOutputID, 0, len(spentSiafundElements)) + for id := range spentSiafundElements { + siafundOutputIDs = append(siafundOutputIDs, id) + } + if err = tx.RemoveSiafundElements(siafundOutputIDs); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + + // add new events + if err = tx.AddEvents(events); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + + // update the siacoin state elements + filteredStateElements := siacoinStateElements[:0] + for _, se := range siacoinStateElements { + if _, ok := spentSiacoinElements[types.SiacoinOutputID(se.ID)]; !ok { + filteredStateElements = append(filteredStateElements, se) + } + } + err = tx.UpdateSiacoinStateElements(filteredStateElements) + if err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) + } + + // update the siafund state elements + filteredStateElements = siafundStateElements[:0] + for _, se := range siafundStateElements { + if _, ok := spentSiafundElements[types.SiafundOutputID(se.ID)]; !ok { + filteredStateElements = append(filteredStateElements, se) + } + } + if err = tx.UpdateSiafundStateElements(filteredStateElements); err != nil { + return fmt.Errorf("failed to update siafund state elements: %w", err) + } + + return nil +} + +// RevertChainUpdate atomically reverts a chain update from a store +func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { + balances := make(map[types.Address]Balance) + newSiacoinElements := make(map[types.SiacoinOutputID]types.SiacoinElement) + newSiafundElements := make(map[types.SiafundOutputID]types.SiafundElement) + spentSiacoinElements := make(map[types.SiacoinOutputID]bool) + spentSiafundElements := make(map[types.SiafundOutputID]bool) + + updateBalance := func(addr types.Address, fn func(b *Balance)) error { + balance, ok := balances[addr] + if !ok { + var err error + balance, err = tx.AddressBalance(addr) + if err != nil { + return fmt.Errorf("failed to get address balance: %w", err) + } + } + + fn(&balance) + balances[addr] = balance + return nil + } + + var siacoinElementErr error + cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) + if err != nil { + siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) + return + } else if !relevant { + return + } + + if !spent { + newSiacoinElements[types.SiacoinOutputID(se.ID)] = se + } else { + spentSiacoinElements[types.SiacoinOutputID(se.ID)] = true + } + + siacoinElementErr = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { + switch { + case se.MaturityHeight > cru.State.Index.Height: + b.Immature = b.Immature.Sub(se.SiacoinOutput.Value) + case !spent: + b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + default: + b.Siacoin = b.Siacoin.Sub(se.SiacoinOutput.Value) + } + }) + }) + if siacoinElementErr != nil { + return fmt.Errorf("failed to update address balance: %w", siacoinElementErr) + } + + var siafundElementErr error + cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) + if err != nil { + siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) + return + } else if !relevant { + return + } + + if !spent { + newSiafundElements[types.SiafundOutputID(se.ID)] = se + } else { + spentSiafundElements[types.SiafundOutputID(se.ID)] = true + } + + siafundElementErr = updateBalance(se.SiafundOutput.Address, func(b *Balance) { + if spent { + b.Siafund -= se.SiafundOutput.Value + } else { + b.Siafund += se.SiafundOutput.Value + } + }) + }) + if siafundElementErr != nil { + return fmt.Errorf("failed to update address balance: %w", siafundElementErr) + } + + balanceChanges := make([]AddressBalance, 0, len(balances)) + for addr, balance := range balances { + balanceChanges = append(balanceChanges, AddressBalance{ + Address: addr, + Balance: balance, + }) + } + if err := tx.UpdateBalances(balanceChanges); err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } + + return tx.RevertEvents(cru.Block.ID()) +} diff --git a/wallet/wallet.go b/wallet/wallet.go index df31f89..ea20199 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -11,9 +11,11 @@ import ( // event type constants const ( - EventTypeTransaction = "transaction" - EventTypeMinerPayout = "miner payout" - EventTypeMissedFileContract = "missed file contract" + EventTypeTransaction = "transaction" + EventTypeMinerPayout = "miner payout" + EventTypeContractPayout = "contract payout" + EventTypeSiafundClaim = "siafund claim" + EventTypeFoundationSubsidy = "foundation subsidy" ) type ( @@ -153,12 +155,18 @@ func Annotate(txn types.Transaction, ownsAddress func(types.Address) bool) PoolT return ptxn } +type eventData interface { + EventType() string +} + // An Event is something interesting that happened on the Sia blockchain. type Event struct { - Index types.ChainIndex - Timestamp time.Time - Relevant []types.Address - Val interface{ EventType() string } + ID types.Hash256 `json:"id"` + Index types.ChainIndex `json:"index"` + Timestamp time.Time `json:"timestamp"` + MaturityHeight uint64 `json:"maturityHeight"` + Relevant []types.Address `json:"relevant"` + Data eventData `json:"data"` } // EventType implements Event. @@ -168,11 +176,14 @@ func (*EventTransaction) EventType() string { return EventTypeTransaction } func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } // EventType implements Event. -func (*EventMissedFileContract) EventType() string { return EventTypeMissedFileContract } +func (*EventFoundationSubsidy) EventType() string { return EventTypeFoundationSubsidy } + +// EventType implements Event. +func (*EventContractPayout) EventType() string { return EventTypeContractPayout } // MarshalJSON implements json.Marshaler. func (e Event) MarshalJSON() ([]byte, error) { - val, _ := json.Marshal(e.Val) + val, _ := json.Marshal(e.Data) return json.Marshal(struct { Timestamp time.Time `json:"timestamp"` Index types.ChainIndex `json:"index"` @@ -183,7 +194,7 @@ func (e Event) MarshalJSON() ([]byte, error) { Timestamp: e.Timestamp, Index: e.Index, Relevant: e.Relevant, - Type: e.Val.EventType(), + Type: e.Data.EventType(), Val: val, }) } @@ -205,16 +216,16 @@ func (e *Event) UnmarshalJSON(data []byte) error { e.Relevant = s.Relevant switch s.Type { case (*EventTransaction)(nil).EventType(): - e.Val = new(EventTransaction) + e.Data = new(EventTransaction) case (*EventMinerPayout)(nil).EventType(): - e.Val = new(EventMinerPayout) - case (*EventMissedFileContract)(nil).EventType(): - e.Val = new(EventMissedFileContract) + e.Data = new(EventMinerPayout) + case (*EventContractPayout)(nil).EventType(): + e.Data = new(EventContractPayout) } - if e.Val == nil { + if e.Data == nil { return fmt.Errorf("unknown event type %q", s.Type) } - return json.Unmarshal(s.Val, e.Val) + return json.Unmarshal(s.Val, e.Data) } // A HostAnnouncement represents a host announcement within an EventTransaction. @@ -250,7 +261,6 @@ type V2FileContract struct { // An EventTransaction represents a transaction that affects the wallet. type EventTransaction struct { - ID types.TransactionID `json:"id"` SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` SiafundInputs []SiafundInput `json:"siafundInputs"` @@ -266,11 +276,15 @@ type EventMinerPayout struct { SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` } -// An EventMissedFileContract represents a file contract that has expired -// without a storage proof -type EventMissedFileContract struct { +type EventFoundationSubsidy struct { + SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` +} + +// An EventContractPayout represents a file contract payout +type EventContractPayout struct { FileContract types.FileContractElement `json:"fileContract"` - MissedOutputs []types.SiacoinElement `json:"missedOutputs"` + SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` + Missed bool `json:"missed"` } // A ChainUpdate is a set of changes to the consensus state. @@ -284,7 +298,7 @@ type ChainUpdate interface { // AppliedEvents extracts a list of relevant events from a chain update. func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { var events []Event - addEvent := func(v interface{ EventType() string }, relevant []types.Address) { + addEvent := func(id types.Hash256, maturityHeight uint64, v eventData, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] @@ -296,10 +310,11 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } events = append(events, Event{ - Timestamp: b.Timestamp, - Index: cs.Index, - Relevant: unique, - Val: v, + Timestamp: b.Timestamp, + Index: cs.Index, + MaturityHeight: maturityHeight, + Relevant: unique, + Data: v, }) } @@ -469,7 +484,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } e := &EventTransaction{ - ID: txn.ID(), SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), @@ -534,7 +548,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f e.Fee = e.Fee.Add(txn.MinerFees[i]) } - addEvent(e, relevant) + addEvent(types.Hash256(txn.ID()), cs.Index.Height, e, relevant) // transaction maturity height is the current block height } // handle v2 transactions @@ -546,7 +560,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f txid := txn.ID() e := &EventTransaction{ - ID: txid, SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), @@ -605,35 +618,61 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } e.Fee = txn.MinerFee - addEvent(e, relevant) + addEvent(types.Hash256(txid), cs.Index.Height, e, relevant) // transaction maturity height is the current block height } // handle missed contracts cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if resolved && !valid { - relevant := relevantContract(fce.FileContract) - if len(relevant) == 0 { - return + if !resolved { + return + } + + relevant := relevantContract(fce.FileContract) + if len(relevant) == 0 { + return + } + + if valid { + for i := range fce.FileContract.ValidProofOutputs { + outputID := types.FileContractID(fce.ID).ValidOutputID(i) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ + FileContract: fce, + SiacoinOutput: sces[outputID], + Missed: false, + }, relevant) } - missedOutputs := make([]types.SiacoinElement, len(fce.FileContract.MissedProofOutputs)) - for i := range missedOutputs { - missedOutputs[i] = sces[types.FileContractID(fce.ID).MissedOutputID(i)] + } else { + for i := range fce.FileContract.MissedProofOutputs { + outputID := types.FileContractID(fce.ID).MissedOutputID(i) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ + FileContract: fce, + SiacoinOutput: sces[outputID], + Missed: true, + }, relevant) } - addEvent(&EventMissedFileContract{ - FileContract: fce, - MissedOutputs: missedOutputs, - }, relevant) } }) // handle block rewards for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { - addEvent(&EventMinerPayout{ - SiacoinOutput: sces[cs.Index.ID.MinerOutputID(i)], + outputID := cs.Index.ID.MinerOutputID(i) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventMinerPayout{ + SiacoinOutput: sces[outputID], }, []types.Address{b.MinerPayouts[i].Address}) } } + // handle foundation subsidy + if relevant(cs.FoundationPrimaryAddress) { + outputID := cs.Index.ID.FoundationOutputID() + sce, ok := sces[outputID] + if ok { + addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventFoundationSubsidy{ + SiacoinOutput: sce, + }, []types.Address{cs.FoundationPrimaryAddress}) + } + } + return events } From 0410c73c8926e77086af9809d434ec2aa792467e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:02:03 -0800 Subject: [PATCH 082/630] wallet: revert field change --- api/api_test.go | 42 ++++++++++++++++++------------------- cmd/walletd/main.go | 6 +++--- persist/sqlite/consensus.go | 4 ++-- persist/sqlite/wallet.go | 6 +++--- wallet/wallet.go | 6 +++--- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index a23960d..ee386b7 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -90,7 +90,7 @@ func TestWallet(t *testing.T) { balance, err := wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoin.IsZero() || !balance.Immature.IsZero() || balance.Siafund != 0 { + } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafund != 0 { t.Fatal("balance should be 0") } @@ -161,10 +161,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoin.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoin) - } else if !balance.Immature.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.Immature) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } // transaction should appear in history @@ -200,10 +200,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoin.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoin) - } else if !balance.Immature.Equals(b.MinerPayouts[0].Value) { - t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.Immature) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { + t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) } // mine enough blocks for the miner payout to mature @@ -228,10 +228,10 @@ func TestWallet(t *testing.T) { balance, err = wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoin.Equals(expectedBalance) { - t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoin) - } else if !balance.Immature.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.Immature) + } else if !balance.Siacoins.Equals(expectedBalance) { + t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } } @@ -307,13 +307,13 @@ func TestV2(t *testing.T) { t.Helper() if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) - } else if !primaryBalance.Siacoin.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoin) + } else if !primaryBalance.Siacoins.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) } if secondaryBalance, err := secondary.Balance(); err != nil { t.Fatal(err) - } else if !secondaryBalance.Siacoin.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoin) + } else if !secondaryBalance.Siacoins.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) } } sendV1 := func() error { @@ -588,13 +588,13 @@ func TestP2P(t *testing.T) { t.Helper() if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) - } else if !primaryBalance.Siacoin.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoin) + } else if !primaryBalance.Siacoins.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) } if secondaryBalance, err := secondary.Balance(); err != nil { t.Fatal(err) - } else if !secondaryBalance.Siacoin.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoin) + } else if !secondaryBalance.Siacoins.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) } } sendV1 := func() error { diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 2fc911a..2b69192 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -240,9 +240,9 @@ func main() { c := initTestnetClient(apiAddr, network, seed) b, err := c.Wallet("primary").Balance() check("Couldn't get balance:", err) - out := fmt.Sprint(b.Siacoin) - if !b.Immature.IsZero() { - out += fmt.Sprintf(" + %v immature", b.Immature) + out := fmt.Sprint(b.Siacoins) + if !b.ImmatureSiacoins.IsZero() { + out += fmt.Sprintf(" + %v immature", b.ImmatureSiacoins) } poolGained, poolLost := testnetTxpoolBalance(c, seed) if !poolGained.IsZero() || !poolLost.IsZero() { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 99f565f..db86654 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -121,7 +121,7 @@ func (ut *updateTx) AddressRelevant(addr types.Address) (bool, error) { } func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, err error) { - err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoin), decode(&balance.Immature), &balance.Siafund) + err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafund) return } @@ -134,7 +134,7 @@ func (ut *updateTx) UpdateBalances(balances []wallet.AddressBalance) error { defer stmt.Close() for _, ab := range balances { - _, err := stmt.Exec(encode(ab.Balance.Siacoin), encode(ab.Balance.Immature), ab.Balance.Siafund, encode(ab.Address)) + _, err := stmt.Exec(encode(ab.Balance.Siacoins), encode(ab.Balance.ImmatureSiacoins), ab.Balance.Siafund, encode(ab.Address)) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 2b90f80..895c709 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -302,8 +302,8 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro if err := rows.Scan(decode(&addressSC), decode(&addressISC), &addressSF); err != nil { return fmt.Errorf("failed to scan address balance: %w", err) } - balance.Siacoin = balance.Siacoin.Add(addressSC) - balance.Immature = balance.Immature.Add(addressISC) + balance.Siacoins = balance.Siacoins.Add(addressSC) + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(addressISC) balance.Siafund += addressSF } return nil @@ -315,7 +315,7 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoin), decode(&balance.Immature), &balance.Siafund) + return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafund) }) return } diff --git a/wallet/wallet.go b/wallet/wallet.go index ea20199..fae53a2 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -20,9 +20,9 @@ const ( type ( Balance struct { - Siacoin types.Currency - Immature types.Currency - Siafund uint64 + Siacoins types.Currency `json:"siacoins"` + ImmatureSiacoins types.Currency `json:"immatureSiacoins"` + Siafund uint64 `json:"siafund"` } ) From c87383f54c4d4faa535d673f7ced1c7b4214f462 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:03:02 -0800 Subject: [PATCH 083/630] sqlite,wallet: fix balance updates --- persist/sqlite/consensus_test.go | 183 ++++++++++++++++++++++++++++++- wallet/update.go | 81 ++++++++++++-- 2 files changed, 246 insertions(+), 18 deletions(-) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 679b478..1858108 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -23,8 +23,8 @@ func testNetwork() (*consensus.Network, types.Block) { n.HardforkOak.Height = 1 n.HardforkASIC.Height = 1 n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 5 - n.HardforkV2.RequireHeight = 10 + n.HardforkV2.AllowHeight = 1000 + n.HardforkV2.RequireHeight = 1000 return n, genesisBlock } @@ -88,8 +88,8 @@ func TestReorg(t *testing.T) { balance, err := db.AddressBalance(addr) if err != nil { t.Fatal(err) - } else if !balance.Immature.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.Immature) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) } // check that a payout event was recorded @@ -118,8 +118,8 @@ func TestReorg(t *testing.T) { balance, err = db.AddressBalance(addr) if err != nil { t.Fatal(err) - } else if !balance.Immature.IsZero() { - t.Fatalf("expected 0, got %v", balance.Immature) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatalf("expected 0, got %v", balance.ImmatureSiacoins) } // check that the payout event was reverted @@ -130,3 +130,174 @@ func TestReorg(t *testing.T) { t.Fatalf("expected 0 events, got %v", len(events)) } } + +func TestEphemeralBalance(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testNetwork() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + cm := chain.NewManager(store, genesisState) + + if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + if err := db.AddWallet("test", nil); err != nil { + t.Fatal(err) + } else if err := db.AddAddress("test", addr, nil); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + 1 + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + + // create a transaction that spends the matured payout + utxos, err := db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + unlockConditions := types.StandardUnlockConditions(pk.PublicKey()) + parentTxn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(100)}, + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100))}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: utxos[0].ID, + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + parentSigHash := cm.TipState().WholeSigHash(parentTxn, utxos[0].ID, 0, 0, nil) + parentSig := pk.SignHash(parentSigHash) + parentTxn.Signatures[0].Signature = parentSig[:] + + outputID := parentTxn.SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: outputID, + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(100)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(outputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(outputID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + txnset := []types.Transaction{parentTxn, txn} + + // broadcast the transactions + revertState := cm.TipState() + if err := cm.AddBlocks([]types.Block{mineBlock(revertState, txnset, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + + // check that the payout was spent + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.IsZero() { + t.Fatalf("expected 0, got %v", balance.Siacoins) + } + + // trigger a reorg + var blocks []types.Block + state := revertState + for i := 0; i < 2; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height = state.Index.Height + 1 + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + // check that the transaction was reverted + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) + } + + // check that only the payout event remains + events, err = db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } +} diff --git a/wallet/update.go b/wallet/update.go index 27a8346..ce36c43 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -95,22 +95,41 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { } for _, se := range matured { err := updateBalance(se.SiacoinOutput.Address, func(b *Balance) { - b.Immature = b.Immature.Sub(se.SiacoinOutput.Value) - b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + b.ImmatureSiacoins = b.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) + b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) }) if err != nil { return fmt.Errorf("failed to update address balance: %w", err) } } + // determine which siacoin and siafund elements are ephemeral + // + // note: I thought we could use LeafIndex == EphemeralLeafIndex, but + // it seems to be set before the subscriber is called. + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cau.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true + } + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true + } + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + } + // add new siacoin elements to the store var siacoinElementErr error cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if siacoinElementErr != nil { return - } - - if se.LeafIndex == types.EphemeralLeafIndex { + } else if ephemeral[se.ID] { return } @@ -132,11 +151,11 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { err = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { switch { case se.MaturityHeight > cau.State.Index.Height: - b.Immature = b.Immature.Add(se.SiacoinOutput.Value) + b.ImmatureSiacoins = b.ImmatureSiacoins.Add(se.SiacoinOutput.Value) case spent: - b.Siacoin = b.Siacoin.Sub(se.SiacoinOutput.Value) + b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) default: - b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) } }) if err != nil { @@ -152,6 +171,8 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { if siafundElementErr != nil { return + } else if ephemeral[se.ID] { + return } relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) @@ -171,6 +192,9 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { err = updateBalance(se.SiafundOutput.Address, func(b *Balance) { if spent { + if b.Siafund < se.SiafundOutput.Value { + panic(fmt.Errorf("negative siafund balance")) + } b.Siafund -= se.SiafundOutput.Value } else { b.Siafund += se.SiafundOutput.Value @@ -318,14 +342,41 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { return nil } + // determine which siacoin and siafund elements are ephemeral + // + // note: I thought we could use LeafIndex == EphemeralLeafIndex, but + // it seems to be set before the subscriber is called. + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cru.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true + } + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true + } + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + } + var siacoinElementErr error cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if siacoinElementErr != nil { + return + } + relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) if err != nil { siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) return } else if !relevant { return + } else if ephemeral[se.ID] { + return } if !spent { @@ -337,11 +388,11 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { siacoinElementErr = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { switch { case se.MaturityHeight > cru.State.Index.Height: - b.Immature = b.Immature.Sub(se.SiacoinOutput.Value) - case !spent: - b.Siacoin = b.Siacoin.Add(se.SiacoinOutput.Value) + b.ImmatureSiacoins = b.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) + case spent: + b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) default: - b.Siacoin = b.Siacoin.Sub(se.SiacoinOutput.Value) + b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) } }) }) @@ -351,12 +402,18 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { var siafundElementErr error cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if siafundElementErr != nil { + return + } + relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) if err != nil { siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) return } else if !relevant { return + } else if ephemeral[se.ID] { + return } if !spent { From a183189e7fbcace42692c377bfee7f6df1a79bf5 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:04:48 -0800 Subject: [PATCH 084/630] sqlite,wallet: fix lint errors --- persist/sqlite/consensus.go | 1 - wallet/update.go | 3 +++ wallet/wallet.go | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index db86654..4145769 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -317,7 +317,6 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { used[addr] = true } - } return nil } diff --git a/wallet/update.go b/wallet/update.go index ce36c43..3dad430 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -8,11 +8,13 @@ import ( ) type ( + // AddressBalance pairs an address with its balance. AddressBalance struct { Address types.Address `json:"address"` Balance } + // An ApplyTx atomically applies a set of updates to a store. ApplyTx interface { SiacoinStateElements() ([]types.StateElement, error) UpdateSiacoinStateElements([]types.StateElement) error @@ -34,6 +36,7 @@ type ( AddEvents([]Event) error } + // RevertTx atomically reverts an update from a store. RevertTx interface { RevertEvents(types.BlockID) error diff --git a/wallet/wallet.go b/wallet/wallet.go index fae53a2..f1db9be 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -19,6 +19,7 @@ const ( ) type ( + // Balance is a summary of a siacoin and siafund balance Balance struct { Siacoins types.Currency `json:"siacoins"` ImmatureSiacoins types.Currency `json:"immatureSiacoins"` @@ -276,6 +277,7 @@ type EventMinerPayout struct { SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` } +// EventFoundationSubsidy represents a foundation subsidy from a block. type EventFoundationSubsidy struct { SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` } From e3fae7c9077de2252c24fa82241bc04b34e723c8 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:10:53 -0800 Subject: [PATCH 085/630] api,sqlite,wallet: revert Balance.Siafunds field change --- api/api_test.go | 2 +- persist/sqlite/consensus.go | 4 ++-- persist/sqlite/wallet.go | 4 ++-- wallet/update.go | 10 +++++----- wallet/wallet.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index ee386b7..71f79cf 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -90,7 +90,7 @@ func TestWallet(t *testing.T) { balance, err := wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafund != 0 { + } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { t.Fatal("balance should be 0") } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 4145769..b5fbea3 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -121,7 +121,7 @@ func (ut *updateTx) AddressRelevant(addr types.Address) (bool, error) { } func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, err error) { - err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafund) + err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) return } @@ -134,7 +134,7 @@ func (ut *updateTx) UpdateBalances(balances []wallet.AddressBalance) error { defer stmt.Close() for _, ab := range balances { - _, err := stmt.Exec(encode(ab.Balance.Siacoins), encode(ab.Balance.ImmatureSiacoins), ab.Balance.Siafund, encode(ab.Address)) + _, err := stmt.Exec(encode(ab.Balance.Siacoins), encode(ab.Balance.ImmatureSiacoins), ab.Balance.Siafunds, encode(ab.Address)) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 895c709..1e09cf8 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -304,7 +304,7 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro } balance.Siacoins = balance.Siacoins.Add(addressSC) balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(addressISC) - balance.Siafund += addressSF + balance.Siafunds += addressSF } return nil }) @@ -315,7 +315,7 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafund) + return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) }) return } diff --git a/wallet/update.go b/wallet/update.go index 3dad430..4784c79 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -195,12 +195,12 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { err = updateBalance(se.SiafundOutput.Address, func(b *Balance) { if spent { - if b.Siafund < se.SiafundOutput.Value { + if b.Siafunds < se.SiafundOutput.Value { panic(fmt.Errorf("negative siafund balance")) } - b.Siafund -= se.SiafundOutput.Value + b.Siafunds -= se.SiafundOutput.Value } else { - b.Siafund += se.SiafundOutput.Value + b.Siafunds += se.SiafundOutput.Value } }) if err != nil { @@ -427,9 +427,9 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { siafundElementErr = updateBalance(se.SiafundOutput.Address, func(b *Balance) { if spent { - b.Siafund -= se.SiafundOutput.Value + b.Siafunds -= se.SiafundOutput.Value } else { - b.Siafund += se.SiafundOutput.Value + b.Siafunds += se.SiafundOutput.Value } }) }) diff --git a/wallet/wallet.go b/wallet/wallet.go index f1db9be..2e1dc95 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -23,7 +23,7 @@ type ( Balance struct { Siacoins types.Currency `json:"siacoins"` ImmatureSiacoins types.Currency `json:"immatureSiacoins"` - Siafund uint64 `json:"siafund"` + Siafunds uint64 `json:"siafunds"` } ) From cb61d5c0b768372562fc69b018507c2696c6683f Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:14:15 -0800 Subject: [PATCH 086/630] sqlite: set foundation event type --- persist/sqlite/wallet.go | 1 + 1 file changed, 1 insertion(+) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 1e09cf8..ace68de 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -68,6 +68,7 @@ func getWalletEvents(tx *txn, walletID string, offset, limit int) (events []wall if err = json.Unmarshal(eventBuf, &m); err != nil { return nil, nil, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) } + event.Data = &m default: return nil, nil, fmt.Errorf("unknown event type: %s", eventType) } From 83c940562ddd7f20df37c860e7566800059da738 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:22:38 -0800 Subject: [PATCH 087/630] sqlite: fix test database leak --- api/api_test.go | 1 + persist/sqlite/consensus_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index 71f79cf..59c40fa 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -71,6 +71,7 @@ func TestWallet(t *testing.T) { t.Fatal(err) } defer ws.Close() + wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) if err != nil { t.Fatal(err) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 1858108..0c9967a 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -48,6 +48,7 @@ func TestReorg(t *testing.T) { if err != nil { t.Fatal(err) } + defer db.Close() bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) if err != nil { @@ -138,6 +139,7 @@ func TestEphemeralBalance(t *testing.T) { if err != nil { t.Fatal(err) } + defer db.Close() bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) if err != nil { From 856cffa48bd04e18d73e8cf599f12697c0dedf2b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 14:52:40 -0800 Subject: [PATCH 088/630] sqlite: add v2 test --- persist/sqlite/consensus_test.go | 156 ++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 3 deletions(-) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 0c9967a..16ff48e 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -13,7 +13,7 @@ import ( "go.uber.org/zap/zaptest" ) -func testNetwork() (*consensus.Network, types.Block) { +func testV1Network() (*consensus.Network, types.Block) { // use a modified version of Zen n, genesisBlock := chain.TestnetZen() n.InitialTarget = types.BlockID{0xFF} @@ -28,6 +28,21 @@ func testNetwork() (*consensus.Network, types.Block) { return n, genesisBlock } +func testV2Network() (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 100 + n.HardforkV2.RequireHeight = 110 + return n, genesisBlock +} + func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { b := types.Block{ ParentID: state.Index.ID, @@ -41,6 +56,24 @@ func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types. return b } +func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + + V2: &types.V2BlockData{ + Transactions: txns, + Height: state.Index.Height + 1, + }, + } + b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + func TestReorg(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() @@ -56,7 +89,7 @@ func TestReorg(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testNetwork() + network, genesisBlock := testV1Network() store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -147,7 +180,7 @@ func TestEphemeralBalance(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testNetwork() + network, genesisBlock := testV1Network() store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -303,3 +336,120 @@ func TestEphemeralBalance(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) } } + +func TestV2(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + cm := chain.NewManager(store, genesisState) + + if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + if err := db.AddWallet("test", nil); err != nil { + t.Fatal(err) + } else if err := db.AddAddress("test", addr, nil); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // mine until the payout matures + maturityHeight := cm.TipState().MaturityHeight() + 1 + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + + // create a v2 transaction that spends the matured payout + utxos, err := db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } + + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, + {Address: addr, Value: types.Siacoins(100)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + + // check that the change was received + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(100)) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a transaction event was recorded + events, err = db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Relevant[0] != addr { + t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) + } +} From 96bc3340b1f4055ecba572f46bab4b5f6489b590 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 15:22:48 -0800 Subject: [PATCH 089/630] sqlite: set last committed index --- persist/sqlite/consensus.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index b5fbea3..a16246b 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -345,12 +345,15 @@ func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) } if err := wallet.ApplyChainUpdates(utx, s.updates); err != nil { - return err + return fmt.Errorf("failed to apply updates: %w", err) + } else if err := setLastCommittedIndex(tx, cau.State.Index); err != nil { + return fmt.Errorf("failed to set last committed index: %w", err) } s.updates = nil return nil }) } + return nil } @@ -373,7 +376,12 @@ func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { relevantAddresses: make(map[types.Address]bool), } - return wallet.RevertChainUpdate(utx, cru) + if err := wallet.RevertChainUpdate(utx, cru); err != nil { + return fmt.Errorf("failed to revert update: %w", err) + } else if err := setLastCommittedIndex(tx, cru.State.Index); err != nil { + return fmt.Errorf("failed to set last committed index: %w", err) + } + return nil }) } @@ -383,6 +391,11 @@ func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { return } +func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(index)) + return err +} + func insertAddressStatement(tx *txn) (*stmt, error) { return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) } From c39cf30b5e5b7dbe0ebda0ff8f35882ccdbb84c9 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 15 Feb 2024 15:48:13 -0800 Subject: [PATCH 090/630] sqlite: remove extra foreign key constraint PRAGMA --- persist/sqlite/init.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index 24ae378..a29a00c 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -70,10 +70,6 @@ func (s *Store) upgradeDatabase(current, target int64) error { func (s *Store) init() error { // calculate the expected final database version target := int64(len(migrations) + 1) - // disable foreign key constraints during migration - if _, err := s.db.Exec("PRAGMA foreign_keys = OFF"); err != nil { - return fmt.Errorf("failed to disable foreign key constraints: %w", err) - } version := getDBVersion(s.db) switch { From 67361d5a2d67a9a9fc837ac163f6e768a4e9649a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Feb 2024 11:09:44 -0800 Subject: [PATCH 091/630] sqlite: extend reorg and balance test --- persist/sqlite/consensus_test.go | 113 +++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 16ff48e..85cde59 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -113,6 +113,7 @@ func TestReorg(t *testing.T) { } expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() // mine a block sending the payout to the wallet if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) @@ -136,6 +137,18 @@ func TestReorg(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) } + // check that the utxo was created + utxos, err := db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + // mine to trigger a reorg var blocks []types.Block state := genesisState @@ -163,6 +176,106 @@ func TestReorg(t *testing.T) { } else if len(events) != 0 { t.Fatalf("expected 0 events, got %v", len(events)) } + + // check that the utxo was removed + utxos, err = db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } + + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight = cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // check that the payout was received + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err = db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err = db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + + // check that the balance was updated + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatalf("expected %v, got %v", types.ZeroCurrency, balance.ImmatureSiacoins) + } else if !balance.Siacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) + } + + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height = state.Index.Height + 1 + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + // check that the balance is correct + balance, err = db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatalf("expected %v, got %v", types.ZeroCurrency, balance.ImmatureSiacoins) + } else if !balance.Siacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) + } + + // check that only the single utxo still exists + utxos, err = db.UnspentSiacoinOutputs("test") + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } } func TestEphemeralBalance(t *testing.T) { From 3a7e79f16ddf2034e4b500b021b736e63a9a042d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Feb 2024 11:11:07 -0800 Subject: [PATCH 092/630] sqlite,wallet: consolidate update tx, fix reorg balance, fix element revert --- persist/sqlite/consensus.go | 28 +++++---- persist/sqlite/init.sql | 3 +- wallet/update.go | 118 +++++++++++++++++++++++++----------- 3 files changed, 102 insertions(+), 47 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index a16246b..0f3e67b 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -170,20 +170,20 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement) error { } defer addrStmt.Close() - inserStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } - defer inserStmt.Close() + defer insertStmt.Close() for _, se := range elements { var addressID int64 - err := addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) + err = addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) if err != nil { return fmt.Errorf("failed to query address: %w", err) } - _, err = inserStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addressID) + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addressID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -215,20 +215,20 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement) error { } defer addrStmt.Close() - inserStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer inserStmt.Close() + defer insertStmt.Close() for _, se := range elements { var addressID int64 - err := addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) + err = addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) if err != nil { return fmt.Errorf("failed to query address: %w", err) } - _, err = inserStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addressID) + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addressID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -254,7 +254,7 @@ func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundOutputID) erro } func (ut *updateTx) AddEvents(events []wallet.Event) error { - indexStmt, err := ut.tx.Prepare(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`) + indexStmt, err := insertIndexStmt(ut.tx) if err != nil { return fmt.Errorf("failed to prepare index statement: %w", err) } @@ -321,10 +321,10 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { return nil } -// RevertEvents reverts the events that were added in the given block. -func (ut *updateTx) RevertEvents(blockID types.BlockID) error { +// RevertEvents reverts any events that were added by the index +func (ut *updateTx) RevertEvents(index types.ChainIndex) error { var id int64 - err := ut.tx.QueryRow(`DELETE FROM chain_indices WHERE block_id=$1 RETURNING id`, encode(blockID)).Scan(&id) + err := ut.tx.QueryRow(`DELETE FROM chain_indices WHERE block_id=$1 AND height=$2 RETURNING id`, encode(index.ID), index.Height).Scan(&id) if errors.Is(err, sql.ErrNoRows) { return nil } @@ -399,3 +399,7 @@ func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { func insertAddressStatement(tx *txn) (*stmt, error) { return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) } + +func insertIndexStmt(tx *txn) (*stmt, error) { + return tx.Prepare(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`) +} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 2366b64..b7c1767 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -21,6 +21,7 @@ CREATE TABLE siacoin_elements ( address_id INTEGER NOT NULL REFERENCES sia_addresses (id) ); CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height ON siacoin_elements (maturity_height); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -48,9 +49,9 @@ CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); CREATE TABLE events ( id INTEGER PRIMARY KEY, event_id BLOB NOT NULL, + index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, - index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, event_type TEXT NOT NULL, event_data TEXT NOT NULL ); diff --git a/wallet/update.go b/wallet/update.go index 4784c79..a48e984 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -14,45 +14,38 @@ type ( Balance } - // An ApplyTx atomically applies a set of updates to a store. - ApplyTx interface { + UpdateTx interface { SiacoinStateElements() ([]types.StateElement, error) UpdateSiacoinStateElements([]types.StateElement) error SiafundStateElements() ([]types.StateElement, error) UpdateSiafundStateElements([]types.StateElement) error - AddressRelevant(types.Address) (bool, error) - AddressBalance(types.Address) (Balance, error) - UpdateBalances([]AddressBalance) error - - MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) AddSiacoinElements([]types.SiacoinElement) error RemoveSiacoinElements([]types.SiacoinOutputID) error AddSiafundElements([]types.SiafundElement) error RemoveSiafundElements([]types.SiafundOutputID) error + MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) + + AddressRelevant(types.Address) (bool, error) + AddressBalance(types.Address) (Balance, error) + UpdateBalances([]AddressBalance) error + } + + // An ApplyTx atomically applies a set of updates to a store. + ApplyTx interface { + UpdateTx + AddEvents([]Event) error } // RevertTx atomically reverts an update from a store. RevertTx interface { - RevertEvents(types.BlockID) error + UpdateTx - SiacoinStateElements() ([]types.StateElement, error) - UpdateSiacoinStateElements([]types.StateElement) error - - SiafundStateElements() ([]types.StateElement, error) - UpdateSiafundStateElements([]types.StateElement) error - - AddressRelevant(types.Address) (bool, error) - AddressBalance(types.Address) (Balance, error) - UpdateBalances([]AddressBalance) error - - MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) - AddSiacoinElements([]types.SiacoinElement) error - RemoveSiacoinElements([]types.SiacoinOutputID) error + RevertEvents(index types.ChainIndex) error } ) @@ -325,10 +318,11 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { // RevertChainUpdate atomically reverts a chain update from a store func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { balances := make(map[types.Address]Balance) - newSiacoinElements := make(map[types.SiacoinOutputID]types.SiacoinElement) - newSiafundElements := make(map[types.SiafundOutputID]types.SiafundElement) - spentSiacoinElements := make(map[types.SiacoinOutputID]bool) - spentSiafundElements := make(map[types.SiafundOutputID]bool) + + var deletedSiacoinElements []types.SiacoinOutputID + var addedSiacoinElements []types.SiacoinElement + var deletedSiafundElements []types.SiafundOutputID + var addedSiafundElements []types.SiafundElement updateBalance := func(addr types.Address, fn func(b *Balance)) error { balance, ok := balances[addr] @@ -366,6 +360,26 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { } } + // revert the immature balance of each relevant address + revertedIndex := types.ChainIndex{ + Height: cru.State.Index.Height + 1, + ID: cru.Block.ID(), + } + + matured, err := tx.MaturedSiacoinElements(revertedIndex) + if err != nil { + return fmt.Errorf("failed to get matured siacoin elements: %w", err) + } + for _, se := range matured { + err := updateBalance(se.SiacoinOutput.Address, func(b *Balance) { + b.ImmatureSiacoins = b.ImmatureSiacoins.Add(se.SiacoinOutput.Value) + b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) + }) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } + } + var siacoinElementErr error cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if siacoinElementErr != nil { @@ -382,10 +396,12 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { return } - if !spent { - newSiacoinElements[types.SiacoinOutputID(se.ID)] = se + if spent { + // re-add any spent siacoin elements + addedSiacoinElements = append(addedSiacoinElements, se) } else { - spentSiacoinElements[types.SiacoinOutputID(se.ID)] = true + // delete any created siacoin elements + deletedSiacoinElements = append(deletedSiacoinElements, types.SiacoinOutputID(se.ID)) } siacoinElementErr = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { @@ -419,17 +435,19 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { return } - if !spent { - newSiafundElements[types.SiafundOutputID(se.ID)] = se + if spent { + // re-add any spent siafund elements + addedSiafundElements = append(addedSiafundElements, se) } else { - spentSiafundElements[types.SiafundOutputID(se.ID)] = true + // delete any created siafund elements + deletedSiafundElements = append(deletedSiafundElements, types.SiafundOutputID(se.ID)) } siafundElementErr = updateBalance(se.SiafundOutput.Address, func(b *Balance) { if spent { - b.Siafunds -= se.SiafundOutput.Value - } else { b.Siafunds += se.SiafundOutput.Value + } else { + b.Siafunds -= se.SiafundOutput.Value } }) }) @@ -448,5 +466,37 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { return fmt.Errorf("failed to update address balance: %w", err) } - return tx.RevertEvents(cru.Block.ID()) + // revert siacoin element changes + if err := tx.AddSiacoinElements(addedSiacoinElements); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } else if err := tx.RemoveSiacoinElements(deletedSiacoinElements); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) + } + + // update siacoin element proofs + siacoinElements, err := tx.SiacoinStateElements() + if err != nil { + return fmt.Errorf("failed to get siacoin state elements: %w", err) + } + for i := range siacoinElements { + cru.UpdateElementProof(&siacoinElements[i]) + } + + // revert siafund element changes + if err := tx.AddSiafundElements(addedSiafundElements); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } else if err := tx.RemoveSiafundElements(deletedSiafundElements); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + + // update siafund element proofs + siafundElements, err := tx.SiafundStateElements() + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } + for i := range siafundElements { + cru.UpdateElementProof(&siafundElements[i]) + } + + return tx.RevertEvents(revertedIndex) } From ddc18965d2c4eed95bae00e56d63a228ba0d9dfa Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Feb 2024 11:24:43 -0800 Subject: [PATCH 093/630] wallet: add doc string --- wallet/update.go | 1 + 1 file changed, 1 insertion(+) diff --git a/wallet/update.go b/wallet/update.go index a48e984..f9a140a 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -14,6 +14,7 @@ type ( Balance } + // An UpdateTx atomically updates the state of a store. UpdateTx interface { SiacoinStateElements() ([]types.StateElement, error) UpdateSiacoinStateElements([]types.StateElement) error From b6078b7b139abab77bd001c5fb132a4e99c56d2e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Feb 2024 15:09:37 -0800 Subject: [PATCH 094/630] wallet: fix missing event ID --- wallet/wallet.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 2e1dc95..510b239 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -186,12 +186,14 @@ func (*EventContractPayout) EventType() string { return EventTypeContractPayout func (e Event) MarshalJSON() ([]byte, error) { val, _ := json.Marshal(e.Data) return json.Marshal(struct { + ID types.Hash256 `json:"id"` Timestamp time.Time `json:"timestamp"` Index types.ChainIndex `json:"index"` Relevant []types.Address `json:"relevant"` Type string `json:"type"` Val json.RawMessage `json:"val"` }{ + ID: e.ID, Timestamp: e.Timestamp, Index: e.Index, Relevant: e.Relevant, @@ -203,15 +205,17 @@ func (e Event) MarshalJSON() ([]byte, error) { // UnmarshalJSON implements json.Unarshaler. func (e *Event) UnmarshalJSON(data []byte) error { var s struct { - Timestamp time.Time - Index types.ChainIndex - Relevant []types.Address - Type string - Val json.RawMessage + ID types.Hash256 `json:"id"` + Timestamp time.Time `json:"timestamp"` + Index types.ChainIndex `json:"index"` + Relevant []types.Address `json:"relevant"` + Type string `json:"type"` + Val json.RawMessage `json:"val"` } if err := json.Unmarshal(data, &s); err != nil { return err } + e.ID = s.ID e.Timestamp = s.Timestamp e.Index = s.Index e.Relevant = s.Relevant @@ -312,6 +316,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } events = append(events, Event{ + ID: id, Timestamp: b.Timestamp, Index: cs.Index, MaturityHeight: maturityHeight, From 182983ae47c8adf0e043425c627d8c27a040a39d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Feb 2024 15:09:58 -0800 Subject: [PATCH 095/630] sqlite: extend test to check ids, fix event sort order --- persist/sqlite/consensus_test.go | 24 +++++++++++++++++++++++- persist/sqlite/wallet.go | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 85cde59..1d064ba 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -318,8 +318,10 @@ func TestEphemeralBalance(t *testing.T) { expectedPayout := cm.TipState().BlockReward() maturityHeight := cm.TipState().MaturityHeight() + 1 + block := mineBlock(cm.TipState(), nil, addr) + minerPayoutID := block.ID().MinerOutputID(0) // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + if err := cm.AddBlocks([]types.Block{block}); err != nil { t.Fatal(err) } @@ -339,6 +341,8 @@ func TestEphemeralBalance(t *testing.T) { t.Fatalf("expected 1 event, got %v", len(events)) } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].ID != types.Hash256(minerPayoutID) { + t.Fatalf("expected %v, got %v", minerPayoutID, events[0].ID) } // mine until the payout matures @@ -419,6 +423,24 @@ func TestEphemeralBalance(t *testing.T) { t.Fatalf("expected 0, got %v", balance.Siacoins) } + // check that both transactions were added + events, err = db.WalletEvents("test", 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 3 { // 1 payout, 2 transactions + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[2].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[2].Data.EventType()) + } else if events[1].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[1].Data.EventType()) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first + t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) + } else if events[0].ID != types.Hash256(txn.ID()) { // child txn second + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } + // trigger a reorg var blocks []types.Block state := revertState diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index ace68de..b2136e9 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -24,7 +24,7 @@ func getWalletEvents(tx *txn, walletID string, offset, limit int) (events []wall FROM events ev INNER JOIN chain_indices ci ON (ev.index_id = ci.id) WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) - ORDER BY ev.maturity_height DESC + ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` rows, err := tx.Query(query, walletID, limit, offset) From d17b96fd6c35216b8df8943c3f88a21873fba932 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Thu, 22 Feb 2024 11:44:12 -0500 Subject: [PATCH 096/630] ci: fix ui workflow --- .github/workflows/ui.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 74dc4dc..ba2fbff 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -15,9 +15,9 @@ jobs: uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: '1.20.0' + go-version: stable - name: Check for new walletd tag in SiaFoundation/web id: check-tag From 3aae04927d41b1fc34eae76928fb5ebac4304c4f Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Thu, 22 Feb 2024 14:04:07 -0500 Subject: [PATCH 097/630] update version --- .github/workflows/ui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index ba2fbff..dcb530a 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: stable + go-version: '1.21' - name: Check for new walletd tag in SiaFoundation/web id: check-tag From fba370bb7d2671eefed0cd2b0475120286e96c81 Mon Sep 17 00:00:00 2001 From: alexfreska Date: Fri, 23 Feb 2024 00:15:33 +0000 Subject: [PATCH 098/630] ui: v0.17.0 --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 084ce0b..6aa530d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.16.0 + go.sia.tech/web/walletd v0.17.0 go.uber.org/zap v1.26.0 golang.org/x/term v0.6.0 lukechampine.com/flagg v1.1.1 diff --git a/go.sum b/go.sum index f29db2e..4ac7bb1 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,6 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc h1:oUCCTOatQIwYkJ2FUWRvJtgU+i/BwlzmzCxoSvmmJVQ= -go.sia.tech/core v0.2.1-0.20240130145801-8067f34b2ecc/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 h1:G2l6fRzAdNZy2z7+FhoG2y8ARtFpR6PkXXTB5tkdfZ8= @@ -24,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.16.0 h1:tCERgjsz4orokM94kt7PH2tNweHdOwK5aoPsCXes5HM= -go.sia.tech/web/walletd v0.16.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.17.0 h1:8k/m1L50LIylw1HYLlTuc3e4bYlx//qZ8xG4C/YNeA0= +go.sia.tech/web/walletd v0.17.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 4c11e4ddf7cc50d872c8d650313c82e954d5f718 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:20:52 -0800 Subject: [PATCH 099/630] sqlite, wallet: add fixed structure to wallets and addresses --- persist/sqlite/addresses.go | 51 ++++++ persist/sqlite/consensus_test.go | 43 ++--- persist/sqlite/init.sql | 18 +- persist/sqlite/wallet.go | 271 +++++++++++++++++++------------ wallet/manager.go | 77 +++++---- wallet/seed.go | 12 +- wallet/wallet.go | 22 +++ 7 files changed, 327 insertions(+), 167 deletions(-) create mode 100644 persist/sqlite/addresses.go diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go new file mode 100644 index 0000000..9a628ac --- /dev/null +++ b/persist/sqlite/addresses.go @@ -0,0 +1,51 @@ +package sqlite + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// AddressBalance returns the balance of a single address. +func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` + return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) + }) + return +} + +// AddressEvents returns the events of a single address. +func (s *Store) AddressEvents(address types.Address, limit, offset int) (events []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + FROM events ev + INNER JOIN chain_indices ci ON (ev.index_id = ci.id) + INNER JOIN event_addresses ea ON (ev.id = ea.event_id) + INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) + WHERE sa.sia_address = $1 + ORDER BY ev.maturity_height DESC, ev.id DESC + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + event, _, err := scanEvent(rows) + if err != nil { + return fmt.Errorf("failed to scan event: %w", err) + } + + events = append(events, event) + } + if err = rows.Err(); err != nil { + return err + } + return nil + }) + return +} diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 1d064ba..8462428 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -106,9 +106,10 @@ func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - if err := db.AddWallet("test", nil); err != nil { + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { t.Fatal(err) - } else if err := db.AddAddress("test", addr, nil); err != nil { + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -128,7 +129,7 @@ func TestReorg(t *testing.T) { } // check that a payout event was recorded - events, err := db.WalletEvents("test", 0, 100) + events, err := db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -138,7 +139,7 @@ func TestReorg(t *testing.T) { } // check that the utxo was created - utxos, err := db.UnspentSiacoinOutputs("test") + utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -170,7 +171,7 @@ func TestReorg(t *testing.T) { } // check that the payout event was reverted - events, err = db.WalletEvents("test", 0, 100) + events, err = db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 0 { @@ -178,7 +179,7 @@ func TestReorg(t *testing.T) { } // check that the utxo was removed - utxos, err = db.UnspentSiacoinOutputs("test") + utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -201,7 +202,7 @@ func TestReorg(t *testing.T) { } // check that a payout event was recorded - events, err = db.WalletEvents("test", 0, 100) + events, err = db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -211,7 +212,7 @@ func TestReorg(t *testing.T) { } // check that the utxo was created - utxos, err = db.UnspentSiacoinOutputs("test") + utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -266,7 +267,7 @@ func TestReorg(t *testing.T) { } // check that only the single utxo still exists - utxos, err = db.UnspentSiacoinOutputs("test") + utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -310,9 +311,10 @@ func TestEphemeralBalance(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - if err := db.AddWallet("test", nil); err != nil { + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { t.Fatal(err) - } else if err := db.AddAddress("test", addr, nil); err != nil { + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -334,7 +336,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that a payout event was recorded - events, err := db.WalletEvents("test", 0, 100) + events, err := db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -353,7 +355,7 @@ func TestEphemeralBalance(t *testing.T) { } // create a transaction that spends the matured payout - utxos, err := db.UnspentSiacoinOutputs("test") + utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -424,7 +426,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that both transactions were added - events, err = db.WalletEvents("test", 0, 100) + events, err = db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 3 { // 1 payout, 2 transactions @@ -462,7 +464,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that only the payout event remains - events, err = db.WalletEvents("test", 0, 100) + events, err = db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -504,9 +506,10 @@ func TestV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - if err := db.AddWallet("test", nil); err != nil { + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { t.Fatal(err) - } else if err := db.AddAddress("test", addr, nil); err != nil { + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -525,7 +528,7 @@ func TestV2(t *testing.T) { } // check that a payout event was recorded - events, err := db.WalletEvents("test", 0, 100) + events, err := db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -543,7 +546,7 @@ func TestV2(t *testing.T) { } // create a v2 transaction that spends the matured payout - utxos, err := db.UnspentSiacoinOutputs("test") + utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } @@ -577,7 +580,7 @@ func TestV2(t *testing.T) { } // check that a transaction event was recorded - events, err = db.WalletEvents("test", 0, 100) + events, err = db.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 2 { diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index b7c1767..7d12cc0 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -34,16 +34,23 @@ CREATE TABLE siafund_elements ( CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); CREATE TABLE wallets ( - id TEXT PRIMARY KEY NOT NULL, - extra_data BLOB NOT NULL + id INTEGER PRIMARY KEY, + friendly_name TEXT NOT NULL, + description TEXT NOT NULL, + date_created INTEGER NOT NULL, + last_updated INTEGER NOT NULL, + extra_data BLOB ); CREATE TABLE wallet_addresses ( - wallet_id TEXT NOT NULL REFERENCES wallets (id), + wallet_id INTEGER NOT NULL REFERENCES wallets (id), address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - extra_data BLOB NOT NULL, + description TEXT NOT NULL, + spend_policy BLOB, + extra_data BLOB, UNIQUE (wallet_id, address_id) ); +CREATE INDEX wallet_addresses_wallet_id ON wallet_addresses (wallet_id); CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); CREATE TABLE events ( @@ -53,9 +60,10 @@ CREATE TABLE events ( maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, event_type TEXT NOT NULL, - event_data TEXT NOT NULL + event_data BLOB NOT NULL ); + CREATE TABLE event_addresses ( event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index b2136e9..ad7801a 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" @@ -19,7 +20,47 @@ RETURNING id` return } -func getWalletEvents(tx *txn, walletID string, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { +func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { + var eventType string + var eventBuf []byte + + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &eventType, &eventBuf) + if err != nil { + return + } + + switch eventType { + case wallet.EventTypeTransaction: + var tx wallet.EventTransaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + ev.Data = &tx + case wallet.EventTypeContractPayout: + var m wallet.EventContractPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) + } + ev.Data = &m + case wallet.EventTypeMinerPayout: + var m wallet.EventMinerPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal payout event: %w", err) + } + ev.Data = &m + case wallet.EventTypeFoundationSubsidy: + var m wallet.EventFoundationSubsidy + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) + } + ev.Data = &m + default: + return wallet.Event{}, 0, fmt.Errorf("unknown event type: %s", eventType) + } + return +} + +func getWalletEvents(tx *txn, walletID int64, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INNER JOIN chain_indices ci ON (ev.index_id = ci.id) @@ -34,52 +75,21 @@ func getWalletEvents(tx *txn, walletID string, offset, limit int) (events []wall defer rows.Close() for rows.Next() { - var eventID int64 - var event wallet.Event - var eventType string - var eventBuf []byte - - err := rows.Scan(&eventID, decode(&event.ID), &event.MaturityHeight, decode(&event.Timestamp), &event.Index.Height, decode(&event.Index.ID), &eventType, &eventBuf) + event, eventID, err := scanEvent(rows) if err != nil { return nil, nil, fmt.Errorf("failed to scan event: %w", err) } - switch eventType { - case wallet.EventTypeTransaction: - var tx wallet.EventTransaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - event.Data = &tx - case wallet.EventTypeContractPayout: - var m wallet.EventContractPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - event.Data = &m - case wallet.EventTypeMinerPayout: - var m wallet.EventMinerPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal payout event: %w", err) - } - event.Data = &m - case wallet.EventTypeFoundationSubsidy: - var m wallet.EventFoundationSubsidy - if err = json.Unmarshal(eventBuf, &m); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) - } - event.Data = &m - default: - return nil, nil, fmt.Errorf("unknown event type: %s", eventType) - } - events = append(events, event) eventIDs = append(eventIDs, eventID) } + if err = rows.Err(); err != nil { + return nil, nil, err + } return } -func (s *Store) getWalletEventRelevantAddresses(tx *txn, walletID string, eventIDs []int64) (map[int64][]types.Address, error) { +func (s *Store) getWalletEventRelevantAddresses(tx *txn, walletID int64, eventIDs []int64) (map[int64][]types.Address, error) { query := `SELECT ea.event_id, sa.sia_address FROM event_addresses ea INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) @@ -104,7 +114,7 @@ WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN ( } // WalletEvents returns the events relevant to a wallet, sorted by height descending. -func (s *Store) WalletEvents(walletID string, offset, limit int) (events []wallet.Event, err error) { +func (s *Store) WalletEvents(walletID int64, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { var dbIDs []int64 events, dbIDs, err = getWalletEvents(tx, walletID, offset, limit) @@ -126,35 +136,48 @@ func (s *Store) WalletEvents(walletID string, offset, limit int) (events []walle } // AddWallet adds a wallet to the database. -func (s *Store) AddWallet(name string, info json.RawMessage) error { - if info == nil { - info = json.RawMessage("{}") - } - return s.transaction(func(tx *txn) error { - const query = `INSERT INTO wallets (id, extra_data) VALUES ($1, $2)` +func (s *Store) AddWallet(w wallet.Wallet) (wallet.Wallet, error) { + w.DateCreated = time.Now() + w.LastUpdated = time.Now() - _, err := tx.Exec(query, name, info) - if err != nil { - return fmt.Errorf("failed to insert wallet: %w", err) + err := s.transaction(func(tx *txn) error { + const query = `INSERT INTO wallets (friendly_name, description, date_created, last_updated, extra_data) VALUES ($1, $2, $3, $4, $5) RETURNING id` + return tx.QueryRow(query, w.Name, w.Description, encode(w.DateCreated), encode(w.LastUpdated), w.Metadata).Scan(&w.ID) + }) + return w, err +} + +func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { + w.LastUpdated = time.Now() + err := s.transaction(func(tx *txn) error { + var dummyID int64 + const query = `UPDATE wallets SET friendly_name=$1, description=$2, last_updated=$3, extra_data=$4 WHERE id=$5 RETURNING id, date_created, last_updated` + err := tx.QueryRow(query, w.Name, w.Description, encode(w.LastUpdated), w.Metadata, w.ID).Scan(&dummyID, decode(&w.DateCreated), decode(&w.LastUpdated)) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound } - return nil + return err }) + return w, err } // DeleteWallet deletes a wallet from the database. This does not stop tracking // addresses that were previously associated with the wallet. -func (s *Store) DeleteWallet(name string) error { +func (s *Store) DeleteWallet(walletID int64) error { return s.transaction(func(tx *txn) error { - _, err := tx.Exec(`DELETE FROM wallets WHERE id=$1`, name) + var dummyID int64 + err := tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, walletID).Scan(&dummyID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } return err }) } // Wallets returns a map of wallet names to wallet extra data. -func (s *Store) Wallets() (map[string]json.RawMessage, error) { - wallets := make(map[string]json.RawMessage) - err := s.transaction(func(tx *txn) error { - const query = `SELECT id, extra_data FROM wallets` +func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT id, friendly_name, description, date_created, last_updated, extra_data FROM wallets` rows, err := tx.Query(query) if err != nil { @@ -163,48 +186,61 @@ func (s *Store) Wallets() (map[string]json.RawMessage, error) { defer rows.Close() for rows.Next() { - var friendlyName string - var extraData json.RawMessage - if err := rows.Scan(&friendlyName, &extraData); err != nil { + var w wallet.Wallet + if err := rows.Scan(&w.ID, &w.Name, &w.Description, decode(&w.DateCreated), decode(&w.LastUpdated), &w.Metadata); err != nil { return fmt.Errorf("failed to scan wallet: %w", err) } - wallets[friendlyName] = extraData + wallets = append(wallets, w) } - return nil + return rows.Err() }) - return wallets, err + return } -// AddAddress adds an address to a wallet. -func (s *Store) AddAddress(walletID string, address types.Address, info json.RawMessage) error { - if info == nil { - info = json.RawMessage("{}") - } +// AddWalletAddress adds an address to a wallet. +func (s *Store) AddWalletAddress(walletID int64, addr wallet.Address) error { return s.transaction(func(tx *txn) error { - addressID, err := insertAddress(tx, address) + if err := walletExists(tx, walletID); err != nil { + return err + } + + addressID, err := insertAddress(tx, addr.Address) if err != nil { return fmt.Errorf("failed to insert address: %w", err) } - _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, extra_data, address_id) VALUES ($1, $2, $3)`, walletID, info, addressID) + + var encodedPolicy any + if addr.SpendPolicy != nil { + encodedPolicy = encode(*addr.SpendPolicy) + } + + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5)`, walletID, addressID, addr.Description, encodedPolicy, addr.Metadata) return err }) } -// RemoveAddress removes an address from a wallet. This does not stop tracking +// RemoveWalletAddress removes an address from a wallet. This does not stop tracking // the address. -func (s *Store) RemoveAddress(walletID string, address types.Address) error { +func (s *Store) RemoveWalletAddress(walletID int64, address types.Address) error { return s.transaction(func(tx *txn) error { - const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2)` - _, err := tx.Exec(query, walletID, encode(address)) + const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING id` + var dummyID int64 + err := tx.QueryRow(query, walletID, encode(address)).Scan(&dummyID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } return err }) } -// Addresses returns a map of addresses to their extra data for a wallet. -func (s *Store) Addresses(walletID string) (map[types.Address]json.RawMessage, error) { - addresses := make(map[types.Address]json.RawMessage) - err := s.transaction(func(tx *txn) error { - const query = `SELECT sa.sia_address, wa.extra_data +// WalletAddresses returns a slice of addresses registered to the wallet. +func (s *Store) WalletAddresses(walletID int64) (addresses []wallet.Address, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, walletID); err != nil { + return err + } + + const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data FROM wallet_addresses wa INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` @@ -216,27 +252,48 @@ WHERE wa.wallet_id=$1` defer rows.Close() for rows.Next() { - var address types.Address - var extraData json.RawMessage - if err := rows.Scan(decode(&address), &extraData); err != nil { + var address wallet.Address + + var decodedPolicy any + if err := rows.Scan(decode(&address.Address), &address.Description, &decodedPolicy, &address.Metadata); err != nil { return fmt.Errorf("failed to scan address: %w", err) } - addresses[address] = extraData + + if decodedPolicy != nil { + switch v := decodedPolicy.(type) { + case []byte: + dec := types.NewBufDecoder(v) + address.SpendPolicy = new(types.SpendPolicy) + address.SpendPolicy.DecodeFrom(dec) + if err := dec.Err(); err != nil { + return fmt.Errorf("failed to decode spend policy: %w", err) + } + default: + return fmt.Errorf("unexpected spend policy type: %T", decodedPolicy) + } + } + + addresses = append(addresses, address) } return nil }) - return addresses, err + return } -// UnspentSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.SiacoinElement, err error) { +// WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. +func (s *Store) WalletSiacoinOutputs(walletID int64, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, walletID); err != nil { + return err + } + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)` + WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) + LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, walletID) + rows, err := tx.Query(query, walletID, limit, offset) if err != nil { return err } @@ -256,15 +313,20 @@ func (s *Store) UnspentSiacoinOutputs(walletID string) (siacoins []types.Siacoin return } -// UnspentSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.SiafundElement, err error) { +// WalletSiafundOutputs returns the unspent siafund outputs for a wallet. +func (s *Store) WalletSiafundOutputs(walletID int64, offset, limit int) (siafunds []types.SiafundElement, err error) { err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, walletID); err != nil { + return err + } + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)` + WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) + LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, walletID) + rows, err := tx.Query(query, walletID, limit, offset) if err != nil { return err } @@ -284,8 +346,12 @@ func (s *Store) UnspentSiafundOutputs(walletID string) (siafunds []types.Siafund } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err error) { +func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, walletID); err != nil { + return err + } + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` @@ -312,18 +378,13 @@ func (s *Store) WalletBalance(walletID string) (balance wallet.Balance, err erro return } -// AddressBalance returns the balance of a single address. -func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { - err = s.transaction(func(tx *txn) error { - const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) - }) - return -} - // Annotate annotates a list of transactions using the wallet's addresses. -func (s *Store) Annotate(walletID string, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { +func (s *Store) Annotate(walletID int64, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, walletID); err != nil { + return err + } + const query = `SELECT sa.id FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` @@ -358,3 +419,13 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` }) return } + +func walletExists(tx *txn, walletID int64) error { + const query = `SELECT id FROM wallets WHERE id=$1` + var dummyID int64 + err := tx.QueryRow(query, walletID).Scan(&dummyID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } + return err +} diff --git a/wallet/manager.go b/wallet/manager.go index 8ecb5b3..0c496c5 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -1,7 +1,6 @@ package wallet import ( - "encoding/json" "errors" "fmt" "sync" @@ -25,20 +24,20 @@ type ( Store interface { chain.Subscriber - WalletEvents(name string, offset, limit int) ([]Event, error) - AddWallet(name string, info json.RawMessage) error - DeleteWallet(name string) error - Wallets() (map[string]json.RawMessage, error) + WalletEvents(id int64, offset, limit int) ([]Event, error) + AddWallet(Wallet) (Wallet, error) + UpdateWallet(Wallet) (Wallet, error) + DeleteWallet(id int64) error + WalletBalance(id int64) (Balance, error) + WalletSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) + WalletSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) + WalletAddresses(walletID int64) ([]Address, error) + Wallets() ([]Wallet, error) - AddAddress(walletID string, address types.Address, info json.RawMessage) error - RemoveAddress(walletID string, address types.Address) error - Addresses(walletID string) (map[types.Address]json.RawMessage, error) - UnspentSiacoinOutputs(walletID string) ([]types.SiacoinElement, error) - UnspentSiafundOutputs(walletID string) ([]types.SiafundElement, error) - Annotate(walletID string, txns []types.Transaction) ([]PoolTransaction, error) - WalletBalance(walletID string) (Balance, error) + AddWalletAddress(walletID int64, address Address) error + RemoveWalletAddress(walletID int64, address types.Address) error - AddressBalance(address types.Address) (Balance, error) + Annotate(walletID int64, txns []types.Transaction) ([]PoolTransaction, error) LastCommittedIndex() (types.ChainIndex, error) } @@ -55,65 +54,65 @@ type ( ) // AddWallet adds the given wallet. -func (m *Manager) AddWallet(name string, info json.RawMessage) error { - return m.store.AddWallet(name, info) +func (m *Manager) AddWallet(w Wallet) (Wallet, error) { + return m.store.AddWallet(w) +} + +func (m *Manager) UpdateWallet(w Wallet) (Wallet, error) { + return m.store.UpdateWallet(w) } // DeleteWallet deletes the given wallet. -func (m *Manager) DeleteWallet(name string) error { - return m.store.DeleteWallet(name) +func (m *Manager) DeleteWallet(id int64) error { + return m.store.DeleteWallet(id) } // Wallets returns the wallets of the wallet manager. -func (m *Manager) Wallets() (map[string]json.RawMessage, error) { +func (m *Manager) Wallets() ([]Wallet, error) { return m.store.Wallets() } // AddAddress adds the given address to the given wallet. -func (m *Manager) AddAddress(name string, addr types.Address, info json.RawMessage) error { - return m.store.AddAddress(name, addr, info) +func (m *Manager) AddAddress(walletID int64, addr Address) error { + return m.store.AddWalletAddress(walletID, addr) } // RemoveAddress removes the given address from the given wallet. -func (m *Manager) RemoveAddress(name string, addr types.Address) error { - return m.store.RemoveAddress(name, addr) +func (m *Manager) RemoveAddress(walletID int64, addr types.Address) error { + return m.store.RemoveWalletAddress(walletID, addr) } // Addresses returns the addresses of the given wallet. -func (m *Manager) Addresses(name string) (map[types.Address]json.RawMessage, error) { - return m.store.Addresses(name) +func (m *Manager) Addresses(walletID int64) ([]Address, error) { + return m.store.WalletAddresses(walletID) } // Events returns the events of the given wallet. -func (m *Manager) Events(name string, offset, limit int) ([]Event, error) { - return m.store.WalletEvents(name, offset, limit) +func (m *Manager) Events(walletID int64, offset, limit int) ([]Event, error) { + return m.store.WalletEvents(walletID, offset, limit) } -// UnspentSiacoinOutputs returns the unspent siacoin outputs of the given wallet -func (m *Manager) UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) { - return m.store.UnspentSiacoinOutputs(name) +// UnspentSiacoinOutputs returns a paginated list of unspent siacoin outputs of +// the given wallet and the total number of unspent siacoin outputs. +func (m *Manager) UnspentSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) { + return m.store.WalletSiacoinOutputs(walletID, offset, limit) } // UnspentSiafundOutputs returns the unspent siafund outputs of the given wallet -func (m *Manager) UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) { - return m.store.UnspentSiafundOutputs(name) +func (m *Manager) UnspentSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) { + return m.store.WalletSiafundOutputs(walletID, offset, limit) } // Annotate annotates the given transactions with the wallet they belong to. -func (m *Manager) Annotate(name string, pool []types.Transaction) ([]PoolTransaction, error) { - return m.store.Annotate(name, pool) +func (m *Manager) Annotate(walletID int64, pool []types.Transaction) ([]PoolTransaction, error) { + return m.store.Annotate(walletID, pool) } // WalletBalance returns the balance of the given wallet. -func (m *Manager) WalletBalance(walletID string) (Balance, error) { +func (m *Manager) WalletBalance(walletID int64) (Balance, error) { return m.store.WalletBalance(walletID) } -// AddressBalance returns the balance of the given address. -func (m *Manager) AddressBalance(address types.Address) (Balance, error) { - return m.store.AddressBalance(address) -} - // Reserve reserves the given ids for the given duration. func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { m.mu.Lock() diff --git a/wallet/seed.go b/wallet/seed.go index 09288bb..ecc4a65 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -70,13 +70,19 @@ func (sav *SeedAddressVault) OwnsAddress(addr types.Address) bool { // NewAddress returns a new address derived from the seed, along with // descriptive metadata. -func (sav *SeedAddressVault) NewAddress(desc string) (types.Address, json.RawMessage) { +func (sav *SeedAddressVault) NewAddress(desc string) Address { sav.mu.Lock() defer sav.mu.Unlock() index := uint64(len(sav.addrs)) - sav.lookahead + 1 sav.gen(index + sav.lookahead) - addr := types.StandardAddress(sav.seed.PublicKey(index)) - return addr, json.RawMessage(fmt.Sprintf(`{"desc":"%s","keyIndex":%d}`, desc, index)) + policy := types.PolicyPublicKey(sav.seed.PublicKey(index)) + addr := policy.Address() + return Address{ + Address: addr, + Description: desc, + SpendPolicy: &policy, + Metadata: json.RawMessage(fmt.Sprintf(`{"keyIndex":%d}`, index)), + } } // SignTransaction signs the specified transaction using keys derived from the diff --git a/wallet/wallet.go b/wallet/wallet.go index 510b239..43e1b21 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -2,6 +2,7 @@ package wallet import ( "encoding/json" + "errors" "fmt" "time" @@ -25,8 +26,29 @@ type ( ImmatureSiacoins types.Currency `json:"immatureSiacoins"` Siafunds uint64 `json:"siafunds"` } + + // A Wallet is a collection of addresses and metadata. + Wallet struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DateCreated time.Time `json:"dateCreated"` + LastUpdated time.Time `json:"lastUpdated"` + Metadata json.RawMessage `json:"metadata"` + } + + // A Address is an address associated with a wallet. + Address struct { + Address types.Address `json:"address"` + Description string `json:"description"` + SpendPolicy *types.SpendPolicy `json:"spendPolicy,omitempty"` + Metadata json.RawMessage `json:"metadata"` + } ) +// ErrNotFound is returned when a requested wallet or address is not found. +var ErrNotFound = errors.New("not found") + // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. From ef976ba40d8852c2d38f6a74442d119d8e5fffb9 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:23:38 -0800 Subject: [PATCH 100/630] all: update jape dependency --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6aa530d..de2866e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.21 go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 - go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.17.0 + go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 + go.sia.tech/web/walletd v0.16.0 go.uber.org/zap v1.26.0 golang.org/x/term v0.6.0 lukechampine.com/flagg v1.1.1 diff --git a/go.sum b/go.sum index 4ac7bb1..6bff896 100644 --- a/go.sum +++ b/go.sum @@ -16,14 +16,14 @@ go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 h1:G2l6fRzAdNZy2z7+FhoG2y8ARtFpR6PkXXTB5tkdfZ8= go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7/go.mod h1:3Mb206QDd3NtRiaHZ2kN87/HKXhcBF6lHVatS7PkViY= -go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= -go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= +go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= +go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.17.0 h1:8k/m1L50LIylw1HYLlTuc3e4bYlx//qZ8xG4C/YNeA0= -go.sia.tech/web/walletd v0.17.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.16.0 h1:tCERgjsz4orokM94kt7PH2tNweHdOwK5aoPsCXes5HM= +go.sia.tech/web/walletd v0.16.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 0dc1554cb91ab9a455badcb9c6b1b1b586fed2d2 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:25:11 -0800 Subject: [PATCH 101/630] cmd: remove testnet commands --- cmd/walletd/main.go | 64 ++-------- cmd/walletd/testnet.go | 260 +---------------------------------------- 2 files changed, 16 insertions(+), 308 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 2b69192..bc19658 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -10,6 +10,7 @@ import ( "strings" "go.sia.tech/core/types" + "go.sia.tech/walletd/api" "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -122,6 +123,9 @@ func main() { var gatewayAddr, apiAddr, dir, network, seed string var upnp, v2 bool + var minerAddrStr string + var minerBlocks int + rootCmd := flagg.Root rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) rootCmd.StringVar(&gatewayAddr, "addr", ":9981", "p2p address to listen on") @@ -133,6 +137,8 @@ func main() { versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) mineCmd := flagg.New("mine", mineUsage) + mineCmd.IntVar(&minerBlocks, "n", -1, "mine this many blocks. If negative, mine indefinitely") + mineCmd.StringVar(&minerAddrStr, "addr", "", "address to send block rewards to (required)") balanceCmd := flagg.New("balance", balanceUsage) sendCmd := flagg.New("send", sendUsage) sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") @@ -228,61 +234,13 @@ func main() { cmd.Usage() return } - seed := loadTestnetSeed(seed) - c := initTestnetClient(apiAddr, network, seed) - runTestnetMiner(c, seed) - case balanceCmd: - if len(cmd.Args()) != 0 { - cmd.Usage() - return - } - seed := loadTestnetSeed(seed) - c := initTestnetClient(apiAddr, network, seed) - b, err := c.Wallet("primary").Balance() - check("Couldn't get balance:", err) - out := fmt.Sprint(b.Siacoins) - if !b.ImmatureSiacoins.IsZero() { - out += fmt.Sprintf(" + %v immature", b.ImmatureSiacoins) - } - poolGained, poolLost := testnetTxpoolBalance(c, seed) - if !poolGained.IsZero() || !poolLost.IsZero() { - if poolGained.Cmp(poolLost) >= 0 { - out += fmt.Sprintf(" + %v unconfirmed", poolGained.Sub(poolLost)) - } else { - out += fmt.Sprintf(" - %v unconfirmed", poolLost.Sub(poolGained)) - } - } - fmt.Println(out) - case sendCmd: - if len(cmd.Args()) != 2 { - cmd.Usage() - return - } - seed := loadTestnetSeed(seed) - c := initTestnetClient(apiAddr, network, seed) - amount, err := types.ParseCurrency(cmd.Arg(0)) - check("Couldn't parse amount:", err) - dest, err := types.ParseAddress(cmd.Arg(1)) - check("Couldn't parse recipient address:", err) - sendTestnet(c, seed, amount, dest, v2) - - case txnsCmd: - if len(cmd.Args()) != 0 { - cmd.Usage() - return + minerAddr, err := types.ParseAddress(minerAddrStr) + if err != nil { + log.Fatal(err) } - seed := loadTestnetSeed(seed) - c := initTestnetClient(apiAddr, network, seed) - printTestnetEvents(c, seed) - case txpoolCmd: - if len(cmd.Args()) != 0 { - cmd.Usage() - return - } - seed := loadTestnetSeed(seed) - c := initTestnetClient(apiAddr, network, seed) - printTestnetTxpool(c, seed) + c := api.NewClient("http://"+apiAddr+"/api", getAPIPassword()) + runTestnetMiner(c, minerAddr, minerBlocks) } } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 9a66bd0..64ff63c 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -2,19 +2,14 @@ package main import ( "encoding/binary" - "encoding/hex" "fmt" "log" "math/big" - "os" - "reflect" "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/walletd/api" - "go.sia.tech/walletd/wallet" - "golang.org/x/term" "lukechampine.com/frand" ) @@ -69,62 +64,6 @@ func TestnetAnagami() (*consensus.Network, types.Block) { return n, b } -func loadTestnetSeed(s string) wallet.Seed { - if s == "" { - fmt.Println("Seed not supplied via -seed flag, falling back to manual entry.") - fmt.Print("Seed: ") - pw, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - check("Could not read API password:", err) - if err != nil { - log.Fatal(err) - } - s = string(pw) - } - b, err := hex.DecodeString(s) - if err != nil || len(b) != 8 { - log.Fatal("Seed must be 16 hex characters") - } - var entropy [32]byte - copy(entropy[:], b) - return wallet.NewSeedFromEntropy(&entropy) -} - -func initTestnetClient(addr string, network string, seed wallet.Seed) *api.Client { - if network == "mainnet" { - log.Fatal("Testnet actions cannot be used on mainnet") - } - c := api.NewClient("http://"+addr+"/api", getAPIPassword()) - cs, err := c.ConsensusTipState() - check("Couldn't connect to API:", err) - if cs.Network.Name != network { - log.Fatalf("Testnet %q was specified, but walletd is running %v", network, cs.Network.Name) - } - ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - wc := c.Wallet("primary") - if addrs, err := wc.Addresses(); err == nil && len(addrs) > 0 { - if _, ok := addrs[ourAddr]; !ok { - log.Fatal("Wallet already initialized with a different testnet address") - } - } - if ws, _ := c.Wallets(); len(ws) == 0 { - fmt.Print("Initializing testnet wallet...") - if err := c.AddWallet("primary", nil); err != nil { - fmt.Println() - log.Fatal(err) - } else if err := wc.AddAddress(ourAddr, nil); err != nil { - fmt.Println() - log.Fatal(err) - } else if err := c.Resubscribe(0); err != nil { - fmt.Println() - log.Fatal(err) - } - fmt.Println("done.") - } - - return c -} - func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { buf := make([]byte, 32+8+8+32) binary.LittleEndian.PutUint64(buf[32:], b.Nonce) @@ -150,8 +89,7 @@ func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { return hashes, true } -func runTestnetMiner(c *api.Client, seed wallet.Seed) { - minerAddr := types.StandardUnlockHash(seed.PublicKey(0)) +func runTestnetMiner(c *api.Client, minerAddr types.Address, n int) { log.Println("Started mining into", minerAddr) start := time.Now() @@ -159,7 +97,10 @@ func runTestnetMiner(c *api.Client, seed wallet.Seed) { var blocks uint64 var last types.ChainIndex outer: - for { + for i := 0; ; i++ { + if n <= 0 && i >= n { + return + } elapsed := time.Since(start) cs, err := c.ConsensusTipState() check("Couldn't get consensus tip state:", err) @@ -216,194 +157,3 @@ outer: } } } - -func sendTestnet(c *api.Client, seed wallet.Seed, amount types.Currency, dest types.Address, v2 bool) { - ourKey := seed.PrivateKey(0) - ourUC := types.StandardUnlockConditions(seed.PublicKey(0)) - ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - - cs, err := c.ConsensusTipState() - check("Couldn't get consensus tip state:", err) - utxos, _, err := c.Wallet("primary").Outputs() - check("Couldn't get outputs:", err) - txns, v2txns, err := c.TxpoolTransactions() - if err != nil { - log.Fatal(err) - } - inPool := make(map[types.Hash256]bool) - for _, ptxn := range txns { - for _, in := range ptxn.SiacoinInputs { - inPool[types.Hash256(in.ParentID)] = true - } - } - for _, ptxn := range v2txns { - for _, in := range ptxn.SiacoinInputs { - inPool[in.Parent.ID] = true - } - } - - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) - var inputSum types.Currency - rem := utxos[:0] - for _, utxo := range utxos { - if inputSum.Cmp(amount) >= 0 { - break - } else if cs.Index.Height > utxo.MaturityHeight && !inPool[utxo.ID] { - rem = append(rem, utxo) - inputSum = inputSum.Add(utxo.SiacoinOutput.Value) - } - } - utxos = rem - if inputSum.Cmp(amount) < 0 { - log.Fatal("Insufficient balance") - } - outputs := []types.SiacoinOutput{ - {Address: dest, Value: amount}, - } - minerFee := inputSum.Sub(amount) - if maxFee := types.Siacoins(1); minerFee.Cmp(maxFee) > 0 { - minerFee = maxFee - } - if change := inputSum.Sub(amount.Add(minerFee)); !change.IsZero() { - outputs = append(outputs, types.SiacoinOutput{ - Address: ourAddr, - Value: change, - }) - } - - if v2 { - txn := types.V2Transaction{ - SiacoinInputs: make([]types.V2SiacoinInput, len(utxos)), - SiacoinOutputs: outputs, - MinerFee: minerFee, - } - for i, sce := range utxos { - txn.SiacoinInputs[i].Parent = sce - txn.SiacoinInputs[i].SatisfiedPolicy.Policy = types.SpendPolicy{ - Type: types.PolicyTypeUnlockConditions(ourUC), - } - } - sigHash := cs.InputSigHash(txn) - for i := range utxos { - txn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{ourKey.SignHash(sigHash)} - } - if err := c.TxpoolBroadcast(nil, []types.V2Transaction{txn}); err != nil { - log.Fatal(err) - } - log.Println("Broadcast", txn.ID(), "successfully") - } else { - txn := types.Transaction{ - SiacoinInputs: make([]types.SiacoinInput, len(utxos)), - SiacoinOutputs: outputs, - Signatures: make([]types.TransactionSignature, len(utxos)), - } - if !minerFee.IsZero() { - txn.MinerFees = append(txn.MinerFees, minerFee) - } - for i, sce := range utxos { - txn.SiacoinInputs[i] = types.SiacoinInput{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: ourUC, - } - } - cs, _ := c.ConsensusTipState() - for i, sce := range utxos { - txn.Signatures[i] = wallet.StandardTransactionSignature(sce.ID) - wallet.SignTransaction(cs, &txn, i, ourKey) - } - if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { - log.Fatal(err) - } - log.Println("Broadcast", txn.ID(), "successfully") - } -} - -func printTestnetEvents(c *api.Client, seed wallet.Seed) { - ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - events, err := c.Wallet("primary").Events(0, -1) - check("Couldn't get events:", err) - for i := range events { - e := events[len(events)-1-i] - switch t := e.Data.(type) { - case *wallet.EventTransaction: - if len(t.SiacoinInputs) == 0 || len(t.SiacoinOutputs) == 0 { - continue - } - sci := t.SiacoinInputs[0].SiacoinOutput - sco := t.SiacoinOutputs[0].SiacoinOutput - if sci.Address == ourAddr { - fmt.Printf("%14v (%v): Sent %v (+ %v fee) to %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, t.Fee, sco.Address) - } else { - fmt.Printf("%14v (%v): Received %v from %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, sci.Address) - } - case *wallet.EventMinerPayout: - sco := t.SiacoinOutput.SiacoinOutput - fmt.Printf("%14v (%v): Earned %v miner payout from block %v\n", e.Index, e.Timestamp.Format("Jan _2 @ 15:04:05"), sco.Value, e.Index) - } - } -} - -func testnetTxpoolBalance(c *api.Client, seed wallet.Seed) (gained, lost types.Currency) { - ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - txns, v2txns, err := c.TxpoolTransactions() - check("Couldn't get txpool transactions:", err) - for _, txn := range txns { - if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { - continue - } - sco := txn.SiacoinOutputs[0] - if txn.SiacoinInputs[0].UnlockConditions.UnlockHash() == ourAddr { - lost = lost.Add(sco.Value).Add(txn.TotalFees()) - } else if sco.Address == ourAddr { - gained = gained.Add(sco.Value) - } - } - for _, txn := range v2txns { - if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { - continue - } - sco := txn.SiacoinOutputs[0] - if txn.SiacoinInputs[0].Parent.SiacoinOutput.Address == ourAddr { - lost = lost.Add(sco.Value).Add(txn.MinerFee) - } else if sco.Address == ourAddr { - gained = gained.Add(sco.Value) - } - } - return -} - -func printTestnetTxpool(c *api.Client, seed wallet.Seed) { - ourAddr := types.StandardUnlockHash(seed.PublicKey(0)) - txns, v2txns, err := c.TxpoolTransactions() - check("Couldn't get txpool transactions:", err) - if len(txns) == 0 && len(v2txns) == 0 { - fmt.Println("No transactions in txpool.") - return - } - for _, txn := range txns { - if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { - continue - } - id := txn.ID() - sci := txn.SiacoinInputs[0] - sco := txn.SiacoinOutputs[0] - if sci.UnlockConditions.UnlockHash() == ourAddr { - fmt.Printf("%x (v1): Sending %v (+ %v fee) to %v\n", id[:4], sco.Value, txn.TotalFees(), sco.Address) - } else if sco.Address == ourAddr { - fmt.Printf("%x (v1): Receiving %v from %v\n", id[:4], sco.Value, sci.UnlockConditions.UnlockHash()) - } - } - for _, txn := range v2txns { - if len(txn.SiacoinInputs) == 0 || len(txn.SiacoinOutputs) == 0 { - continue - } - id := txn.ID() - sci := txn.SiacoinInputs[0].Parent.SiacoinOutput - sco := txn.SiacoinOutputs[0] - if sci.Address == ourAddr { - fmt.Printf("%x (v2): Sending %v (+ %v fee) to %v\n", id[:4], sco.Value, txn.MinerFee, sco.Address) - } else if sco.Address == ourAddr { - fmt.Printf("%x (v2): Receiving %v from %v\n", id[:4], sco.Value, sci.Address) - } - } -} From db830f5fd690a7ff261e01e319ba3bd3587f1980 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:25:41 -0800 Subject: [PATCH 102/630] api: use new wallet structure, split sc and sf utxo requests, add pagination to sc and sf utxo requests --- api/api.go | 26 +++--- api/api_test.go | 68 ++++++++------- api/client.go | 60 ++++++++------ api/server.go | 215 +++++++++++++++++++++++++++++++----------------- 4 files changed, 226 insertions(+), 143 deletions(-) diff --git a/api/api.go b/api/api.go index 28c20d4..c7fbcde 100644 --- a/api/api.go +++ b/api/api.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "time" "go.sia.tech/core/types" @@ -31,36 +32,37 @@ type TxpoolTransactionsResponse struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } -// BalanceResponse is the response type for /wallets/:name/balance. +// BalanceResponse is the response type for /wallets/:id/balance. type BalanceResponse wallet.Balance -// WalletOutputsResponse is the response type for /wallets/:name/outputs. -type WalletOutputsResponse struct { - SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` - SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` -} - -// WalletReserveRequest is the request type for /wallets/:name/reserve. +// WalletReserveRequest is the request type for /wallets/:id/reserve. type WalletReserveRequest struct { SiacoinOutputs []types.SiacoinOutputID `json:"siacoinOutputs"` SiafundOutputs []types.SiafundOutputID `json:"siafundOutputs"` Duration time.Duration `json:"duration"` } -// WalletReleaseRequest is the request type for /wallets/:name/release. +// A WalletUpdateRequest is a request to update a wallet +type WalletUpdateRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Metadata json.RawMessage `json:"metadata"` +} + +// WalletReleaseRequest is the request type for /wallets/:id/release. type WalletReleaseRequest struct { SiacoinOutputs []types.SiacoinOutputID `json:"siacoinOutputs"` SiafundOutputs []types.SiafundOutputID `json:"siafundOutputs"` } -// WalletFundRequest is the request type for /wallets/:name/fund. +// WalletFundRequest is the request type for /wallets/:id/fund. type WalletFundRequest struct { Transaction types.Transaction `json:"transaction"` Amount types.Currency `json:"amount"` ChangeAddress types.Address `json:"changeAddress"` } -// WalletFundSFRequest is the request type for /wallets/:name/fundsf. +// WalletFundSFRequest is the request type for /wallets/:id/fundsf. type WalletFundSFRequest struct { Transaction types.Transaction `json:"transaction"` Amount uint64 `json:"amount"` @@ -68,7 +70,7 @@ type WalletFundSFRequest struct { ClaimAddress types.Address `json:"claimAddress"` } -// WalletFundResponse is the response type for /wallets/:name/fund. +// WalletFundResponse is the response type for /wallets/:id/fund. type WalletFundResponse struct { Transaction types.Transaction `json:"transaction"` ToSign []types.Hash256 `json:"toSign"` diff --git a/api/api_test.go b/api/api_test.go index 59c40fa..ba8dcda 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -80,10 +80,11 @@ func TestWallet(t *testing.T) { sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) defer shutdown() - if err := c.AddWallet("primary", nil); err != nil { + w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { t.Fatal(err) } - wc := c.Wallet("primary") + wc := c.Wallet(w.ID) if err := c.Resubscribe(0); err != nil { t.Fatal(err) } @@ -112,8 +113,8 @@ func TestWallet(t *testing.T) { } // create and add an address - addr, info := sav.NewAddress("primary") - if err := wc.AddAddress(addr, info); err != nil { + addr := sav.NewAddress("primary") + if err := wc.AddAddress(addr); err != nil { t.Fatal(err) } @@ -121,8 +122,10 @@ func TestWallet(t *testing.T) { addresses, err = wc.Addresses() if err != nil { t.Fatal(err) - } else if _, ok := addresses[addr]; !ok || len(addresses) != 1 { - t.Fatal("bad address list", addresses) + } else if len(addresses) != 1 { + t.Fatal("address list should have one address") + } else if addresses[0].Address != addr.Address { + t.Fatalf("address should be %v, got %v", addr, addresses[0]) } // send gift to wallet @@ -133,8 +136,8 @@ func TestWallet(t *testing.T) { UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), }}, SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr, Value: types.Siacoins(1).Div64(2)}, - {Address: addr, Value: types.Siacoins(1).Div64(2)}, + {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, + {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, }, Signatures: []types.TransactionSignature{{ ParentID: types.Hash256(giftSCOID), @@ -176,7 +179,7 @@ func TestWallet(t *testing.T) { t.Error("transaction should appear in history") } - outputs, _, err := wc.Outputs() + outputs, err := wc.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { @@ -188,7 +191,7 @@ func TestWallet(t *testing.T) { b = types.Block{ ParentID: cs.Index.ID, Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: addr, Value: cs.BlockReward()}}, + MinerPayouts: []types.SiacoinOutput{{Address: addr.Address, Value: cs.BlockReward()}}, } for b.ID().CmpWork(cs.ChildTarget) < 0 { b.Nonce += cs.NonceFactor() @@ -265,18 +268,20 @@ func TestV2(t *testing.T) { } c, shutdown := runServer(cm, nil, wm) defer shutdown() - if err := c.AddWallet("primary", nil); err != nil { + primaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { t.Fatal(err) } - primary := c.Wallet("primary") - if err := primary.AddAddress(primaryAddress, nil); err != nil { + primary := c.Wallet(primaryWallet.ID) + if err := primary.AddAddress(wallet.Address{Address: primaryAddress}); err != nil { t.Fatal(err) } - if err := c.AddWallet("secondary", nil); err != nil { + secondaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "secondary"}) + if err != nil { t.Fatal(err) } - secondary := c.Wallet("secondary") - if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + secondary := c.Wallet(secondaryWallet.ID) + if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { t.Fatal(err) } if err := c.Resubscribe(0); err != nil { @@ -324,12 +329,12 @@ func TestV2(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() + sces, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } if len(sces) == 0 { - sces, _, err = secondary.Outputs() + sces, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -370,12 +375,12 @@ func TestV2(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() + sces, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } if len(sces) == 0 { - sces, _, err = secondary.Outputs() + sces, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -487,11 +492,12 @@ func TestP2P(t *testing.T) { go s1.Run() c1, shutdown := runServer(cm1, s1, wm1) defer shutdown() - if err := c1.AddWallet("primary", nil); err != nil { + w1, err := c1.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { t.Fatal(err) } - primary := c1.Wallet("primary") - if err := primary.AddAddress(primaryAddress, nil); err != nil { + primary := c1.Wallet(w1.ID) + if err := primary.AddAddress(wallet.Address{Address: primaryAddress}); err != nil { t.Fatal(err) } if err := c1.Resubscribe(0); err != nil { @@ -526,11 +532,13 @@ func TestP2P(t *testing.T) { go s2.Run() c2, shutdown2 := runServer(cm2, s2, wm2) defer shutdown2() - if err := c2.AddWallet("secondary", nil); err != nil { + + w2, err := c2.AddWallet(api.WalletUpdateRequest{Name: "secondary"}) + if err != nil { t.Fatal(err) } - secondary := c2.Wallet("secondary") - if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + secondary := c2.Wallet(w2.ID) + if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { t.Fatal(err) } if err := c2.Resubscribe(0); err != nil { @@ -606,7 +614,7 @@ func TestP2P(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() + sces, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -614,7 +622,7 @@ func TestP2P(t *testing.T) { c = c2 key = secondaryPrivateKey dest = primaryAddress - sces, _, err = secondary.Outputs() + sces, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -660,7 +668,7 @@ func TestP2P(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() + sces, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -668,7 +676,7 @@ func TestP2P(t *testing.T) { c = c2 key = secondaryPrivateKey dest = primaryAddress - sces, _, err = secondary.Outputs() + sces, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } diff --git a/api/client.go b/api/client.go index 8729863..dcee7bb 100644 --- a/api/client.go +++ b/api/client.go @@ -88,21 +88,26 @@ func (c *Client) Wallets() (ws map[string]json.RawMessage, err error) { } // AddWallet adds a wallet to the set of tracked wallets. -func (c *Client) AddWallet(name string, info json.RawMessage) (err error) { - err = c.c.PUT(fmt.Sprintf("/wallets/%v", name), info) +func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) { + err = c.c.POST("/wallets", uw, &w) + return +} + +func (c *Client) UpdateWallet(id int64, uw WalletUpdateRequest) (w wallet.Wallet, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) return } // RemoveWallet deletes a wallet. If the wallet is currently subscribed, it will // be unsubscribed. -func (c *Client) RemoveWallet(name string) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v", name)) +func (c *Client) RemoveWallet(id int64) (err error) { + err = c.c.DELETE(fmt.Sprintf("/wallets/%v", id)) return } // Wallet returns a client for interacting with the specified wallet. -func (c *Client) Wallet(name string) *WalletClient { - return &WalletClient{c: c.c, name: name} +func (c *Client) Wallet(id int64) *WalletClient { + return &WalletClient{c: c.c, id: id} } // Resubscribe subscribes the wallet to consensus updates, starting at the @@ -115,57 +120,62 @@ func (c *Client) Resubscribe(height uint64) (err error) { // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { - c jape.Client - name string + c jape.Client + id int64 } // AddAddress adds the specified address and associated metadata to the // wallet. -func (c *WalletClient) AddAddress(addr types.Address, info json.RawMessage) (err error) { - err = c.c.PUT(fmt.Sprintf("/wallets/%v/addresses/%v", c.name, addr), info) +func (c *WalletClient) AddAddress(a wallet.Address) (err error) { + err = c.c.PUT(fmt.Sprintf("/wallets/%v/addresses", c.id), a) return } // RemoveAddress removes the specified address from the wallet. func (c *WalletClient) RemoveAddress(addr types.Address) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v/addresses/%v", c.name, addr)) + err = c.c.DELETE(fmt.Sprintf("/wallets/%v/addresses/%v", c.id, addr)) return } // Addresses the addresses controlled by the wallet. -func (c *WalletClient) Addresses() (resp map[types.Address]json.RawMessage, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/addresses", c.name), &resp) +func (c *WalletClient) Addresses() (resp []wallet.Address, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/addresses", c.id), &resp) return } // Balance returns the current wallet balance. func (c *WalletClient) Balance() (resp BalanceResponse, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.name), &resp) + err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.id), &resp) return } // Events returns all events relevant to the wallet. func (c *WalletClient) Events(offset, limit int) (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.name, offset, limit), &resp) + err = c.c.GET(fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.id, offset, limit), &resp) return } // PoolTransactions returns all txpool transactions relevant to the wallet. func (c *WalletClient) PoolTransactions() (resp []wallet.PoolTransaction, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/txpool", c.name), &resp) + err = c.c.GET(fmt.Sprintf("/wallets/%v/txpool", c.id), &resp) return } -// Outputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) Outputs() (sc []types.SiacoinElement, sf []types.SiafundElement, err error) { - var resp WalletOutputsResponse - err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs", c.name), &resp) - return resp.SiacoinOutputs, resp.SiafundOutputs, err +// SiacoinOutputs returns the set of unspent outputs controlled by the wallet. +func (c *WalletClient) SiacoinOutputs(offset, limit int) (sc []types.SiacoinElement, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &sc) + return +} + +// SiafundOutputs returns the set of unspent outputs controlled by the wallet. +func (c *WalletClient) SiafundOutputs(offset, limit int) (sf []types.SiafundElement, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &sf) + return } // Reserve reserves a set outputs for use in a transaction. func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOutputID, duration time.Duration) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.name), WalletReserveRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.id), WalletReserveRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, Duration: duration, @@ -175,7 +185,7 @@ func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOut // Release releases a set of previously-reserved outputs. func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOutputID) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/release", c.name), WalletReleaseRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/release", c.id), WalletReleaseRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, }, nil) @@ -184,7 +194,7 @@ func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOut // Fund funds a siacoin transaction. func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, changeAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fund", c.name), WalletFundRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/fund", c.id), WalletFundRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, @@ -194,7 +204,7 @@ func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, change // FundSF funds a siafund transaction. func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, claimAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fundsf", c.name), WalletFundSFRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/fundsf", c.id), WalletFundSFRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, diff --git a/api/server.go b/api/server.go index 81f63db..6f75c14 100644 --- a/api/server.go +++ b/api/server.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "errors" "net/http" "reflect" @@ -47,21 +46,21 @@ type ( WalletManager interface { Subscribe(startHeight uint64) error - AddWallet(name string, info json.RawMessage) error - DeleteWallet(name string) error - Wallets() (map[string]json.RawMessage, error) + AddWallet(wallet.Wallet) (wallet.Wallet, error) + UpdateWallet(wallet.Wallet) (wallet.Wallet, error) + DeleteWallet(id int64) error + Wallets() ([]wallet.Wallet, error) - AddAddress(name string, addr types.Address, info json.RawMessage) error - RemoveAddress(name string, addr types.Address) error - Addresses(name string) (map[types.Address]json.RawMessage, error) - Events(name string, offset, limit int) ([]wallet.Event, error) - UnspentSiacoinOutputs(name string) ([]types.SiacoinElement, error) - UnspentSiafundOutputs(name string) ([]types.SiafundElement, error) - WalletBalance(walletID string) (wallet.Balance, error) - Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) + AddAddress(id int64, addr wallet.Address) error + RemoveAddress(id int64, addr types.Address) error + Addresses(walletID int64) ([]wallet.Address, error) + Events(walletID int64, offset, limit int) ([]wallet.Event, error) + UnspentSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) + UnspentSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) + WalletBalance(walletID int64) (wallet.Balance, error) + Annotate(walletID int64, pool []types.Transaction) ([]wallet.PoolTransaction, error) Reserve(ids []types.Hash256, duration time.Duration) error - AddressBalance(address types.Address) (wallet.Balance, error) } ) @@ -178,21 +177,53 @@ func (s *server) walletsHandler(jc jape.Context) { jc.Encode(wallets) } -func (s *server) walletsNameHandlerPUT(jc jape.Context) { - var name string - var info json.RawMessage - if jc.DecodeParam("name", &name) != nil || jc.Decode(&info) != nil { +func (s *server) walletsHandlerPOST(jc jape.Context) { + var req WalletUpdateRequest + w := wallet.Wallet{ + Name: req.Name, + Description: req.Description, + Metadata: req.Metadata, + } + + w, err := s.wm.AddWallet(w) + if jc.Check("couldn't add wallet", err) != nil { + return + } + jc.Encode(w) +} + +func (s *server) walletsIDHandlerPOST(jc jape.Context) { + var walletID int64 + var req WalletUpdateRequest + if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&req) != nil { + return + } + w := wallet.Wallet{ + ID: walletID, + Name: req.Name, + Description: req.Description, + Metadata: req.Metadata, + } + + w, err := s.wm.UpdateWallet(w) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) return - } else if jc.Check("couldn't add wallet", s.wm.AddWallet(name, info)) != nil { + } else if jc.Check("couldn't update wallet", err) != nil { return } + jc.Encode(w) } -func (s *server) walletsNameHandlerDELETE(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { +func (s *server) walletsIDHandlerDELETE(jc jape.Context) { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return - } else if jc.Check("couldn't remove wallet", s.wm.DeleteWallet(name)) != nil { + } + err := s.wm.DeleteWallet(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't remove wallet", err) != nil { return } } @@ -207,32 +238,36 @@ func (s *server) resubscribeHandler(jc jape.Context) { } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { - var name string - var addr types.Address - var info json.RawMessage - if jc.DecodeParam("name", &name) != nil || jc.DecodeParam("addr", &addr) != nil || jc.Decode(&info) != nil { + var walletID int64 + var addr wallet.Address + if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&addr) != nil { return - } else if jc.Check("couldn't add address", s.wm.AddAddress(name, addr, info)) != nil { + } else if jc.Check("couldn't add address", s.wm.AddAddress(walletID, addr)) != nil { return } } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { - var name string + var walletID int64 var addr types.Address - if jc.DecodeParam("name", &name) != nil || jc.DecodeParam("addr", &addr) != nil { + if jc.DecodeParam("id", &walletID) != nil || jc.DecodeParam("addr", &addr) != nil { return - } else if jc.Check("couldn't remove address", s.wm.RemoveAddress(name, addr)) != nil { + } + + err := s.wm.RemoveAddress(walletID, addr) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't remove address", err) != nil { return } } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return } - addrs, err := s.wm.Addresses(name) + addrs, err := s.wm.Addresses(walletID) if jc.Check("couldn't load addresses", err) != nil { return } @@ -240,61 +275,87 @@ func (s *server) walletsAddressesHandlerGET(jc jape.Context) { } func (s *server) walletsBalanceHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return } - b, err := s.wm.WalletBalance(name) - if jc.Check("couldn't load balance", err) != nil { + b, err := s.wm.WalletBalance(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load balance", err) != nil { return } jc.Encode(BalanceResponse(b)) } func (s *server) walletsEventsHandler(jc jape.Context) { - var name string - offset, limit := 0, -1 - if jc.DecodeParam("name", &name) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + var walletID int64 + offset, limit := 0, 500 + if jc.DecodeParam("id", &walletID) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - events, err := s.wm.Events(name, offset, limit) - if jc.Check("couldn't load events", err) != nil { + events, err := s.wm.Events(walletID, offset, limit) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load events", err) != nil { return } jc.Encode(events) } func (s *server) walletsTxpoolHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return } - pool, err := s.wm.Annotate(name, s.cm.PoolTransactions()) - if jc.Check("couldn't annotate pool", err) != nil { + pool, err := s.wm.Annotate(walletID, s.cm.PoolTransactions()) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't annotate pool", err) != nil { return } jc.Encode(pool) } -func (s *server) walletsOutputsHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { +func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return } - scos, err := s.wm.UnspentSiacoinOutputs(name) + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + scos, err := s.wm.UnspentSiacoinOutputs(walletID, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } - sfos, err := s.wm.UnspentSiafundOutputs(name) - if jc.Check("couldn't load siafund outputs", err) != nil { + jc.Encode(scos) +} + +func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { + var walletID int64 + if jc.DecodeParam("id", &walletID) != nil { return } - jc.Encode(WalletOutputsResponse{ - SiacoinOutputs: scos, - SiafundOutputs: sfos, - }) + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + sfos, err := s.wm.UnspentSiafundOutputs(walletID, offset, limit) + if jc.Check("couldn't load siacoin outputs", err) != nil { + return + } + jc.Encode(sfos) } func (s *server) walletsReserveHandler(jc jape.Context) { @@ -384,12 +445,12 @@ func (s *server) walletsFundHandler(jc jape.Context) { return toSign, nil } - var name string + var walletID int64 var wfr WalletFundRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiacoinOutputs(name) + utxos, err := s.wm.UnspentSiacoinOutputs(walletID, 0, 1000) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -458,12 +519,12 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { return toSign, nil } - var name string + var walletID int64 var wfr WalletFundSFRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiafundOutputs(name) + utxos, err := s.wm.UnspentSiafundOutputs(walletID, 0, 1000) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -503,19 +564,21 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "POST /resubscribe": srv.resubscribeHandler, - "GET /wallets": srv.walletsHandler, - "PUT /wallets/:name": srv.walletsNameHandlerPUT, - "DELETE /wallets/:name": srv.walletsNameHandlerDELETE, - "PUT /wallets/:name/addresses/:addr": srv.walletsAddressHandlerPUT, - "DELETE /wallets/:name/addresses/:addr": srv.walletsAddressHandlerDELETE, - "GET /wallets/:name/addresses": srv.walletsAddressesHandlerGET, - "GET /wallets/:name/balance": srv.walletsBalanceHandler, - "GET /wallets/:name/events": srv.walletsEventsHandler, - "GET /wallets/:name/txpool": srv.walletsTxpoolHandler, - "GET /wallets/:name/outputs": srv.walletsOutputsHandler, - "POST /wallets/:name/reserve": srv.walletsReserveHandler, - "POST /wallets/:name/release": srv.walletsReleaseHandler, - "POST /wallets/:name/fund": srv.walletsFundHandler, - "POST /wallets/:name/fundsf": srv.walletsFundSFHandler, + "GET /wallets": srv.walletsHandler, + "POST /wallets": srv.walletsHandlerPOST, + "POST /wallets/:id": srv.walletsIDHandlerPOST, + "DELETE /wallets/:id": srv.walletsIDHandlerDELETE, + "PUT /wallets/:id/addresses": srv.walletsAddressHandlerPUT, + "DELETE /wallets/:id/addresses/:addr": srv.walletsAddressHandlerDELETE, + "GET /wallets/:id/addresses": srv.walletsAddressesHandlerGET, + "GET /wallets/:id/balance": srv.walletsBalanceHandler, + "GET /wallets/:id/events": srv.walletsEventsHandler, + "GET /wallets/:id/txpool": srv.walletsTxpoolHandler, + "GET /wallets/:id/outputs/siacoin": srv.walletsOutputsSiacoinHandler, + "GET /wallets/:id/outputs/siafund": srv.walletsOutputsSiafundHandler, + "POST /wallets/:id/reserve": srv.walletsReserveHandler, + "POST /wallets/:id/release": srv.walletsReleaseHandler, + "POST /wallets/:id/fund": srv.walletsFundHandler, + "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, }) } From be1d150fbd5c0aa61bdb0a6cebb6b839e3bb094e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:46:49 -0800 Subject: [PATCH 103/630] api,wallet: fix lint errors --- api/client.go | 1 + wallet/manager.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/client.go b/api/client.go index dcee7bb..81e2dad 100644 --- a/api/client.go +++ b/api/client.go @@ -93,6 +93,7 @@ func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) return } +// UpdateWallet updates a wallet. func (c *Client) UpdateWallet(id int64, uw WalletUpdateRequest) (w wallet.Wallet, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) return diff --git a/wallet/manager.go b/wallet/manager.go index 0c496c5..656dcb3 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -58,6 +58,7 @@ func (m *Manager) AddWallet(w Wallet) (Wallet, error) { return m.store.AddWallet(w) } +// UpdateWallet updates the given wallet. func (m *Manager) UpdateWallet(w Wallet) (Wallet, error) { return m.store.UpdateWallet(w) } From ee06c36daaf33dec1402c19cfe41c4a4f2e50bee Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 11:47:49 -0800 Subject: [PATCH 104/630] sqlite: fix lint errors, check rows.Err() --- persist/sqlite/addresses.go | 5 +---- persist/sqlite/consensus.go | 7 +++++-- persist/sqlite/peers.go | 2 +- persist/sqlite/wallet.go | 12 +++++++----- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 9a628ac..daf4cd0 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -42,10 +42,7 @@ func (s *Store) AddressEvents(address types.Address, limit, offset int) (events events = append(events, event) } - if err = rows.Err(); err != nil { - return err - } - return nil + return rows.Err() }) return } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 0f3e67b..c110087 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -45,7 +45,7 @@ func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { } elements = append(elements, se) } - return elements, nil + return elements, rows.Err() } func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) error { @@ -82,7 +82,7 @@ func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { } elements = append(elements, se) } - return elements, nil + return elements, rows.Err() } func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) error { @@ -160,6 +160,9 @@ WHERE maturity_height=$1` } elements = append(elements, element) } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to scan siacoin elements: %w", err) + } return } diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 4d8de8d..1427737 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -54,7 +54,7 @@ func (s *Store) Peers() (peers []string) { } peers = append(peers, peer) } - return nil + return rows.Err() }) if err != nil { panic(err) // 😔 diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index ad7801a..4ad9639 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -110,7 +110,7 @@ WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN ( } relevantAddresses[eventID] = append(relevantAddresses[eventID], address) } - return relevantAddresses, nil + return relevantAddresses, rows.Err() } // WalletEvents returns the events relevant to a wallet, sorted by height descending. @@ -147,6 +147,7 @@ func (s *Store) AddWallet(w wallet.Wallet) (wallet.Wallet, error) { return w, err } +// UpdateWallet updates a wallet in the database. func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { w.LastUpdated = time.Now() err := s.transaction(func(tx *txn) error { @@ -275,7 +276,7 @@ WHERE wa.wallet_id=$1` addresses = append(addresses, address) } - return nil + return rows.Err() }) return } @@ -308,7 +309,7 @@ func (s *Store) WalletSiacoinOutputs(walletID int64, offset, limit int) (siacoin siacoins = append(siacoins, siacoin) } - return nil + return rows.Err() }) return } @@ -340,7 +341,7 @@ func (s *Store) WalletSiafundOutputs(walletID int64, offset, limit int) (siafund } siafunds = append(siafunds, siafund) } - return nil + return rows.Err() }) return } @@ -360,6 +361,7 @@ func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error if err != nil { return err } + defer rows.Close() for rows.Next() { var addressSC types.Currency @@ -373,7 +375,7 @@ func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(addressISC) balance.Siafunds += addressSF } - return nil + return rows.Err() }) return } From ada16e1c77c5c493852b7d7ac29d599682381297 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 14:12:32 -0800 Subject: [PATCH 105/630] api,sqlite,wallet: add WalletID type --- api/client.go | 8 ++-- api/server.go | 90 ++++++++++++++++++++-------------------- persist/sqlite/wallet.go | 62 +++++++++++++-------------- wallet/manager.go | 52 +++++++++++------------ wallet/wallet.go | 21 +++++++++- 5 files changed, 126 insertions(+), 107 deletions(-) diff --git a/api/client.go b/api/client.go index 81e2dad..c614f13 100644 --- a/api/client.go +++ b/api/client.go @@ -94,20 +94,20 @@ func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) } // UpdateWallet updates a wallet. -func (c *Client) UpdateWallet(id int64, uw WalletUpdateRequest) (w wallet.Wallet, err error) { +func (c *Client) UpdateWallet(id wallet.WalletID, uw WalletUpdateRequest) (w wallet.Wallet, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) return } // RemoveWallet deletes a wallet. If the wallet is currently subscribed, it will // be unsubscribed. -func (c *Client) RemoveWallet(id int64) (err error) { +func (c *Client) RemoveWallet(id wallet.WalletID) (err error) { err = c.c.DELETE(fmt.Sprintf("/wallets/%v", id)) return } // Wallet returns a client for interacting with the specified wallet. -func (c *Client) Wallet(id int64) *WalletClient { +func (c *Client) Wallet(id wallet.WalletID) *WalletClient { return &WalletClient{c: c.c, id: id} } @@ -122,7 +122,7 @@ func (c *Client) Resubscribe(height uint64) (err error) { // walletd API server. type WalletClient struct { c jape.Client - id int64 + id wallet.WalletID } // AddAddress adds the specified address and associated metadata to the diff --git a/api/server.go b/api/server.go index 6f75c14..24a22ff 100644 --- a/api/server.go +++ b/api/server.go @@ -48,17 +48,17 @@ type ( AddWallet(wallet.Wallet) (wallet.Wallet, error) UpdateWallet(wallet.Wallet) (wallet.Wallet, error) - DeleteWallet(id int64) error + DeleteWallet(wallet.WalletID) error Wallets() ([]wallet.Wallet, error) - AddAddress(id int64, addr wallet.Address) error - RemoveAddress(id int64, addr types.Address) error - Addresses(walletID int64) ([]wallet.Address, error) - Events(walletID int64, offset, limit int) ([]wallet.Event, error) - UnspentSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) - UnspentSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) - WalletBalance(walletID int64) (wallet.Balance, error) - Annotate(walletID int64, pool []types.Transaction) ([]wallet.PoolTransaction, error) + AddAddress(id wallet.WalletID, addr wallet.Address) error + RemoveAddress(id wallet.WalletID, addr types.Address) error + Addresses(id wallet.WalletID) ([]wallet.Address, error) + Events(id wallet.WalletID, offset, limit int) ([]wallet.Event, error) + UnspentSiacoinOutputs(id wallet.WalletID, offset, limit int) ([]types.SiacoinElement, error) + UnspentSiafundOutputs(id wallet.WalletID, offset, limit int) ([]types.SiafundElement, error) + WalletBalance(id wallet.WalletID) (wallet.Balance, error) + Annotate(id wallet.WalletID, pool []types.Transaction) ([]wallet.PoolTransaction, error) Reserve(ids []types.Hash256, duration time.Duration) error } @@ -193,13 +193,13 @@ func (s *server) walletsHandlerPOST(jc jape.Context) { } func (s *server) walletsIDHandlerPOST(jc jape.Context) { - var walletID int64 + var id wallet.WalletID var req WalletUpdateRequest - if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&req) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&req) != nil { return } w := wallet.Wallet{ - ID: walletID, + ID: id, Name: req.Name, Description: req.Description, Metadata: req.Metadata, @@ -216,11 +216,11 @@ func (s *server) walletsIDHandlerPOST(jc jape.Context) { } func (s *server) walletsIDHandlerDELETE(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } - err := s.wm.DeleteWallet(walletID) + err := s.wm.DeleteWallet(id) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("couldn't remove wallet", err) != nil { @@ -238,23 +238,23 @@ func (s *server) resubscribeHandler(jc jape.Context) { } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { - var walletID int64 + var id wallet.WalletID var addr wallet.Address - if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&addr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&addr) != nil { return - } else if jc.Check("couldn't add address", s.wm.AddAddress(walletID, addr)) != nil { + } else if jc.Check("couldn't add address", s.wm.AddAddress(id, addr)) != nil { return } } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { - var walletID int64 + var id wallet.WalletID var addr types.Address - if jc.DecodeParam("id", &walletID) != nil || jc.DecodeParam("addr", &addr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.DecodeParam("addr", &addr) != nil { return } - err := s.wm.RemoveAddress(walletID, addr) + err := s.wm.RemoveAddress(id, addr) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) } else if jc.Check("couldn't remove address", err) != nil { @@ -263,11 +263,11 @@ func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } - addrs, err := s.wm.Addresses(walletID) + addrs, err := s.wm.Addresses(id) if jc.Check("couldn't load addresses", err) != nil { return } @@ -275,12 +275,12 @@ func (s *server) walletsAddressesHandlerGET(jc jape.Context) { } func (s *server) walletsBalanceHandler(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } - b, err := s.wm.WalletBalance(walletID) + b, err := s.wm.WalletBalance(id) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) return @@ -291,12 +291,12 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { } func (s *server) walletsEventsHandler(jc jape.Context) { - var walletID int64 + var id wallet.WalletID offset, limit := 0, 500 - if jc.DecodeParam("id", &walletID) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + if jc.DecodeParam("id", &id) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - events, err := s.wm.Events(walletID, offset, limit) + events, err := s.wm.Events(id, offset, limit) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) return @@ -307,11 +307,11 @@ func (s *server) walletsEventsHandler(jc jape.Context) { } func (s *server) walletsTxpoolHandler(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } - pool, err := s.wm.Annotate(walletID, s.cm.PoolTransactions()) + pool, err := s.wm.Annotate(id, s.cm.PoolTransactions()) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) return @@ -322,8 +322,8 @@ func (s *server) walletsTxpoolHandler(jc jape.Context) { } func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } @@ -332,7 +332,7 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { return } - scos, err := s.wm.UnspentSiacoinOutputs(walletID, offset, limit) + scos, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } @@ -341,8 +341,8 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { } func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { - var walletID int64 - if jc.DecodeParam("id", &walletID) != nil { + var id wallet.WalletID + if jc.DecodeParam("id", &id) != nil { return } @@ -351,7 +351,7 @@ func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { return } - sfos, err := s.wm.UnspentSiafundOutputs(walletID, offset, limit) + sfos, err := s.wm.UnspentSiafundOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } @@ -445,12 +445,12 @@ func (s *server) walletsFundHandler(jc jape.Context) { return toSign, nil } - var walletID int64 + var id wallet.WalletID var wfr WalletFundRequest - if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiacoinOutputs(walletID, 0, 1000) + utxos, err := s.wm.UnspentSiacoinOutputs(id, 0, 1000) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -519,12 +519,12 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { return toSign, nil } - var walletID int64 + var id wallet.WalletID var wfr WalletFundSFRequest - if jc.DecodeParam("id", &walletID) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiafundOutputs(walletID, 0, 1000) + utxos, err := s.wm.UnspentSiafundOutputs(id, 0, 1000) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 4ad9639..8e9185d 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -60,7 +60,7 @@ func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { return } -func getWalletEvents(tx *txn, walletID int64, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { +func getWalletEvents(tx *txn, id wallet.WalletID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INNER JOIN chain_indices ci ON (ev.index_id = ci.id) @@ -68,7 +68,7 @@ func getWalletEvents(tx *txn, walletID int64, offset, limit int) (events []walle ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, walletID, limit, offset) + rows, err := tx.Query(query, id, limit, offset) if err != nil { return nil, nil, err } @@ -89,13 +89,13 @@ func getWalletEvents(tx *txn, walletID int64, offset, limit int) (events []walle return } -func (s *Store) getWalletEventRelevantAddresses(tx *txn, walletID int64, eventIDs []int64) (map[int64][]types.Address, error) { +func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.WalletID, eventIDs []int64) (map[int64][]types.Address, error) { query := `SELECT ea.event_id, sa.sia_address FROM event_addresses ea INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=?)` - rows, err := tx.Query(query, append(queryArgs(eventIDs), walletID)...) + rows, err := tx.Query(query, append(queryArgs(eventIDs), id)...) if err != nil { return nil, err } @@ -114,15 +114,15 @@ WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN ( } // WalletEvents returns the events relevant to a wallet, sorted by height descending. -func (s *Store) WalletEvents(walletID int64, offset, limit int) (events []wallet.Event, err error) { +func (s *Store) WalletEvents(id wallet.WalletID, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { var dbIDs []int64 - events, dbIDs, err = getWalletEvents(tx, walletID, offset, limit) + events, dbIDs, err = getWalletEvents(tx, id, offset, limit) if err != nil { return fmt.Errorf("failed to get wallet events: %w", err) } - eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, walletID, dbIDs) + eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, id, dbIDs) if err != nil { return fmt.Errorf("failed to get relevant addresses: %w", err) } @@ -164,10 +164,10 @@ func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { // DeleteWallet deletes a wallet from the database. This does not stop tracking // addresses that were previously associated with the wallet. -func (s *Store) DeleteWallet(walletID int64) error { +func (s *Store) DeleteWallet(id wallet.WalletID) error { return s.transaction(func(tx *txn) error { var dummyID int64 - err := tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, walletID).Scan(&dummyID) + err := tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, id).Scan(&dummyID) if errors.Is(err, sql.ErrNoRows) { return wallet.ErrNotFound } @@ -199,9 +199,9 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { } // AddWalletAddress adds an address to a wallet. -func (s *Store) AddWalletAddress(walletID int64, addr wallet.Address) error { +func (s *Store) AddWalletAddress(id wallet.WalletID, addr wallet.Address) error { return s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -215,18 +215,18 @@ func (s *Store) AddWalletAddress(walletID int64, addr wallet.Address) error { encodedPolicy = encode(*addr.SpendPolicy) } - _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5)`, walletID, addressID, addr.Description, encodedPolicy, addr.Metadata) + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5)`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) return err }) } // RemoveWalletAddress removes an address from a wallet. This does not stop tracking // the address. -func (s *Store) RemoveWalletAddress(walletID int64, address types.Address) error { +func (s *Store) RemoveWalletAddress(id wallet.WalletID, address types.Address) error { return s.transaction(func(tx *txn) error { const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING id` var dummyID int64 - err := tx.QueryRow(query, walletID, encode(address)).Scan(&dummyID) + err := tx.QueryRow(query, id, encode(address)).Scan(&dummyID) if errors.Is(err, sql.ErrNoRows) { return wallet.ErrNotFound } @@ -235,9 +235,9 @@ func (s *Store) RemoveWalletAddress(walletID int64, address types.Address) error } // WalletAddresses returns a slice of addresses registered to the wallet. -func (s *Store) WalletAddresses(walletID int64) (addresses []wallet.Address, err error) { +func (s *Store) WalletAddresses(id wallet.WalletID) (addresses []wallet.Address, err error) { err = s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -246,7 +246,7 @@ FROM wallet_addresses wa INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` - rows, err := tx.Query(query, walletID) + rows, err := tx.Query(query, id) if err != nil { return err } @@ -282,9 +282,9 @@ WHERE wa.wallet_id=$1` } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(walletID int64, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) WalletSiacoinOutputs(id wallet.WalletID, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -294,7 +294,7 @@ func (s *Store) WalletSiacoinOutputs(walletID int64, offset, limit int) (siacoin WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, walletID, limit, offset) + rows, err := tx.Query(query, id, limit, offset) if err != nil { return err } @@ -315,9 +315,9 @@ func (s *Store) WalletSiacoinOutputs(walletID int64, offset, limit int) (siacoin } // WalletSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) WalletSiafundOutputs(walletID int64, offset, limit int) (siafunds []types.SiafundElement, err error) { +func (s *Store) WalletSiafundOutputs(id wallet.WalletID, offset, limit int) (siafunds []types.SiafundElement, err error) { err = s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -327,7 +327,7 @@ func (s *Store) WalletSiafundOutputs(walletID int64, offset, limit int) (siafund WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, walletID, limit, offset) + rows, err := tx.Query(query, id, limit, offset) if err != nil { return err } @@ -347,9 +347,9 @@ func (s *Store) WalletSiafundOutputs(walletID int64, offset, limit int) (siafund } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error) { +func (s *Store) WalletBalance(id wallet.WalletID) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -357,7 +357,7 @@ func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1` - rows, err := tx.Query(query, walletID) + rows, err := tx.Query(query, id) if err != nil { return err } @@ -381,9 +381,9 @@ func (s *Store) WalletBalance(walletID int64) (balance wallet.Balance, err error } // Annotate annotates a list of transactions using the wallet's addresses. -func (s *Store) Annotate(walletID int64, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { +func (s *Store) Annotate(id wallet.WalletID, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { err = s.transaction(func(tx *txn) error { - if err := walletExists(tx, walletID); err != nil { + if err := walletExists(tx, id); err != nil { return err } @@ -404,7 +404,7 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` // addresses into memory. ownsAddress := func(address types.Address) bool { var dbID int64 - err := stmt.QueryRow(walletID, encode(address)).Scan(dbID) + err := stmt.QueryRow(id, encode(address)).Scan(dbID) if err != nil && !errors.Is(err, sql.ErrNoRows) { panic(err) // database error } @@ -422,10 +422,10 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` return } -func walletExists(tx *txn, walletID int64) error { +func walletExists(tx *txn, id wallet.WalletID) error { const query = `SELECT id FROM wallets WHERE id=$1` var dummyID int64 - err := tx.QueryRow(query, walletID).Scan(&dummyID) + err := tx.QueryRow(query, id).Scan(&dummyID) if errors.Is(err, sql.ErrNoRows) { return wallet.ErrNotFound } diff --git a/wallet/manager.go b/wallet/manager.go index 656dcb3..63674b8 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -24,20 +24,20 @@ type ( Store interface { chain.Subscriber - WalletEvents(id int64, offset, limit int) ([]Event, error) + WalletEvents(id WalletID, offset, limit int) ([]Event, error) AddWallet(Wallet) (Wallet, error) UpdateWallet(Wallet) (Wallet, error) - DeleteWallet(id int64) error - WalletBalance(id int64) (Balance, error) - WalletSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) - WalletSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) - WalletAddresses(walletID int64) ([]Address, error) + DeleteWallet(id WalletID) error + WalletBalance(id WalletID) (Balance, error) + WalletSiacoinOutputs(id WalletID, offset, limit int) ([]types.SiacoinElement, error) + WalletSiafundOutputs(id WalletID, offset, limit int) ([]types.SiafundElement, error) + WalletAddresses(id WalletID) ([]Address, error) Wallets() ([]Wallet, error) - AddWalletAddress(walletID int64, address Address) error - RemoveWalletAddress(walletID int64, address types.Address) error + AddWalletAddress(id WalletID, address Address) error + RemoveWalletAddress(id WalletID, address types.Address) error - Annotate(walletID int64, txns []types.Transaction) ([]PoolTransaction, error) + Annotate(id WalletID, txns []types.Transaction) ([]PoolTransaction, error) LastCommittedIndex() (types.ChainIndex, error) } @@ -64,7 +64,7 @@ func (m *Manager) UpdateWallet(w Wallet) (Wallet, error) { } // DeleteWallet deletes the given wallet. -func (m *Manager) DeleteWallet(id int64) error { +func (m *Manager) DeleteWallet(id WalletID) error { return m.store.DeleteWallet(id) } @@ -74,44 +74,44 @@ func (m *Manager) Wallets() ([]Wallet, error) { } // AddAddress adds the given address to the given wallet. -func (m *Manager) AddAddress(walletID int64, addr Address) error { - return m.store.AddWalletAddress(walletID, addr) +func (m *Manager) AddAddress(id WalletID, addr Address) error { + return m.store.AddWalletAddress(id, addr) } // RemoveAddress removes the given address from the given wallet. -func (m *Manager) RemoveAddress(walletID int64, addr types.Address) error { - return m.store.RemoveWalletAddress(walletID, addr) +func (m *Manager) RemoveAddress(id WalletID, addr types.Address) error { + return m.store.RemoveWalletAddress(id, addr) } // Addresses returns the addresses of the given wallet. -func (m *Manager) Addresses(walletID int64) ([]Address, error) { - return m.store.WalletAddresses(walletID) +func (m *Manager) Addresses(id WalletID) ([]Address, error) { + return m.store.WalletAddresses(id) } // Events returns the events of the given wallet. -func (m *Manager) Events(walletID int64, offset, limit int) ([]Event, error) { - return m.store.WalletEvents(walletID, offset, limit) +func (m *Manager) Events(id WalletID, offset, limit int) ([]Event, error) { + return m.store.WalletEvents(id, offset, limit) } // UnspentSiacoinOutputs returns a paginated list of unspent siacoin outputs of // the given wallet and the total number of unspent siacoin outputs. -func (m *Manager) UnspentSiacoinOutputs(walletID int64, offset, limit int) ([]types.SiacoinElement, error) { - return m.store.WalletSiacoinOutputs(walletID, offset, limit) +func (m *Manager) UnspentSiacoinOutputs(id WalletID, offset, limit int) ([]types.SiacoinElement, error) { + return m.store.WalletSiacoinOutputs(id, offset, limit) } // UnspentSiafundOutputs returns the unspent siafund outputs of the given wallet -func (m *Manager) UnspentSiafundOutputs(walletID int64, offset, limit int) ([]types.SiafundElement, error) { - return m.store.WalletSiafundOutputs(walletID, offset, limit) +func (m *Manager) UnspentSiafundOutputs(id WalletID, offset, limit int) ([]types.SiafundElement, error) { + return m.store.WalletSiafundOutputs(id, offset, limit) } // Annotate annotates the given transactions with the wallet they belong to. -func (m *Manager) Annotate(walletID int64, pool []types.Transaction) ([]PoolTransaction, error) { - return m.store.Annotate(walletID, pool) +func (m *Manager) Annotate(id WalletID, pool []types.Transaction) ([]PoolTransaction, error) { + return m.store.Annotate(id, pool) } // WalletBalance returns the balance of the given wallet. -func (m *Manager) WalletBalance(walletID int64) (Balance, error) { - return m.store.WalletBalance(walletID) +func (m *Manager) WalletBalance(id WalletID) (Balance, error) { + return m.store.WalletBalance(id) } // Reserve reserves the given ids for the given duration. diff --git a/wallet/wallet.go b/wallet/wallet.go index 43e1b21..d266e6b 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "time" "go.sia.tech/core/consensus" @@ -27,9 +28,12 @@ type ( Siafunds uint64 `json:"siafunds"` } + // A WalletID is a unique identifier for a wallet. + WalletID int64 + // A Wallet is a collection of addresses and metadata. Wallet struct { - ID int64 `json:"id"` + ID WalletID `json:"id"` Name string `json:"name"` Description string `json:"description"` DateCreated time.Time `json:"dateCreated"` @@ -49,6 +53,21 @@ type ( // ErrNotFound is returned when a requested wallet or address is not found. var ErrNotFound = errors.New("not found") +// UnmarshalText implements encoding.TextUnmarshaler. +func (w *WalletID) UnmarshalText(buf []byte) error { + id, err := strconv.ParseInt(string(buf), 10, 64) + if err != nil { + return err + } + *w = WalletID(id) + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (w WalletID) MarshalText() ([]byte, error) { + return []byte(strconv.FormatInt(int64(w), 10)), nil +} + // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. From 7b1051708454530e75106847226bc119980d5402 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 14:22:43 -0800 Subject: [PATCH 106/630] sqllite: fix address queries, add test --- persist/sqlite/wallet.go | 9 ++- persist/sqlite/wallet_test.go | 106 ++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 persist/sqlite/wallet_test.go diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 8e9185d..b137830 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -188,7 +188,7 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { for rows.Next() { var w wallet.Wallet - if err := rows.Scan(&w.ID, &w.Name, &w.Description, decode(&w.DateCreated), decode(&w.LastUpdated), &w.Metadata); err != nil { + if err := rows.Scan(&w.ID, &w.Name, &w.Description, decode(&w.DateCreated), decode(&w.LastUpdated), (*[]byte)(&w.Metadata)); err != nil { return fmt.Errorf("failed to scan wallet: %w", err) } wallets = append(wallets, w) @@ -215,7 +215,7 @@ func (s *Store) AddWalletAddress(id wallet.WalletID, addr wallet.Address) error encodedPolicy = encode(*addr.SpendPolicy) } - _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5)`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) return err }) } @@ -224,7 +224,7 @@ func (s *Store) AddWalletAddress(id wallet.WalletID, addr wallet.Address) error // the address. func (s *Store) RemoveWalletAddress(id wallet.WalletID, address types.Address) error { return s.transaction(func(tx *txn) error { - const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING id` + const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING address_id` var dummyID int64 err := tx.QueryRow(query, id, encode(address)).Scan(&dummyID) if errors.Is(err, sql.ErrNoRows) { @@ -254,9 +254,8 @@ WHERE wa.wallet_id=$1` for rows.Next() { var address wallet.Address - var decodedPolicy any - if err := rows.Scan(decode(&address.Address), &address.Description, &decodedPolicy, &address.Metadata); err != nil { + if err := rows.Scan(decode(&address.Address), &address.Description, &decodedPolicy, (*[]byte)(&address.Metadata)); err != nil { return fmt.Errorf("failed to scan address: %w", err) } diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go new file mode 100644 index 0000000..5c1e960 --- /dev/null +++ b/persist/sqlite/wallet_test.go @@ -0,0 +1,106 @@ +package sqlite + +import ( + "encoding/json" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" +) + +func TestWalletAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // Add a wallet + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + wallets, err := db.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != 1 { + t.Fatal("expected 1 wallet, got", len(wallets)) + } else if wallets[0].ID != w.ID { + t.Fatal("unexpected wallet ID", wallets[0].ID) + } else if wallets[0].Name != "test" { + t.Fatal("unexpected wallet name", wallets[0].Name) + } else if wallets[0].Metadata != nil { + t.Fatal("unexpected metadata", wallets[0].Metadata) + } + + // Add an address + pk := types.GeneratePrivateKey() + spendPolicy := types.PolicyPublicKey(pk.PublicKey()) + address := spendPolicy.Address() + + addr := wallet.Address{ + Address: address, + SpendPolicy: &spendPolicy, + Description: "hello, world", + } + err = db.AddWalletAddress(w.ID, addr) + if err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err := db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "hello, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } + + // update the addresses metadata and description + addr.Description = "goodbye, world" + addr.Metadata = json.RawMessage(`{"foo": "bar"}`) + + if err := db.AddWalletAddress(w.ID, addr); err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err = db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "goodbye, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } else if string(addresses[0].Metadata) != `{"foo": "bar"}` { + t.Fatal("unexpected metadata", addresses[0].Metadata) + } + + // Remove the address + err = db.RemoveWalletAddress(w.ID, address) + if err != nil { + t.Fatal(err) + } + + // Check that the address was removed + addresses, err = db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 0 { + t.Fatal("expected 0 addresses, got", len(addresses)) + } +} From 2f1a478ddf2c422a4d9101e21aab92b50373dae6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 23 Feb 2024 15:41:06 -0800 Subject: [PATCH 107/630] Change wallet.WalletID to wallet.ID --- api/client.go | 8 +++--- api/server.go | 42 +++++++++++++++---------------- persist/sqlite/wallet.go | 24 +++++++++--------- wallet/manager.go | 54 ++++++++++++++++++++-------------------- wallet/wallet.go | 12 ++++----- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/api/client.go b/api/client.go index c614f13..04d0c90 100644 --- a/api/client.go +++ b/api/client.go @@ -94,20 +94,20 @@ func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) } // UpdateWallet updates a wallet. -func (c *Client) UpdateWallet(id wallet.WalletID, uw WalletUpdateRequest) (w wallet.Wallet, err error) { +func (c *Client) UpdateWallet(id wallet.ID, uw WalletUpdateRequest) (w wallet.Wallet, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) return } // RemoveWallet deletes a wallet. If the wallet is currently subscribed, it will // be unsubscribed. -func (c *Client) RemoveWallet(id wallet.WalletID) (err error) { +func (c *Client) RemoveWallet(id wallet.ID) (err error) { err = c.c.DELETE(fmt.Sprintf("/wallets/%v", id)) return } // Wallet returns a client for interacting with the specified wallet. -func (c *Client) Wallet(id wallet.WalletID) *WalletClient { +func (c *Client) Wallet(id wallet.ID) *WalletClient { return &WalletClient{c: c.c, id: id} } @@ -122,7 +122,7 @@ func (c *Client) Resubscribe(height uint64) (err error) { // walletd API server. type WalletClient struct { c jape.Client - id wallet.WalletID + id wallet.ID } // AddAddress adds the specified address and associated metadata to the diff --git a/api/server.go b/api/server.go index 24a22ff..f9f33fe 100644 --- a/api/server.go +++ b/api/server.go @@ -48,17 +48,17 @@ type ( AddWallet(wallet.Wallet) (wallet.Wallet, error) UpdateWallet(wallet.Wallet) (wallet.Wallet, error) - DeleteWallet(wallet.WalletID) error + DeleteWallet(wallet.ID) error Wallets() ([]wallet.Wallet, error) - AddAddress(id wallet.WalletID, addr wallet.Address) error - RemoveAddress(id wallet.WalletID, addr types.Address) error - Addresses(id wallet.WalletID) ([]wallet.Address, error) - Events(id wallet.WalletID, offset, limit int) ([]wallet.Event, error) - UnspentSiacoinOutputs(id wallet.WalletID, offset, limit int) ([]types.SiacoinElement, error) - UnspentSiafundOutputs(id wallet.WalletID, offset, limit int) ([]types.SiafundElement, error) - WalletBalance(id wallet.WalletID) (wallet.Balance, error) - Annotate(id wallet.WalletID, pool []types.Transaction) ([]wallet.PoolTransaction, error) + AddAddress(id wallet.ID, addr wallet.Address) error + RemoveAddress(id wallet.ID, addr types.Address) error + Addresses(id wallet.ID) ([]wallet.Address, error) + Events(id wallet.ID, offset, limit int) ([]wallet.Event, error) + UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) + UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) + WalletBalance(id wallet.ID) (wallet.Balance, error) + Annotate(id wallet.ID, pool []types.Transaction) ([]wallet.PoolTransaction, error) Reserve(ids []types.Hash256, duration time.Duration) error } @@ -193,7 +193,7 @@ func (s *server) walletsHandlerPOST(jc jape.Context) { } func (s *server) walletsIDHandlerPOST(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID var req WalletUpdateRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&req) != nil { return @@ -216,7 +216,7 @@ func (s *server) walletsIDHandlerPOST(jc jape.Context) { } func (s *server) walletsIDHandlerDELETE(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -238,7 +238,7 @@ func (s *server) resubscribeHandler(jc jape.Context) { } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID var addr wallet.Address if jc.DecodeParam("id", &id) != nil || jc.Decode(&addr) != nil { return @@ -248,7 +248,7 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID var addr types.Address if jc.DecodeParam("id", &id) != nil || jc.DecodeParam("addr", &addr) != nil { return @@ -263,7 +263,7 @@ func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -275,7 +275,7 @@ func (s *server) walletsAddressesHandlerGET(jc jape.Context) { } func (s *server) walletsBalanceHandler(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -291,7 +291,7 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { } func (s *server) walletsEventsHandler(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID offset, limit := 0, 500 if jc.DecodeParam("id", &id) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return @@ -307,7 +307,7 @@ func (s *server) walletsEventsHandler(jc jape.Context) { } func (s *server) walletsTxpoolHandler(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -322,7 +322,7 @@ func (s *server) walletsTxpoolHandler(jc jape.Context) { } func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -341,7 +341,7 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { } func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { - var id wallet.WalletID + var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } @@ -445,7 +445,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { return toSign, nil } - var id wallet.WalletID + var id wallet.ID var wfr WalletFundRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return @@ -519,7 +519,7 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { return toSign, nil } - var id wallet.WalletID + var id wallet.ID var wfr WalletFundSFRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index b137830..7f76eda 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -60,7 +60,7 @@ func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { return } -func getWalletEvents(tx *txn, id wallet.WalletID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { +func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INNER JOIN chain_indices ci ON (ev.index_id = ci.id) @@ -89,7 +89,7 @@ func getWalletEvents(tx *txn, id wallet.WalletID, offset, limit int) (events []w return } -func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.WalletID, eventIDs []int64) (map[int64][]types.Address, error) { +func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.ID, eventIDs []int64) (map[int64][]types.Address, error) { query := `SELECT ea.event_id, sa.sia_address FROM event_addresses ea INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) @@ -114,7 +114,7 @@ WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN ( } // WalletEvents returns the events relevant to a wallet, sorted by height descending. -func (s *Store) WalletEvents(id wallet.WalletID, offset, limit int) (events []wallet.Event, err error) { +func (s *Store) WalletEvents(id wallet.ID, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { var dbIDs []int64 events, dbIDs, err = getWalletEvents(tx, id, offset, limit) @@ -164,7 +164,7 @@ func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { // DeleteWallet deletes a wallet from the database. This does not stop tracking // addresses that were previously associated with the wallet. -func (s *Store) DeleteWallet(id wallet.WalletID) error { +func (s *Store) DeleteWallet(id wallet.ID) error { return s.transaction(func(tx *txn) error { var dummyID int64 err := tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, id).Scan(&dummyID) @@ -199,7 +199,7 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { } // AddWalletAddress adds an address to a wallet. -func (s *Store) AddWalletAddress(id wallet.WalletID, addr wallet.Address) error { +func (s *Store) AddWalletAddress(id wallet.ID, addr wallet.Address) error { return s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -222,7 +222,7 @@ func (s *Store) AddWalletAddress(id wallet.WalletID, addr wallet.Address) error // RemoveWalletAddress removes an address from a wallet. This does not stop tracking // the address. -func (s *Store) RemoveWalletAddress(id wallet.WalletID, address types.Address) error { +func (s *Store) RemoveWalletAddress(id wallet.ID, address types.Address) error { return s.transaction(func(tx *txn) error { const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING address_id` var dummyID int64 @@ -235,7 +235,7 @@ func (s *Store) RemoveWalletAddress(id wallet.WalletID, address types.Address) e } // WalletAddresses returns a slice of addresses registered to the wallet. -func (s *Store) WalletAddresses(id wallet.WalletID) (addresses []wallet.Address, err error) { +func (s *Store) WalletAddresses(id wallet.ID) (addresses []wallet.Address, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -281,7 +281,7 @@ WHERE wa.wallet_id=$1` } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(id wallet.WalletID, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -314,7 +314,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.WalletID, offset, limit int) (sia } // WalletSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) WalletSiafundOutputs(id wallet.WalletID, offset, limit int) (siafunds []types.SiafundElement, err error) { +func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []types.SiafundElement, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -346,7 +346,7 @@ func (s *Store) WalletSiafundOutputs(id wallet.WalletID, offset, limit int) (sia } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(id wallet.WalletID) (balance wallet.Balance, err error) { +func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -380,7 +380,7 @@ func (s *Store) WalletBalance(id wallet.WalletID) (balance wallet.Balance, err e } // Annotate annotates a list of transactions using the wallet's addresses. -func (s *Store) Annotate(id wallet.WalletID, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { +func (s *Store) Annotate(id wallet.ID, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -421,7 +421,7 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` return } -func walletExists(tx *txn, id wallet.WalletID) error { +func walletExists(tx *txn, id wallet.ID) error { const query = `SELECT id FROM wallets WHERE id=$1` var dummyID int64 err := tx.QueryRow(query, id).Scan(&dummyID) diff --git a/wallet/manager.go b/wallet/manager.go index 63674b8..425f264 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -24,20 +24,20 @@ type ( Store interface { chain.Subscriber - WalletEvents(id WalletID, offset, limit int) ([]Event, error) + WalletEvents(walletID ID, offset, limit int) ([]Event, error) AddWallet(Wallet) (Wallet, error) UpdateWallet(Wallet) (Wallet, error) - DeleteWallet(id WalletID) error - WalletBalance(id WalletID) (Balance, error) - WalletSiacoinOutputs(id WalletID, offset, limit int) ([]types.SiacoinElement, error) - WalletSiafundOutputs(id WalletID, offset, limit int) ([]types.SiafundElement, error) - WalletAddresses(id WalletID) ([]Address, error) + DeleteWallet(walletID ID) error + WalletBalance(walletID ID) (Balance, error) + WalletSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, error) + WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) + WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) - AddWalletAddress(id WalletID, address Address) error - RemoveWalletAddress(id WalletID, address types.Address) error + AddWalletAddress(walletID ID, address Address) error + RemoveWalletAddress(walletID ID, address types.Address) error - Annotate(id WalletID, txns []types.Transaction) ([]PoolTransaction, error) + Annotate(walletID ID, txns []types.Transaction) ([]PoolTransaction, error) LastCommittedIndex() (types.ChainIndex, error) } @@ -64,8 +64,8 @@ func (m *Manager) UpdateWallet(w Wallet) (Wallet, error) { } // DeleteWallet deletes the given wallet. -func (m *Manager) DeleteWallet(id WalletID) error { - return m.store.DeleteWallet(id) +func (m *Manager) DeleteWallet(walletID ID) error { + return m.store.DeleteWallet(walletID) } // Wallets returns the wallets of the wallet manager. @@ -74,44 +74,44 @@ func (m *Manager) Wallets() ([]Wallet, error) { } // AddAddress adds the given address to the given wallet. -func (m *Manager) AddAddress(id WalletID, addr Address) error { - return m.store.AddWalletAddress(id, addr) +func (m *Manager) AddAddress(walletID ID, addr Address) error { + return m.store.AddWalletAddress(walletID, addr) } // RemoveAddress removes the given address from the given wallet. -func (m *Manager) RemoveAddress(id WalletID, addr types.Address) error { - return m.store.RemoveWalletAddress(id, addr) +func (m *Manager) RemoveAddress(walletID ID, addr types.Address) error { + return m.store.RemoveWalletAddress(walletID, addr) } // Addresses returns the addresses of the given wallet. -func (m *Manager) Addresses(id WalletID) ([]Address, error) { - return m.store.WalletAddresses(id) +func (m *Manager) Addresses(walletID ID) ([]Address, error) { + return m.store.WalletAddresses(walletID) } // Events returns the events of the given wallet. -func (m *Manager) Events(id WalletID, offset, limit int) ([]Event, error) { - return m.store.WalletEvents(id, offset, limit) +func (m *Manager) Events(walletID ID, offset, limit int) ([]Event, error) { + return m.store.WalletEvents(walletID, offset, limit) } // UnspentSiacoinOutputs returns a paginated list of unspent siacoin outputs of // the given wallet and the total number of unspent siacoin outputs. -func (m *Manager) UnspentSiacoinOutputs(id WalletID, offset, limit int) ([]types.SiacoinElement, error) { - return m.store.WalletSiacoinOutputs(id, offset, limit) +func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, error) { + return m.store.WalletSiacoinOutputs(walletID, offset, limit) } // UnspentSiafundOutputs returns the unspent siafund outputs of the given wallet -func (m *Manager) UnspentSiafundOutputs(id WalletID, offset, limit int) ([]types.SiafundElement, error) { - return m.store.WalletSiafundOutputs(id, offset, limit) +func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) { + return m.store.WalletSiafundOutputs(walletID, offset, limit) } // Annotate annotates the given transactions with the wallet they belong to. -func (m *Manager) Annotate(id WalletID, pool []types.Transaction) ([]PoolTransaction, error) { - return m.store.Annotate(id, pool) +func (m *Manager) Annotate(walletID ID, pool []types.Transaction) ([]PoolTransaction, error) { + return m.store.Annotate(walletID, pool) } // WalletBalance returns the balance of the given wallet. -func (m *Manager) WalletBalance(id WalletID) (Balance, error) { - return m.store.WalletBalance(id) +func (m *Manager) WalletBalance(walletID ID) (Balance, error) { + return m.store.WalletBalance(walletID) } // Reserve reserves the given ids for the given duration. diff --git a/wallet/wallet.go b/wallet/wallet.go index d266e6b..df76f48 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -28,12 +28,12 @@ type ( Siafunds uint64 `json:"siafunds"` } - // A WalletID is a unique identifier for a wallet. - WalletID int64 + // An ID is a unique identifier for a wallet. + ID int64 // A Wallet is a collection of addresses and metadata. Wallet struct { - ID WalletID `json:"id"` + ID ID `json:"id"` Name string `json:"name"` Description string `json:"description"` DateCreated time.Time `json:"dateCreated"` @@ -54,17 +54,17 @@ type ( var ErrNotFound = errors.New("not found") // UnmarshalText implements encoding.TextUnmarshaler. -func (w *WalletID) UnmarshalText(buf []byte) error { +func (w *ID) UnmarshalText(buf []byte) error { id, err := strconv.ParseInt(string(buf), 10, 64) if err != nil { return err } - *w = WalletID(id) + *w = ID(id) return nil } // MarshalText implements encoding.TextMarshaler. -func (w WalletID) MarshalText() ([]byte, error) { +func (w ID) MarshalText() ([]byte, error) { return []byte(strconv.FormatInt(int64(w), 10)), nil } From c15ed413ff9b1ef92ef6d24c6bf01408cc3e8408 Mon Sep 17 00:00:00 2001 From: alexfreska Date: Sun, 25 Feb 2024 00:17:19 +0000 Subject: [PATCH 108/630] ui: v0.17.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de2866e..dd884ad 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.16.0 + go.sia.tech/web/walletd v0.17.0 go.uber.org/zap v1.26.0 golang.org/x/term v0.6.0 lukechampine.com/flagg v1.1.1 diff --git a/go.sum b/go.sum index 6bff896..3e7e772 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.16.0 h1:tCERgjsz4orokM94kt7PH2tNweHdOwK5aoPsCXes5HM= -go.sia.tech/web/walletd v0.16.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web/walletd v0.17.0 h1:8k/m1L50LIylw1HYLlTuc3e4bYlx//qZ8xG4C/YNeA0= +go.sia.tech/web/walletd v0.17.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 7d1ea9f64f8cb8270b02e21f36af1daafeee9940 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 26 Feb 2024 10:13:47 -0800 Subject: [PATCH 109/630] docker: fix docker build error --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14294c5..ac7f0db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ -FROM docker.io/library/golang:1.20 AS builder +FROM docker.io/library/golang:1.21 AS builder WORKDIR /walletd COPY . . -# build -RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w' ./cmd/walletd + +# Enable CGO for sqlite3 support +ENV CGO_ENABLED=1 + +RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd FROM docker.io/library/alpine:3 LABEL maintainer="The Sia Foundation " \ From db79fbbdc1e29fed82d3a6b89db7087ced2d9692 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 27 Feb 2024 09:17:18 -0800 Subject: [PATCH 110/630] api: fix add wallet not decoding request --- api/api_test.go | 172 ++++++++++++++++++++++++++++++++++++++++++++++++ api/client.go | 3 +- api/server.go | 3 + 3 files changed, 176 insertions(+), 2 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index ba8dcda..c86245f 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,9 +1,13 @@ package api_test import ( + "encoding/hex" + "encoding/json" + "fmt" "net" "net/http" "path/filepath" + "reflect" "testing" "time" @@ -48,6 +52,172 @@ func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Cl return c, func() { l.Close() } } +func TestWalletAdd(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + c, shutdown := runServer(cm, nil, wm) + defer shutdown() + + checkWalletResponse := func(wr api.WalletUpdateRequest, w wallet.Wallet, isUpdate bool) error { + // check wallet + if w.Name != wr.Name { + return fmt.Errorf("expected wallet name to be %v, got %v", wr.Name, w.Name) + } else if w.Description != wr.Description { + return fmt.Errorf("expected wallet description to be %v, got %v", wr.Description, w.Description) + } else if w.DateCreated.After(time.Now()) { + return fmt.Errorf("expected wallet creation date to be in the past, got %v", w.DateCreated) + } else if isUpdate && w.DateCreated == w.LastUpdated { + return fmt.Errorf("expected wallet last updated date to be after creation %v, got %v", w.DateCreated, w.LastUpdated) + } + + if wr.Metadata == nil && string(w.Metadata) == "null" { // zero value encodes as "null" + return nil + } + + // check metadata + var am, bm map[string]any + if err := json.Unmarshal(wr.Metadata, &am); err != nil { + return fmt.Errorf("failed to unmarshal metadata a %q: %v", wr.Metadata, err) + } else if err := json.Unmarshal(w.Metadata, &bm); err != nil { + return fmt.Errorf("failed to unmarshal metadata b: %v", err) + } + + if !reflect.DeepEqual(am, bm) { // not perfect, but probably enough for this test + return fmt.Errorf("expected metadata to be equal %v, got %v", wr.Metadata, w.Metadata) + } + return nil + } + + checkWallet := func(wa, wb wallet.Wallet) error { + // check wallet + if wa.Name != wb.Name { + return fmt.Errorf("expected wallet name to be %v, got %v", wa.Name, wb.Name) + } else if wa.Description != wb.Description { + return fmt.Errorf("expected wallet description to be %v, got %v", wa.Description, wb.Description) + } else if wa.DateCreated.Unix() != wb.DateCreated.Unix() { + return fmt.Errorf("expected wallet creation date to be %v, got %v", wa.DateCreated, wb.DateCreated) + } else if wa.LastUpdated.Unix() != wb.LastUpdated.Unix() { + return fmt.Errorf("expected wallet last updated date to be %v, got %v", wa.LastUpdated, wb.LastUpdated) + } + + if wa.Metadata == nil && string(wb.Metadata) == "null" { // zero value encodes as "null" + return nil + } + + // check metadata + var am, bm map[string]any + if err := json.Unmarshal(wa.Metadata, &am); err != nil { + return fmt.Errorf("failed to unmarshal metadata a %q: %v", wa.Metadata, err) + } else if err := json.Unmarshal(wb.Metadata, &bm); err != nil { + return fmt.Errorf("failed to unmarshal metadata b %q: %v", wb.Metadata, err) + } + + if !reflect.DeepEqual(am, bm) { // not perfect, but probably enough for this test + return fmt.Errorf("expected metadata to be equal %v, got %v", wa.Metadata, wb.Metadata) + } + return nil + } + + tests := []struct { + Initial api.WalletUpdateRequest + Update api.WalletUpdateRequest + }{ + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12))}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12))}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "hello, world!"}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "goodbye, world!"}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "hello, world!", Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "goodbye, world!", Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + { + Initial: api.WalletUpdateRequest{Name: "constant name", Description: "constant description", Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: "constant name", Description: "constant description", Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + } + + var expectedWallets []wallet.Wallet + for i, test := range tests { + w, err := c.AddWallet(test.Initial) + if err != nil { + t.Fatal(err) + } else if err := checkWalletResponse(test.Initial, w, false); err != nil { + t.Fatalf("test %v: %v", i, err) + } + + expectedWallets = append(expectedWallets, w) + // check that the wallet was added + wallets, err := c.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != len(expectedWallets) { + t.Fatalf("test %v: expected %v wallets, got %v", i, len(expectedWallets), len(wallets)) + } + for j, w := range wallets { + if err := checkWallet(expectedWallets[j], w); err != nil { + t.Fatalf("test %v: wallet %v: %v", i, j, err) + } + } + + time.Sleep(time.Second) // ensure LastUpdated is different + + w, err = c.UpdateWallet(w.ID, test.Update) + if err != nil { + t.Fatal(err) + } else if err := checkWalletResponse(test.Update, w, true); err != nil { + t.Fatalf("test %v: %v", i, err) + } + + // check that the wallet was updated + expectedWallets[len(expectedWallets)-1] = w + wallets, err = c.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != len(expectedWallets) { + t.Fatalf("test %v: expected %v wallets, got %v", i, len(expectedWallets), len(wallets)) + } + for j, w := range wallets { + if err := checkWallet(expectedWallets[j], w); err != nil { + t.Fatalf("test %v: wallet %v: %v", i, j, err) + } + } + } +} + func TestWallet(t *testing.T) { log := zaptest.NewLogger(t) @@ -83,6 +253,8 @@ func TestWallet(t *testing.T) { w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) + } else if w.Name != "primary" { + t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) } wc := c.Wallet(w.ID) if err := c.Resubscribe(0); err != nil { diff --git a/api/client.go b/api/client.go index 04d0c90..5dcbc6c 100644 --- a/api/client.go +++ b/api/client.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "fmt" "time" @@ -82,7 +81,7 @@ func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { } // Wallets returns the set of tracked wallets. -func (c *Client) Wallets() (ws map[string]json.RawMessage, err error) { +func (c *Client) Wallets() (ws []wallet.Wallet, err error) { err = c.c.GET("/wallets", &ws) return } diff --git a/api/server.go b/api/server.go index f9f33fe..c417a26 100644 --- a/api/server.go +++ b/api/server.go @@ -179,6 +179,9 @@ func (s *server) walletsHandler(jc jape.Context) { func (s *server) walletsHandlerPOST(jc jape.Context) { var req WalletUpdateRequest + if jc.Decode(&req) != nil { + return + } w := wallet.Wallet{ Name: req.Name, Description: req.Description, From e2d643777eca05f696c3a267d082ea26e8a7535b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 27 Feb 2024 15:44:51 -0800 Subject: [PATCH 111/630] sqlite: fix duplicate elements during resubscribe, move balance updates into store. --- cmd/walletd/main.go | 2 +- persist/sqlite/consensus.go | 331 ++++++++++++++++++++++++++----- persist/sqlite/init.sql | 2 +- persist/sqlite/wallet_test.go | 120 +++++++++++- wallet/update.go | 356 +++++++--------------------------- 5 files changed, 478 insertions(+), 333 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index bc19658..acedcd8 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -189,7 +189,7 @@ func main() { consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg) // only log info messages to console unless stdout logging is enabled - consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.InfoLevel)) + consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.DebugLevel)) logger := zap.New(consoleCore, zap.AddCaller()) defer logger.Sync() // redirect stdlib log to zap diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index c110087..9d22d37 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -19,6 +19,11 @@ type updateTx struct { relevantAddresses map[types.Address]bool } +type addressRef struct { + ID int64 + Balance wallet.Balance +} + func scanStateElement(s scanner) (se types.StateElement, err error) { err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) return @@ -29,6 +34,11 @@ func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { return } +func scanAddress(s scanner) (ab addressRef, err error) { + err = s.Scan(&ab.ID, decode(&ab.Balance.Siacoins), decode(&ab.Balance.ImmatureSiacoins), &ab.Balance.Siafunds) + return +} + func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` rows, err := ut.tx.Query(query) @@ -125,151 +135,378 @@ func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, return } -func (ut *updateTx) UpdateBalances(balances []wallet.AddressBalance) error { - const query = `UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE sia_address=$4` - stmt, err := ut.tx.Prepare(query) +func (ut *updateTx) ApplyMatureSiacoinBalance(index types.ChainIndex) error { + const query = `SELECT se.address_id, se.siacoin_value +FROM siacoin_elements se +WHERE maturity_height=$1` + rows, err := ut.tx.Query(query, index.Height) + if err != nil { + return fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + balanceDelta := make(map[int64]types.Currency) + for rows.Next() { + var addressID int64 + var value types.Currency + + if err := rows.Scan(&addressID, decode(&value)); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) + } + balanceDelta[addressID] = balanceDelta[addressID].Add(value) + } + + getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } - defer stmt.Close() + defer getAddressBalanceStmt.Close() - for _, ab := range balances { - _, err := stmt.Exec(encode(ab.Balance.Siacoins), encode(ab.Balance.ImmatureSiacoins), ab.Balance.Siafunds, encode(ab.Address)) + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addressID, delta := range balanceDelta { + var balance, immatureBalance types.Currency + err := getAddressBalanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) if err != nil { - return fmt.Errorf("failed to execute statement: %w", err) + return fmt.Errorf("failed to get address balance: %w", err) + } + + balance = balance.Add(delta) + immatureBalance = immatureBalance.Sub(delta) + + res, err := updateAddressBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) } } return nil } -func (ut *updateTx) MaturedSiacoinElements(index types.ChainIndex) (elements []types.SiacoinElement, err error) { - const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, a.sia_address -FROM siacoin_elements se -INNER JOIN sia_addresses a ON (se.address_id=a.id) -WHERE maturity_height=$1` +func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { + const query = `SELECT se.address_id, se.siacoin_value + FROM siacoin_elements se + WHERE maturity_height=$1` rows, err := ut.tx.Query(query, index.Height) if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + return fmt.Errorf("failed to query siacoin elements: %w", err) } defer rows.Close() + balanceDelta := make(map[int64]types.Currency) for rows.Next() { - element, err := scanSiacoinElement(rows) - if err != nil { - return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + var addressID int64 + var value types.Currency + + if err := rows.Scan(&addressID, decode(&value)); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) } - elements = append(elements, element) + balanceDelta[addressID] = balanceDelta[addressID].Add(value) } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("failed to scan siacoin elements: %w", err) + + getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) } - return + defer getAddressBalanceStmt.Close() + + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addressID, delta := range balanceDelta { + var balance, immatureBalance types.Currency + err := getAddressBalanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) + if err != nil { + return fmt.Errorf("failed to get address balance: %w", err) + } + + balance = balance.Sub(delta) + immatureBalance = immatureBalance.Add(delta) + + res, err := updateAddressBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil } -func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement) error { +func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index types.ChainIndex) error { + if len(elements) == 0 { + return nil + } + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) + // ignore elements already in the database. + insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } defer insertStmt.Close() + balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - var addressID int64 - err = addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance } - _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addressID) - if err != nil { + var dummyID types.Hash256 + err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID).Scan(decode(&dummyID)) + if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element already exists + } else if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } + + // update the balance if the element does not exist + balance := balanceChanges[addrRef.ID] + if se.MaturityHeight <= index.Height { + balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) + } else { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(se.SiacoinOutput.Value) + } + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), encode(balance.ImmatureSiacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } } return nil } -func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinOutputID) error { +func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinElement, index types.ChainIndex) error { + if len(elements) == 0 { + return nil + } + + addrStmt, err := insertAddressStatement(ut.tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer stmt.Close() - for _, id := range elements { + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + var dummy types.Hash256 - err := stmt.QueryRow(encode(id)).Scan(decode(&dummy)) + err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) + if err != nil { + return fmt.Errorf("failed to delete element %q: %w", se.ID, err) + } + + balance := balanceChanges[addrRef.ID] + if se.MaturityHeight < index.Height { + balance.Siacoins = balance.Siacoins.Sub(se.SiacoinOutput.Value) + } else { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) + } + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), encode(balance.ImmatureSiacoins), addrID) if err != nil { - return fmt.Errorf("failed to delete element %q: %w", id, err) + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) } } return nil } -func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement) error { +func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index types.ChainIndex) error { + if len(elements) == 0 { + return nil + } + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6)`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer insertStmt.Close() + balanceChanges := make(map[types.Address]uint64) for _, se := range elements { - var addressID int64 - err = addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0).Scan(&addressID) + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[se.SiafundOutput.Address]; !ok { + balanceChanges[se.SiafundOutput.Address] = addrRef.Balance.Siafunds } - _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addressID) - if err != nil { + var dummy types.Hash256 + err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID).Scan(decode(&dummy)) + if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element already exists + } else if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } + balanceChanges[se.SiafundOutput.Address] += se.SiafundOutput.Value + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE sia_address=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addr, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, encode(addr)) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } } return nil } -func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundOutputID) error { +func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index types.ChainIndex) error { + addrStmt, err := insertAddressStatement(ut.tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer stmt.Close() - for _, id := range elements { + balanceChanges := make(map[types.Address]uint64) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[se.SiafundOutput.Address]; !ok { + balanceChanges[se.SiafundOutput.Address] = addrRef.Balance.Siafunds + } + var dummy types.Hash256 - err := stmt.QueryRow(encode(id)).Scan(decode(&dummy)) + err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) if err != nil { - return fmt.Errorf("failed to delete element %q: %w", id, err) + return fmt.Errorf("failed to delete element %q: %w", se.ID, err) + } + + if balanceChanges[se.SiafundOutput.Address] < se.SiafundOutput.Value { + panic("siafund balance cannot be negative") + } + balanceChanges[se.SiafundOutput.Address] -= se.SiafundOutput.Value + } + + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE sia_address=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addr, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, encode(addr)) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) } } return nil } func (ut *updateTx) AddEvents(events []wallet.Event) error { + if len(events) == 0 { + return nil + } + indexStmt, err := insertIndexStmt(ut.tx) if err != nil { return fmt.Errorf("failed to prepare index statement: %w", err) } defer indexStmt.Close() - eventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`) + insertEventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (event_id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare event statement: %w", err) } - defer eventStmt.Close() + defer insertEventStmt.Close() - addrStmt, err := insertAddressStatement(ut.tx) + addrStmt, err := ut.tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } @@ -296,8 +533,10 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { } var eventID int64 - err = eventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), chainIndexID, event.Data.EventType(), buf.String()).Scan(&eventID) - if err != nil { + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), chainIndexID, event.Data.EventType(), buf.String()).Scan(&eventID) + if errors.Is(err, sql.ErrNoRows) { + continue // skip if the event already exists + } else if err != nil { return fmt.Errorf("failed to add event: %w", err) } @@ -400,7 +639,7 @@ func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { } func insertAddressStatement(tx *txn) (*stmt, error) { - return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) + return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) } func insertIndexStmt(tx *txn) (*stmt, error) { diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 7d12cc0..6a9ee54 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -55,7 +55,7 @@ CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); CREATE TABLE events ( id INTEGER PRIMARY KEY, - event_id BLOB NOT NULL, + event_id BLOB UNIQUE NOT NULL, index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index 5c1e960..08465f0 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -1,4 +1,4 @@ -package sqlite +package sqlite_test import ( "encoding/json" @@ -6,13 +6,16 @@ import ( "testing" "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" "go.uber.org/zap/zaptest" ) func TestWalletAddresses(t *testing.T) { log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } @@ -104,3 +107,116 @@ func TestWalletAddresses(t *testing.T) { t.Fatal("expected 0 addresses, got", len(addresses)) } } + +func TestResubscribe(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + cm := chain.NewManager(store, genesisState) + + if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // check that the payout was received + balance, err := db.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := db.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + cm.RemoveSubscriber(db) + if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the balance, events, and utxos did not change + // check that the payout was received + balance, err = db.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err = db.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } +} diff --git a/wallet/update.go b/wallet/update.go index f9a140a..2890d3c 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -22,23 +22,20 @@ type ( SiafundStateElements() ([]types.StateElement, error) UpdateSiafundStateElements([]types.StateElement) error - AddSiacoinElements([]types.SiacoinElement) error - RemoveSiacoinElements([]types.SiacoinOutputID) error + AddSiacoinElements([]types.SiacoinElement, types.ChainIndex) error + RemoveSiacoinElements([]types.SiacoinElement, types.ChainIndex) error - AddSiafundElements([]types.SiafundElement) error - RemoveSiafundElements([]types.SiafundOutputID) error - - MaturedSiacoinElements(types.ChainIndex) ([]types.SiacoinElement, error) + AddSiafundElements([]types.SiafundElement, types.ChainIndex) error + RemoveSiafundElements([]types.SiafundElement, types.ChainIndex) error AddressRelevant(types.Address) (bool, error) - AddressBalance(types.Address) (Balance, error) - UpdateBalances([]AddressBalance) error } // An ApplyTx atomically applies a set of updates to a store. ApplyTx interface { UpdateTx + ApplyMatureSiacoinBalance(types.ChainIndex) error AddEvents([]Event) error } @@ -46,59 +43,18 @@ type ( RevertTx interface { UpdateTx + RevertMatureSiacoinBalance(types.ChainIndex) error RevertEvents(index types.ChainIndex) error } ) // ApplyChainUpdates atomically applies a set of chain updates to a store func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { - var events []Event - balances := make(map[types.Address]Balance) - newSiacoinElements := make(map[types.SiacoinOutputID]types.SiacoinElement) - newSiafundElements := make(map[types.SiafundOutputID]types.SiafundElement) - spentSiacoinElements := make(map[types.SiacoinOutputID]bool) - spentSiafundElements := make(map[types.SiafundOutputID]bool) - - updateBalance := func(addr types.Address, fn func(b *Balance)) error { - balance, ok := balances[addr] - if !ok { - var err error - balance, err = tx.AddressBalance(addr) - if err != nil { - return fmt.Errorf("failed to get address balance: %w", err) - } - } - - fn(&balance) - balances[addr] = balance - return nil - } - - // fetch all siacoin and siafund state elements - siacoinStateElements, err := tx.SiacoinStateElements() - if err != nil { - return fmt.Errorf("failed to get siacoin state elements: %w", err) - } - siafundStateElements, err := tx.SiafundStateElements() - if err != nil { - return fmt.Errorf("failed to get siafund state elements: %w", err) - } - for _, cau := range updates { // update the immature balance of each relevant address - matured, err := tx.MaturedSiacoinElements(cau.State.Index) - if err != nil { + if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { return fmt.Errorf("failed to get matured siacoin elements: %w", err) } - for _, se := range matured { - err := updateBalance(se.SiacoinOutput.Address, func(b *Balance) { - b.ImmatureSiacoins = b.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) - b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) - }) - if err != nil { - return fmt.Errorf("failed to update address balance: %w", err) - } - } // determine which siacoin and siafund elements are ephemeral // @@ -122,87 +78,58 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { } // add new siacoin elements to the store - var siacoinElementErr error + var newSiacoinElements, spentSiacoinElements []types.SiacoinElement cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if siacoinElementErr != nil { - return - } else if ephemeral[se.ID] { + if ephemeral[se.ID] { return } relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) if err != nil { - siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) - return + panic(err) } else if !relevant { return } if spent { - delete(newSiacoinElements, types.SiacoinOutputID(se.ID)) - spentSiacoinElements[types.SiacoinOutputID(se.ID)] = true + spentSiacoinElements = append(spentSiacoinElements, se) } else { - newSiacoinElements[types.SiacoinOutputID(se.ID)] = se - } - - err = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { - switch { - case se.MaturityHeight > cau.State.Index.Height: - b.ImmatureSiacoins = b.ImmatureSiacoins.Add(se.SiacoinOutput.Value) - case spent: - b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) - default: - b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) - } - }) - if err != nil { - siacoinElementErr = fmt.Errorf("failed to update address balance: %w", err) - return + newSiacoinElements = append(newSiacoinElements, se) } }) - if siacoinElementErr != nil { - return fmt.Errorf("failed to add siacoin elements: %w", siacoinElementErr) + + if err := tx.AddSiacoinElements(newSiacoinElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } else if err := tx.RemoveSiacoinElements(spentSiacoinElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) } - var siafundElementErr error + var newSiafundElements, spentSiafundElements []types.SiafundElement cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if siafundElementErr != nil { - return - } else if ephemeral[se.ID] { + if ephemeral[se.ID] { return } relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) if err != nil { - siafundElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) - return + panic(err) } else if !relevant { return } if spent { - delete(newSiafundElements, types.SiafundOutputID(se.ID)) - spentSiafundElements[types.SiafundOutputID(se.ID)] = true + spentSiafundElements = append(spentSiafundElements, se) } else { - newSiafundElements[types.SiafundOutputID(se.ID)] = se - } - - err = updateBalance(se.SiafundOutput.Address, func(b *Balance) { - if spent { - if b.Siafunds < se.SiafundOutput.Value { - panic(fmt.Errorf("negative siafund balance")) - } - b.Siafunds -= se.SiafundOutput.Value - } else { - b.Siafunds += se.SiafundOutput.Value - } - }) - if err != nil { - siafundElementErr = fmt.Errorf("failed to update address balance: %w", err) - return + newSiafundElements = append(newSiafundElements, se) } }) + if err := tx.AddSiafundElements(newSiafundElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } else if err := tx.RemoveSiafundElements(spentSiafundElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + // add events relevant := func(addr types.Address) bool { relevant, err := tx.AddressRelevant(addr) @@ -211,135 +138,44 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { } return relevant } + if err := tx.AddEvents(AppliedEvents(cau.State, cau.Block, cau, relevant)); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + + // fetch all siacoin and siafund state elements + siacoinStateElements, err := tx.SiacoinStateElements() if err != nil { - return fmt.Errorf("failed to get applied events: %w", err) + return fmt.Errorf("failed to get siacoin state elements: %w", err) } - events = append(events, AppliedEvents(cau.State, cau.Block, cau, relevant)...) // update siacoin element proofs - for id := range newSiacoinElements { - ele := newSiacoinElements[id] - cau.UpdateElementProof(&ele.StateElement) - newSiacoinElements[id] = ele - } for i := range siacoinStateElements { cau.UpdateElementProof(&siacoinStateElements[i]) } - // update siafund element proofs - for id := range newSiafundElements { - ele := newSiafundElements[id] - cau.UpdateElementProof(&ele.StateElement) - newSiafundElements[id] = ele + if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) } - for i := range siafundStateElements { - cau.UpdateElementProof(&siafundStateElements[i]) - } - } - - // update the address balances - balanceChanges := make([]AddressBalance, 0, len(balances)) - for addr, balance := range balances { - balanceChanges = append(balanceChanges, AddressBalance{ - Address: addr, - Balance: balance, - }) - } - if err = tx.UpdateBalances(balanceChanges); err != nil { - return fmt.Errorf("failed to update address balance: %w", err) - } - // add the new siacoin elements - siacoinElements := make([]types.SiacoinElement, 0, len(newSiacoinElements)) - for _, ele := range newSiacoinElements { - siacoinElements = append(siacoinElements, ele) - } - if err = tx.AddSiacoinElements(siacoinElements); err != nil { - return fmt.Errorf("failed to add siacoin elements: %w", err) - } - - // remove the spent siacoin elements - siacoinOutputIDs := make([]types.SiacoinOutputID, 0, len(spentSiacoinElements)) - for id := range spentSiacoinElements { - siacoinOutputIDs = append(siacoinOutputIDs, id) - } - if err = tx.RemoveSiacoinElements(siacoinOutputIDs); err != nil { - return fmt.Errorf("failed to remove siacoin elements: %w", err) - } - - // add the new siafund elements - siafundElements := make([]types.SiafundElement, 0, len(newSiafundElements)) - for _, ele := range newSiafundElements { - siafundElements = append(siafundElements, ele) - } - if err = tx.AddSiafundElements(siafundElements); err != nil { - return fmt.Errorf("failed to add siafund elements: %w", err) - } - - // remove the spent siafund elements - siafundOutputIDs := make([]types.SiafundOutputID, 0, len(spentSiafundElements)) - for id := range spentSiafundElements { - siafundOutputIDs = append(siafundOutputIDs, id) - } - if err = tx.RemoveSiafundElements(siafundOutputIDs); err != nil { - return fmt.Errorf("failed to remove siafund elements: %w", err) - } - - // add new events - if err = tx.AddEvents(events); err != nil { - return fmt.Errorf("failed to add events: %w", err) - } + siafundStateElements, err := tx.SiafundStateElements() + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } - // update the siacoin state elements - filteredStateElements := siacoinStateElements[:0] - for _, se := range siacoinStateElements { - if _, ok := spentSiacoinElements[types.SiacoinOutputID(se.ID)]; !ok { - filteredStateElements = append(filteredStateElements, se) + // update siafund element proofs + for i := range siafundStateElements { + cau.UpdateElementProof(&siafundStateElements[i]) } - } - err = tx.UpdateSiacoinStateElements(filteredStateElements) - if err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) - } - // update the siafund state elements - filteredStateElements = siafundStateElements[:0] - for _, se := range siafundStateElements { - if _, ok := spentSiafundElements[types.SiafundOutputID(se.ID)]; !ok { - filteredStateElements = append(filteredStateElements, se) + if err := tx.UpdateSiafundStateElements(siafundStateElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) } } - if err = tx.UpdateSiafundStateElements(filteredStateElements); err != nil { - return fmt.Errorf("failed to update siafund state elements: %w", err) - } - return nil } // RevertChainUpdate atomically reverts a chain update from a store func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { - balances := make(map[types.Address]Balance) - - var deletedSiacoinElements []types.SiacoinOutputID - var addedSiacoinElements []types.SiacoinElement - var deletedSiafundElements []types.SiafundOutputID - var addedSiafundElements []types.SiafundElement - - updateBalance := func(addr types.Address, fn func(b *Balance)) error { - balance, ok := balances[addr] - if !ok { - var err error - balance, err = tx.AddressBalance(addr) - if err != nil { - return fmt.Errorf("failed to get address balance: %w", err) - } - } - - fn(&balance) - balances[addr] = balance - return nil - } - // determine which siacoin and siafund elements are ephemeral // // note: I thought we could use LeafIndex == EphemeralLeafIndex, but @@ -367,34 +203,17 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { ID: cru.Block.ID(), } - matured, err := tx.MaturedSiacoinElements(revertedIndex) - if err != nil { - return fmt.Errorf("failed to get matured siacoin elements: %w", err) - } - for _, se := range matured { - err := updateBalance(se.SiacoinOutput.Address, func(b *Balance) { - b.ImmatureSiacoins = b.ImmatureSiacoins.Add(se.SiacoinOutput.Value) - b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) - }) - if err != nil { - return fmt.Errorf("failed to update address balance: %w", err) - } - } - - var siacoinElementErr error + var removedSiacoinElements, addedSiacoinElements []types.SiacoinElement cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if siacoinElementErr != nil { + if ephemeral[se.ID] { return } relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) if err != nil { - siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) - return + panic(err) } else if !relevant { return - } else if ephemeral[se.ID] { - return } if spent { @@ -402,38 +221,27 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { addedSiacoinElements = append(addedSiacoinElements, se) } else { // delete any created siacoin elements - deletedSiacoinElements = append(deletedSiacoinElements, types.SiacoinOutputID(se.ID)) + removedSiacoinElements = append(removedSiacoinElements, se) } - - siacoinElementErr = updateBalance(se.SiacoinOutput.Address, func(b *Balance) { - switch { - case se.MaturityHeight > cru.State.Index.Height: - b.ImmatureSiacoins = b.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) - case spent: - b.Siacoins = b.Siacoins.Add(se.SiacoinOutput.Value) - default: - b.Siacoins = b.Siacoins.Sub(se.SiacoinOutput.Value) - } - }) }) - if siacoinElementErr != nil { - return fmt.Errorf("failed to update address balance: %w", siacoinElementErr) + + if err := tx.AddSiacoinElements(addedSiacoinElements, revertedIndex); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } else if err := tx.RemoveSiacoinElements(removedSiacoinElements, revertedIndex); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) } - var siafundElementErr error + var removedSiafundElements, addedSiafundElements []types.SiafundElement cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if siafundElementErr != nil { + if ephemeral[se.ID] { return } relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) if err != nil { - siacoinElementErr = fmt.Errorf("failed to check if address is relevant: %w", err) - return + panic(err) } else if !relevant { return - } else if ephemeral[se.ID] { - return } if spent { @@ -441,40 +249,22 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { addedSiafundElements = append(addedSiafundElements, se) } else { // delete any created siafund elements - deletedSiafundElements = append(deletedSiafundElements, types.SiafundOutputID(se.ID)) + removedSiafundElements = append(removedSiafundElements, se) } - - siafundElementErr = updateBalance(se.SiafundOutput.Address, func(b *Balance) { - if spent { - b.Siafunds += se.SiafundOutput.Value - } else { - b.Siafunds -= se.SiafundOutput.Value - } - }) }) - if siafundElementErr != nil { - return fmt.Errorf("failed to update address balance: %w", siafundElementErr) - } - balanceChanges := make([]AddressBalance, 0, len(balances)) - for addr, balance := range balances { - balanceChanges = append(balanceChanges, AddressBalance{ - Address: addr, - Balance: balance, - }) - } - if err := tx.UpdateBalances(balanceChanges); err != nil { - return fmt.Errorf("failed to update address balance: %w", err) + // revert siafund element changes + if err := tx.AddSiafundElements(addedSiafundElements, revertedIndex); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } else if err := tx.RemoveSiafundElements(removedSiafundElements, revertedIndex); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) } - // revert siacoin element changes - if err := tx.AddSiacoinElements(addedSiacoinElements); err != nil { - return fmt.Errorf("failed to add siacoin elements: %w", err) - } else if err := tx.RemoveSiacoinElements(deletedSiacoinElements); err != nil { - return fmt.Errorf("failed to remove siacoin elements: %w", err) + // revert mature siacoin balance for each relevant address + if err := tx.RevertMatureSiacoinBalance(revertedIndex); err != nil { + return fmt.Errorf("failed to get matured siacoin elements: %w", err) } - // update siacoin element proofs siacoinElements, err := tx.SiacoinStateElements() if err != nil { return fmt.Errorf("failed to get siacoin state elements: %w", err) @@ -482,12 +272,8 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { for i := range siacoinElements { cru.UpdateElementProof(&siacoinElements[i]) } - - // revert siafund element changes - if err := tx.AddSiafundElements(addedSiafundElements); err != nil { - return fmt.Errorf("failed to add siafund elements: %w", err) - } else if err := tx.RemoveSiafundElements(deletedSiafundElements); err != nil { - return fmt.Errorf("failed to remove siafund elements: %w", err) + if err := tx.UpdateSiacoinStateElements(siacoinElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) } // update siafund element proofs @@ -498,6 +284,10 @@ func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { for i := range siafundElements { cru.UpdateElementProof(&siafundElements[i]) } + if err := tx.UpdateSiafundStateElements(siafundElements); err != nil { + return fmt.Errorf("failed to update siafund state elements: %w", err) + } + // revert events return tx.RevertEvents(revertedIndex) } From b9279f998068a2e55c53340533df15c7b29eb136 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 27 Feb 2024 16:00:09 -0800 Subject: [PATCH 112/630] sqlite: set siafund address, fix siafund balance update --- persist/sqlite/consensus.go | 34 ++++++++++++++++++-------------- persist/sqlite/consensus_test.go | 30 +++++++++++++++------------- persist/sqlite/wallet_test.go | 8 ++++---- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9d22d37..7d96a4c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -394,13 +394,13 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } defer insertStmt.Close() - balanceChanges := make(map[types.Address]uint64) + balanceChanges := make(map[int64]uint64) for _, se := range elements { addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) - } else if _, ok := balanceChanges[se.SiafundOutput.Address]; !ok { - balanceChanges[se.SiafundOutput.Address] = addrRef.Balance.Siafunds + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance.Siafunds } var dummy types.Hash256 @@ -410,21 +410,21 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } else if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } - balanceChanges[se.SiafundOutput.Address] += se.SiafundOutput.Value + balanceChanges[addrRef.ID] += se.SiafundOutput.Value } if len(balanceChanges) == 0 { return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE sia_address=$2`) + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } defer updateAddressBalanceStmt.Close() - for addr, balance := range balanceChanges { - res, err := updateAddressBalanceStmt.Exec(balance, encode(addr)) + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, addrID) if err != nil { return fmt.Errorf("failed to update balance: %w", err) } else if n, err := res.RowsAffected(); err != nil { @@ -449,13 +449,13 @@ func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index } defer stmt.Close() - balanceChanges := make(map[types.Address]uint64) + balanceChanges := make(map[int64]uint64) for _, se := range elements { addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) - } else if _, ok := balanceChanges[se.SiafundOutput.Address]; !ok { - balanceChanges[se.SiafundOutput.Address] = addrRef.Balance.Siafunds + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance.Siafunds } var dummy types.Hash256 @@ -464,20 +464,24 @@ func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index return fmt.Errorf("failed to delete element %q: %w", se.ID, err) } - if balanceChanges[se.SiafundOutput.Address] < se.SiafundOutput.Value { + if balanceChanges[addrRef.ID] < se.SiafundOutput.Value { panic("siafund balance cannot be negative") } - balanceChanges[se.SiafundOutput.Address] -= se.SiafundOutput.Value + balanceChanges[addrRef.ID] -= se.SiafundOutput.Value + } + + if len(balanceChanges) == 0 { + return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE sia_address=$2`) + updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } defer updateAddressBalanceStmt.Close() - for addr, balance := range balanceChanges { - res, err := updateAddressBalanceStmt.Exec(balance, encode(addr)) + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, addrID) if err != nil { return fmt.Errorf("failed to update balance: %w", err) } else if n, err := res.RowsAffected(); err != nil { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 8462428..1892c57 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -13,9 +13,10 @@ import ( "go.uber.org/zap/zaptest" ) -func testV1Network() (*consensus.Network, types.Block) { +func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { // use a modified version of Zen n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 @@ -28,9 +29,10 @@ func testV1Network() (*consensus.Network, types.Block) { return n, genesisBlock } -func testV2Network() (*consensus.Network, types.Block) { +func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) { // use a modified version of Zen n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 @@ -75,6 +77,9 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty } func TestReorg(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + log := zaptest.NewLogger(t) dir := t.TempDir() db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) @@ -89,7 +94,7 @@ func TestReorg(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network() + network, genesisBlock := testV1Network(addr) store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -103,9 +108,6 @@ func TestReorg(t *testing.T) { t.Fatal(err) } - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -280,6 +282,9 @@ func TestReorg(t *testing.T) { } func TestEphemeralBalance(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + log := zaptest.NewLogger(t) dir := t.TempDir() db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) @@ -294,7 +299,7 @@ func TestEphemeralBalance(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network() + network, genesisBlock := testV1Network(addr) store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -308,9 +313,6 @@ func TestEphemeralBalance(t *testing.T) { t.Fatal(err) } - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -475,6 +477,9 @@ func TestEphemeralBalance(t *testing.T) { } func TestV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + log := zaptest.NewLogger(t) dir := t.TempDir() db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) @@ -489,7 +494,7 @@ func TestV2(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV2Network() + network, genesisBlock := testV2Network(addr) store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -503,9 +508,6 @@ func TestV2(t *testing.T) { t.Fatal(err) } - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index 08465f0..20db201 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -109,6 +109,9 @@ func TestWalletAddresses(t *testing.T) { } func TestResubscribe(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + log := zaptest.NewLogger(t) dir := t.TempDir() db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) @@ -123,7 +126,7 @@ func TestResubscribe(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network() + network, genesisBlock := testV1Network(types.VoidAddress) store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -137,9 +140,6 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) From 03c1d048a98c4c21df5e2f47751868846302807f Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 28 Feb 2024 09:47:36 -0800 Subject: [PATCH 113/630] wallet: remove check for relevant contracts --- wallet/wallet.go | 88 ++++++------------------------------------------ 1 file changed, 10 insertions(+), 78 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index df76f48..9d182ce 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -366,38 +366,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f }) } - // do a first pass to see if there's anything relevant in the block - relevantContract := func(fc types.FileContract) (addrs []types.Address) { - for _, sco := range fc.ValidProofOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) - } - } - for _, sco := range fc.MissedProofOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) - } - } - return - } - relevantV2Contract := func(fc types.V2FileContract) (addrs []types.Address) { - if relevant(fc.RenterOutput.Address) { - addrs = append(addrs, fc.RenterOutput.Address) - } - if relevant(fc.HostOutput.Address) { - addrs = append(addrs, fc.HostOutput.Address) - } - return - } - relevantV2ContractResolution := func(res types.V2FileContractResolutionType) (addrs []types.Address) { - switch r := res.(type) { - case *types.V2FileContractFinalization: - return relevantV2Contract(types.V2FileContract(*r)) - case *types.V2FileContractRenewal: - return append(relevantV2Contract(r.InitialRevision), relevantV2Contract(r.FinalRevision)...) - } - return - } anythingRelevant := func() (ok bool) { cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { if ok || relevant(sce.SiacoinOutput.Address) { @@ -409,19 +377,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f ok = true } }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if ok || len(relevantContract(fce.FileContract)) > 0 || (rev != nil && len(relevantContract(rev.FileContract)) > 0) { - ok = true - } - }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - if ok || - len(relevantV2Contract(fce.V2FileContract)) > 0 || - (rev != nil && len(relevantV2Contract(rev.V2FileContract)) > 0) || - (res != nil && len(relevantV2ContractResolution(res)) > 0) { - ok = true - } - }) return }() if !anythingRelevant { @@ -471,15 +426,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f addrs = append(addrs, sfo.Address) } } - for _, fc := range txn.FileContracts { - addrs = append(addrs, relevantContract(fc)...) - } - for _, fcr := range txn.FileContractRevisions { - addrs = append(addrs, relevantContract(fcr.FileContract)...) - } - for _, sp := range txn.StorageProofs { - addrs = append(addrs, relevantContract(fces[sp.ParentID].FileContract)...) - } return } @@ -504,23 +450,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f addrs = append(addrs, sfo.Address) } } - for _, fc := range txn.FileContracts { - addrs = append(addrs, relevantV2Contract(fc)...) - } - for _, fcr := range txn.FileContractRevisions { - addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) - addrs = append(addrs, relevantV2Contract(fcr.Revision)...) - } - for _, fcr := range txn.FileContractResolutions { - addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) - switch r := fcr.Resolution.(type) { - case *types.V2FileContractFinalization: - addrs = append(addrs, relevantV2Contract(types.V2FileContract(*r))...) - case *types.V2FileContractRenewal: - addrs = append(addrs, relevantV2Contract(r.InitialRevision)...) - addrs = append(addrs, relevantV2Contract(r.FinalRevision)...) - } - } return } @@ -675,28 +604,31 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return } - relevant := relevantContract(fce.FileContract) - if len(relevant) == 0 { - return - } - if valid { for i := range fce.FileContract.ValidProofOutputs { + if !relevant(fce.FileContract.ValidProofOutputs[i].Address) { + continue + } + outputID := types.FileContractID(fce.ID).ValidOutputID(i) addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ FileContract: fce, SiacoinOutput: sces[outputID], Missed: false, - }, relevant) + }, []types.Address{fce.FileContract.ValidProofOutputs[i].Address}) } } else { for i := range fce.FileContract.MissedProofOutputs { + if !relevant(fce.FileContract.ValidProofOutputs[i].Address) { + continue + } + outputID := types.FileContractID(fce.ID).MissedOutputID(i) addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ FileContract: fce, SiacoinOutput: sces[outputID], Missed: true, - }, relevant) + }, []types.Address{fce.FileContract.ValidProofOutputs[i].Address}) } } }) From f8da865b2c237378cd53b62eeb9eb0a9fb2d719a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 7 Mar 2024 13:25:43 +0100 Subject: [PATCH 114/630] ci: add dependabot --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..cd88554 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" From db2b2839244f31824cb46ccff383568e9df5d39f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:42:02 +0000 Subject: [PATCH 115/630] build(deps): bump github.com/mattn/go-sqlite3 from 1.14.21 to 1.14.22 Bumps [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) from 1.14.21 to 1.14.22. - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.21...v1.14.22) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd884ad..f80ec28 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd go 1.21 require ( - github.com/mattn/go-sqlite3 v1.14.21 + github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 diff --git a/go.sum b/go.sum index 3e7e772..06d276f 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/mattn/go-sqlite3 v1.14.21 h1:IXocQLOykluc3xPE0Lvy8FtggMz1G+U3mEjg+0zGizc= -github.com/mattn/go-sqlite3 v1.14.21/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= From 22d66494b028e14ada220df5993bbe9df4230d54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:42:06 +0000 Subject: [PATCH 116/630] build(deps): bump golang.org/x/term from 0.6.0 to 0.18.0 Bumps [golang.org/x/term](https://github.com/golang/term) from 0.6.0 to 0.18.0. - [Commits](https://github.com/golang/term/compare/v0.6.0...v0.18.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index dd884ad..7cc3ce5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.17.0 go.uber.org/zap v1.26.0 - golang.org/x/term v0.6.0 + golang.org/x/term v0.18.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 @@ -23,6 +23,6 @@ require ( go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect - golang.org/x/sys v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 3e7e772..dc8a42f 100644 --- a/go.sum +++ b/go.sum @@ -35,10 +35,10 @@ golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 5623dcf6f7e66e063f4d3fba995b4228133cfffd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:53:49 +0000 Subject: [PATCH 117/630] build(deps): bump go.uber.org/zap from 1.26.0 to 1.27.0 Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.26.0 to 1.27.0. - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.26.0...v1.27.0) --- updated-dependencies: - dependency-name: go.uber.org/zap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index ad6dab8..70d7041 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.17.0 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 golang.org/x/term v0.18.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 diff --git a/go.sum b/go.sum index afc541c..34b4773 100644 --- a/go.sum +++ b/go.sum @@ -24,12 +24,12 @@ go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= go.sia.tech/web/walletd v0.17.0 h1:8k/m1L50LIylw1HYLlTuc3e4bYlx//qZ8xG4C/YNeA0= go.sia.tech/web/walletd v0.17.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= From 9ec0a2394fbf853374eddf858f261b19bdf05c10 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 7 Mar 2024 10:52:56 -0800 Subject: [PATCH 118/630] api,sqlite,wallet: add single address endpoints --- api/api_test.go | 193 ++++++++++++++++++++++++++++++++++++ api/client.go | 24 +++++ api/server.go | 76 ++++++++++++++ persist/sqlite/addresses.go | 59 ++++++++++- wallet/addresses.go | 23 +++++ wallet/manager.go | 5 + 6 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 wallet/addresses.go diff --git a/api/api_test.go b/api/api_test.go index c86245f..064c452 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -411,6 +411,199 @@ func TestWallet(t *testing.T) { } } +func TestAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) + c, shutdown := runServer(cm, nil, wm) + defer shutdown() + w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { + t.Fatal(err) + } else if w.Name != "primary" { + t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) + } + wc := c.Wallet(w.ID) + if err := c.Resubscribe(0); err != nil { + t.Fatal(err) + } + + balance, err := wc.Balance() + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { + t.Fatal("balance should be 0") + } + + // shouldn't have any events yet + events, err := wc.Events(0, -1) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatal("event history should be empty") + } + + // shouldn't have any addresses yet + addresses, err := wc.Addresses() + if err != nil { + t.Fatal(err) + } else if len(addresses) != 0 { + t.Fatal("address list should be empty") + } + + // create and add an address + addr := sav.NewAddress("primary") + if err := wc.AddAddress(addr); err != nil { + t.Fatal(err) + } + + // should have an address now + addresses, err = wc.Addresses() + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("address list should have one address") + } else if addresses[0].Address != addr.Address { + t.Fatalf("address should be %v, got %v", addr, addresses[0]) + } + + // send gift to wallet + giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: giftSCOID, + UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, + {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, + }, + Signatures: []types.TransactionSignature{{ + ParentID: types.Hash256(giftSCOID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + + cs := cm.TipState() + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + Transactions: []types.Transaction{txn}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // get new balance + balance, err = c.AddressBalance(addr.Address) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + } + + // transaction should appear in history + events, err = c.AddressEvents(addr.Address, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) == 0 { + t.Error("transaction should appear in history") + } + + outputs, err := c.AddressSiacoinOutputs(addr.Address, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(outputs) != 2 { + t.Error("should have two UTXOs, got", len(outputs)) + } + + // mine a block to add an immature balance + cs = cm.TipState() + b = types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: addr.Address, Value: cs.BlockReward()}}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // get new balance + balance, err = c.AddressBalance(addr.Address) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Error("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { + t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + } + + // mine enough blocks for the miner payout to mature + expectedBalance := types.Siacoins(1).Add(b.MinerPayouts[0].Value) + target := cs.MaturityHeight() + for cs.Index.Height < target { + cs = cm.TipState() + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + // get new balance + balance, err = c.AddressBalance(addr.Address) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedBalance) { + t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + } +} + func TestV2(t *testing.T) { log := zaptest.NewLogger(t) diff --git a/api/client.go b/api/client.go index 5dcbc6c..1e614d3 100644 --- a/api/client.go +++ b/api/client.go @@ -117,6 +117,30 @@ func (c *Client) Resubscribe(height uint64) (err error) { return } +// AddressBalance returns the balance of a single address. +func (c *Client) AddressBalance(addr types.Address) (resp BalanceResponse, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/balance", addr), &resp) + return +} + +// AddressEvents returns the events of a single address. +func (c *Client) AddressEvents(addr types.Address, offset, limit int) (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/events?offset=%d&limit=%d", addr, offset, limit), &resp) + return +} + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) (resp []types.SiacoinElement, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) + return +} + +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) (resp []types.SiafundElement, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) + return +} + // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { diff --git a/api/server.go b/api/server.go index c417a26..c228676 100644 --- a/api/server.go +++ b/api/server.go @@ -60,6 +60,11 @@ type ( WalletBalance(id wallet.ID) (wallet.Balance, error) Annotate(id wallet.ID, pool []types.Transaction) ([]wallet.PoolTransaction, error) + AddressBalance(address types.Address) (balance wallet.Balance, err error) + AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) + AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) + AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + Reserve(ids []types.Hash256, duration time.Duration) error } ) @@ -544,6 +549,72 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { }) } +func (s *server) addressesAddrBalanceHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + b, err := s.wm.AddressBalance(addr) + if jc.Check("couldn't load balance", err) != nil { + return + } + jc.Encode(BalanceResponse(b)) +} + +func (s *server) addressesAddrEventsHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + events, err := s.wm.AddressEvents(addr, offset, limit) + if jc.Check("couldn't load events", err) != nil { + return + } + jc.Encode(events) +} + +func (s *server) addressesAddrOutputsSCHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, err := s.wm.AddressSiacoinOutputs(addr, offset, limit) + if jc.Check("couldn't load utxos", err) != nil { + return + } + jc.Encode(utxos) +} + +func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, err := s.wm.AddressSiafundOutputs(addr, offset, limit) + if jc.Check("couldn't load utxos", err) != nil { + return + } + jc.Encode(utxos) +} + // NewServer returns an HTTP handler that serves the walletd API. func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { srv := server{ @@ -583,5 +654,10 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "POST /wallets/:id/release": srv.walletsReleaseHandler, "POST /wallets/:id/fund": srv.walletsFundHandler, "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, + + "GET /addresses/:addr/balance": srv.addressesAddrBalanceHandler, + "GET /addresses/:addr/events": srv.addressesAddrEventsHandler, + "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, + "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, }) } diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index daf4cd0..8d45a72 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -17,7 +17,7 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e } // AddressEvents returns the events of a single address. -func (s *Store) AddressEvents(address types.Address, limit, offset int) (events []wallet.Event, err error) { +func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev @@ -46,3 +46,60 @@ func (s *Store) AddressEvents(address types.Address, limit, offset int) (events }) return } + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address=$1 + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var siacoin types.SiacoinElement + err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), decode(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + + siacoins = append(siacoins, siacoin) + } + return rows.Err() + }) + return +} + +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address = $1 + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var siafund types.SiafundElement + err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + siafunds = append(siafunds, siafund) + } + return rows.Err() + }) + return +} diff --git a/wallet/addresses.go b/wallet/addresses.go new file mode 100644 index 0000000..85e22b1 --- /dev/null +++ b/wallet/addresses.go @@ -0,0 +1,23 @@ +package wallet + +import "go.sia.tech/core/types" + +// AddressBalance returns the balance of a single address. +func (m *Manager) AddressBalance(address types.Address) (balance Balance, err error) { + return m.store.AddressBalance(address) +} + +// AddressEvents returns the events of a single address. +func (m *Manager) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) { + return m.store.AddressEvents(address, offset, limit) +} + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { + return m.store.AddressSiacoinOutputs(address, offset, limit) +} + +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) { + return m.store.AddressSiafundOutputs(address, offset, limit) +} diff --git a/wallet/manager.go b/wallet/manager.go index 425f264..8c34b27 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -39,6 +39,11 @@ type ( Annotate(walletID ID, txns []types.Transaction) ([]PoolTransaction, error) + AddressBalance(address types.Address) (balance Balance, err error) + AddressEvents(address types.Address, offset, limit int) (events []Event, err error) + AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) + AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + LastCommittedIndex() (types.ChainIndex, error) } From 8a339b8c5fa4824f3fda40dfcf850a5d81ecffaa Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 8 Mar 2024 16:50:18 +0100 Subject: [PATCH 119/630] go.mod: upgrade coreutils --- .github/workflows/main.yml | 2 +- api/api_test.go | 3 +- api/server.go | 16 +++++---- cmd/walletd/node.go | 4 ++- go.mod | 8 ++--- go.sum | 14 ++++---- persist/sqlite/peers.go | 68 +++++++++++++++--------------------- persist/sqlite/peers_test.go | 28 ++++++++------- 8 files changed, 71 insertions(+), 72 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 863f1a5..d07b53f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.20', '1.21' ] + go-version: [ '1.21', '1.22' ] steps: - name: Configure git run: git config --global core.autocrlf false # required on Windows diff --git a/api/api_test.go b/api/api_test.go index 064c452..9edea87 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,6 +1,7 @@ package api_test import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -1078,7 +1079,7 @@ func TestP2P(t *testing.T) { } // connect the syncers - if _, err := s1.Connect(s2.Addr()); err != nil { + if _, err := s1.Connect(context.Background(), s2.Addr()); err != nil { t.Fatal(err) } diff --git a/api/server.go b/api/server.go index c228676..b822b2d 100644 --- a/api/server.go +++ b/api/server.go @@ -1,6 +1,7 @@ package api import ( + "context" "errors" "net/http" "reflect" @@ -34,8 +35,8 @@ type ( Syncer interface { Addr() string Peers() []*syncer.Peer - PeerInfo(peer string) (syncer.PeerInfo, bool) - Connect(addr string) (*syncer.Peer, error) + PeerInfo(addr string) (syncer.PeerInfo, error) + Connect(ctx context.Context, addr string) (*syncer.Peer, error) BroadcastHeader(bh gateway.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) @@ -94,9 +95,12 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { func (s *server) syncerPeersHandler(jc jape.Context) { var peers []GatewayPeer for _, p := range s.s.Peers() { - info, ok := s.s.PeerInfo(p.Addr()) - if !ok { - jc.Error(errors.New("peer not found"), http.StatusNotFound) + info, err := s.s.PeerInfo(p.Addr()) + if errors.Is(err, syncer.ErrPeerNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if err != nil { + jc.Error(err, http.StatusInternalServerError) return } peers = append(peers, GatewayPeer{ @@ -118,7 +122,7 @@ func (s *server) syncerConnectHandler(jc jape.Context) { if jc.Decode(&addr) != nil { return } - _, err := s.s.Connect(addr) + _, err := s.s.Connect(jc.Request.Context(), addr) jc.Check("couldn't connect to peer", err) } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index b5a6808..b2a7554 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -167,7 +167,9 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logge } for _, peer := range bootstrapPeers { - store.AddPeer(peer) + if err := store.AddPeer(peer); err != nil { + return nil, fmt.Errorf("failed to add bootstrap peer '%s': %w", peer, err) + } } header := gateway.Header{ GenesisID: genesisBlock.ID(), diff --git a/go.mod b/go.mod index 70d7041..ebc464b 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module go.sia.tech/walletd -go 1.21 +go 1.21.6 require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.2.1 - go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 + go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.17.0 go.uber.org/zap v1.27.0 @@ -18,11 +18,11 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - go.etcd.io/bbolt v1.3.8 // indirect + go.etcd.io/bbolt v1.3.9 // indirect go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 34b4773..151ee65 100644 --- a/go.sum +++ b/go.sum @@ -10,12 +10,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= +go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= -go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7 h1:G2l6fRzAdNZy2z7+FhoG2y8ARtFpR6PkXXTB5tkdfZ8= -go.sia.tech/coreutils v0.0.0-20240130201319-8303550528d7/go.mod h1:3Mb206QDd3NtRiaHZ2kN87/HKXhcBF6lHVatS7PkViY= +go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8 h1:d7AYumkRcNZSgzQne/pP2bAqF9SL071Wqqj0YouZk5g= +go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -30,10 +30,12 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= -golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 1427737..3a5e0f3 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -13,11 +13,14 @@ import ( "go.uber.org/zap" ) +func scanPeerInfo(s scanner) (pi syncer.PeerInfo, err error) { + err = s.Scan(&pi.Address, decode(&pi.FirstSeen), decode(&pi.LastConnect), &pi.SyncedBlocks, &pi.SyncDuration) + return +} + func getPeerInfo(tx *txn, peer string) (syncer.PeerInfo, error) { - const query = `SELECT first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` - var info syncer.PeerInfo - err := tx.QueryRow(query, peer).Scan(decode(&info.FirstSeen), decode(&info.LastConnect), &info.SyncedBlocks, &info.SyncDuration) - return info, err + const query = `SELECT peer_address, first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` + return scanPeerInfo(tx.QueryRow(query, peer)) } func (s *Store) updatePeerInfo(tx *txn, peer string, info syncer.PeerInfo) error { @@ -27,19 +30,16 @@ func (s *Store) updatePeerInfo(tx *txn, peer string, info syncer.PeerInfo) error } // AddPeer adds the given peer to the store. -func (s *Store) AddPeer(peer string) { - err := s.transaction(func(tx *txn) error { +func (s *Store) AddPeer(peer string) error { + return s.transaction(func(tx *txn) error { const query = `INSERT INTO syncer_peers (peer_address, first_seen, last_connect, synced_blocks, sync_duration) VALUES ($1, $2, 0, 0, 0) ON CONFLICT (peer_address) DO NOTHING` _, err := tx.Exec(query, peer, encode(time.Now())) return err }) - if err != nil { - s.log.Error("failed to add peer", zap.Error(err)) - } } // Peers returns the addresses of all known peers. -func (s *Store) Peers() (peers []string) { +func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { err := s.transaction(func(tx *txn) error { const query = `SELECT peer_address FROM syncer_peers` rows, err := tx.Query(query) @@ -48,23 +48,20 @@ func (s *Store) Peers() (peers []string) { } defer rows.Close() for rows.Next() { - var peer string - if err := rows.Scan(&peer); err != nil { - return err + peer, err := scanPeerInfo(rows) + if err != nil { + return fmt.Errorf("failed to scan peer info: %w", err) } peers = append(peers, peer) } return rows.Err() }) - if err != nil { - panic(err) // 😔 - } - return + return peers, err } // UpdatePeerInfo updates the info for the given peer. -func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - err := s.transaction(func(tx *txn) error { +func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) error { + return s.transaction(func(tx *txn) error { info, err := getPeerInfo(tx, peer) if err != nil { return fmt.Errorf("failed to get peer info: %w", err) @@ -72,13 +69,10 @@ func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { fn(&info) return s.updatePeerInfo(tx, peer, info) }) - if err != nil { - panic(err) // 😔 - } } // PeerInfo returns the info for the given peer. -func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, bool) { +func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, error) { var info syncer.PeerInfo var err error err = s.transaction(func(tx *txn) error { @@ -86,11 +80,11 @@ func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, bool) { return err }) if errors.Is(err, sql.ErrNoRows) { - return info, false + return syncer.PeerInfo{}, syncer.ErrPeerNotFound } else if err != nil { - panic(err) // 😔 + return syncer.PeerInfo{}, err } - return info, true + return info, nil } // normalizePeer normalizes a peer address to a CIDR subnet. @@ -128,35 +122,29 @@ func normalizePeer(peer string) (string, error) { // Ban temporarily bans one or more IPs. The addr should either be a single // IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). -func (s *Store) Ban(peer string, duration time.Duration, reason string) { +func (s *Store) Ban(peer string, duration time.Duration, reason string) error { address, err := normalizePeer(peer) if err != nil { - s.log.Error("failed to normalize peer", zap.Error(err)) - return + return err } - err = s.transaction(func(tx *txn) error { + return s.transaction(func(tx *txn) error { const query = `INSERT INTO syncer_bans (net_cidr, expiration, reason) VALUES ($1, $2, $3) ON CONFLICT (net_cidr) DO UPDATE SET expiration=EXCLUDED.expiration, reason=EXCLUDED.reason` _, err := tx.Exec(query, address, encode(time.Now().Add(duration)), reason) return err }) - if err != nil { - s.log.Error("failed to ban peer", zap.Error(err)) - } } // Banned returns true if the peer is banned. -func (s *Store) Banned(peer string) (banned bool) { +func (s *Store) Banned(peer string) (banned bool, _ error) { // normalize the peer into a CIDR subnet peer, err := normalizePeer(peer) if err != nil { - s.log.Error("failed to normalize peer", zap.Error(err)) - return false + return false, fmt.Errorf("failed to normalize peer: %w", err) } _, subnet, err := net.ParseCIDR(peer) if err != nil { - s.log.Error("failed to parse CIDR", zap.Error(err)) - return false + return false, fmt.Errorf("failed to parse CIDR: %w", err) } // check all subnets from the given subnet to the max subnet length @@ -189,7 +177,7 @@ func (s *Store) Banned(peer string) (banned bool) { return err }) if err != nil && !errors.Is(err, sql.ErrNoRows) { - s.log.Error("failed to check ban status", zap.Error(err)) + return false, fmt.Errorf("failed to check ban status: %w", err) } - return + return banned, nil } diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 4f3e26e..16b0732 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -20,13 +20,15 @@ func TestAddPeer(t *testing.T) { const peer = "1.2.3.4:9981" - db.AddPeer(peer) + if err := db.AddPeer(peer); err != nil { + t.Fatal(err) + } lastConnect := time.Now().Truncate(time.Second) // stored as unix milliseconds syncedBlocks := uint64(15) syncDuration := 5 * time.Second - db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { + err = db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { info.LastConnect = lastConnect info.SyncedBlocks = syncedBlocks info.SyncDuration = syncDuration @@ -35,9 +37,9 @@ func TestAddPeer(t *testing.T) { t.Fatal(err) } - info, ok := db.PeerInfo(peer) - if !ok { - t.Fatal("expected peer to be in database") + info, err := db.PeerInfo(peer) + if err != nil { + t.Fatal(err) } if !info.LastConnect.Equal(lastConnect) { @@ -61,22 +63,22 @@ func TestBanPeer(t *testing.T) { const peer = "1.2.3.4" - if db.Banned(peer) { - t.Fatal("expected peer to not be banned") + if banned, err := db.Banned(peer); err != nil || banned { + t.Fatal("expected peer to not be banned", err) } // ban the peer db.Ban(peer, time.Second, "test") - if !db.Banned(peer) { - t.Fatal("expected peer to be banned") + if banned, err := db.Banned(peer); err != nil || !banned { + t.Fatal("expected peer to be banned", err) } // wait for the ban to expire time.Sleep(time.Second) - if db.Banned(peer) { - t.Fatal("expected peer to not be banned") + if banned, err := db.Banned(peer); err != nil || banned { + t.Fatal("expected peer to not be banned", err) } // ban a subnet @@ -87,7 +89,7 @@ func TestBanPeer(t *testing.T) { t.Log("banning", subnet) db.Ban(subnet.String(), time.Second, "test") - if !db.Banned(peer) { - t.Fatal("expected peer to be banned") + if banned, err := db.Banned(peer); err != nil || !banned { + t.Fatal("expected peer to be banned", err) } } From cd74c12eb884bb346c9a1127625075e65ddf327a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 11 Mar 2024 15:59:48 +0100 Subject: [PATCH 120/630] api: address review comment --- api/api.go | 8 ++++---- api/server.go | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/api/api.go b/api/api.go index c7fbcde..6f86e19 100644 --- a/api/api.go +++ b/api/api.go @@ -14,10 +14,10 @@ type GatewayPeer struct { Inbound bool `json:"inbound"` Version string `json:"version"` - FirstSeen time.Time `json:"firstSeen"` - ConnectedSince time.Time `json:"connectedSince"` - SyncedBlocks uint64 `json:"syncedBlocks"` - SyncDuration time.Duration `json:"syncDuration"` + FirstSeen time.Time `json:"firstSeen,omitempty"` + ConnectedSince time.Time `json:"connectedSince,omitempty"` + SyncedBlocks uint64 `json:"syncedBlocks,omitempty"` + SyncDuration time.Duration `json:"syncDuration,omitempty"` } // TxpoolBroadcastRequest is the request type for /txpool/broadcast. diff --git a/api/server.go b/api/server.go index b822b2d..007a23c 100644 --- a/api/server.go +++ b/api/server.go @@ -95,24 +95,24 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { func (s *server) syncerPeersHandler(jc jape.Context) { var peers []GatewayPeer for _, p := range s.s.Peers() { + // create peer response with known fields + peer := GatewayPeer{ + Addr: p.Addr(), + Inbound: p.Inbound, + Version: p.Version(), + } + // add more info if available info, err := s.s.PeerInfo(p.Addr()) - if errors.Is(err, syncer.ErrPeerNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if err != nil { + if err != nil && !errors.Is(err, syncer.ErrPeerNotFound) { jc.Error(err, http.StatusInternalServerError) return + } else if err == nil { + peer.FirstSeen = info.FirstSeen + peer.ConnectedSince = info.LastConnect + peer.SyncedBlocks = info.SyncedBlocks + peer.SyncDuration = info.SyncDuration } - peers = append(peers, GatewayPeer{ - Addr: p.Addr(), - Inbound: p.Inbound, - Version: p.Version(), - - FirstSeen: info.FirstSeen, - ConnectedSince: info.LastConnect, - SyncedBlocks: info.SyncedBlocks, - SyncDuration: info.SyncDuration, - }) + peers = append(peers, peer) } jc.Encode(peers) } From 04b22a474d643b4f835b417cba57130e187deffe Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Mar 2024 16:42:36 +0100 Subject: [PATCH 121/630] ci: add project-add.yml --- .github/workflows/project-add.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/project-add.yml diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml new file mode 100644 index 0000000..3304fc0 --- /dev/null +++ b/.github/workflows/project-add.yml @@ -0,0 +1,21 @@ +name: Add issues and PRs to Sia project + +on: + issues: + types: + - opened + pull_request: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.5.0 + with: + # You can target a project in a different organization + # to the issue + project-url: https://github.com/orgs/SiaFoundation/projects/5 + github-token: ${{ secrets.PAT_ADD_TO_PROJECT }} From 9b20927ff0fe0953b2f52d4eae0eeed7323a7cc3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 18 Mar 2024 10:39:25 +0100 Subject: [PATCH 122/630] sqlite: fix Peers method --- persist/sqlite/peers.go | 2 +- persist/sqlite/peers_test.go | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 3a5e0f3..3ba38ac 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -41,7 +41,7 @@ func (s *Store) AddPeer(peer string) error { // Peers returns the addresses of all known peers. func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { err := s.transaction(func(tx *txn) error { - const query = `SELECT peer_address FROM syncer_peers` + const query = `SELECT peer_address, first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers` rows, err := tx.Query(query) if err != nil { return err diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 16b0732..62fe6ad 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -24,7 +24,7 @@ func TestAddPeer(t *testing.T) { t.Fatal(err) } - lastConnect := time.Now().Truncate(time.Second) // stored as unix milliseconds + lastConnect := time.Now().UTC().Truncate(time.Second) // stored as unix milliseconds syncedBlocks := uint64(15) syncDuration := 5 * time.Second @@ -51,6 +51,23 @@ func TestAddPeer(t *testing.T) { if info.SyncDuration != 5*time.Second { t.Errorf("expected SyncDuration = %s; got %s", syncDuration, info.SyncDuration) } + + peers, err := db.Peers() + if err != nil { + t.Fatal(err) + } else if len(peers) != 1 { + t.Fatalf("expected 1 peer; got %d", len(peers)) + } else if peerInfo := peers[0]; peerInfo.Address != peer { + t.Errorf("expected peer address = %q; got %q", peer, peerInfo.Address) + } else if peerInfo.LastConnect != lastConnect { + t.Errorf("expected LastConnect = %v; got %v", lastConnect, peerInfo.LastConnect) + } else if peerInfo.SyncedBlocks != syncedBlocks { + t.Errorf("expected SyncedBlocks = %d; got %d", syncedBlocks, peerInfo.SyncedBlocks) + } else if peerInfo.SyncDuration != syncDuration { + t.Errorf("expected SyncDuration = %s; got %s", syncDuration, peerInfo.SyncDuration) + } else if peerInfo.FirstSeen.IsZero() { + t.Errorf("expected FirstSeen to be non-zero; got %v", peerInfo.FirstSeen) + } } func TestBanPeer(t *testing.T) { From 36f558e74aea6e7faf4a726c4980555d5f3be7d1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 22 Mar 2024 11:56:48 -0700 Subject: [PATCH 123/630] wallet: Valid->Missed --- wallet/wallet.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 9d182ce..3255ebb 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -619,7 +619,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } } else { for i := range fce.FileContract.MissedProofOutputs { - if !relevant(fce.FileContract.ValidProofOutputs[i].Address) { + if !relevant(fce.FileContract.MissedProofOutputs[i].Address) { continue } @@ -628,7 +628,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f FileContract: fce, SiacoinOutput: sces[outputID], Missed: true, - }, []types.Address{fce.FileContract.ValidProofOutputs[i].Address}) + }, []types.Address{fce.FileContract.MissedProofOutputs[i].Address}) } } }) From 745c0d6f710b9292aa57e6da684399a87e95967c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 22 Mar 2024 12:03:36 -0700 Subject: [PATCH 124/630] ci,sqlite: fiix lint complaints --- .golangci.yml | 105 ++++++++++--------------------- persist/sqlite/addresses.go | 3 +- persist/sqlite/consensus.go | 5 -- persist/sqlite/consensus_test.go | 6 +- persist/sqlite/store.go | 4 +- persist/sqlite/wallet.go | 10 ++- 6 files changed, 46 insertions(+), 87 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 041664e..d439ef1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,13 +17,6 @@ run: # list of build tags, all linters use it. Default is empty list. build-tags: [] - # which dirs to skip: issues from them won't be reported; - # can use regexp here: generated.*, regexp is applied on full path; - # default value is empty list, but default dirs are skipped independently - # from this option's value (see skip-dirs-use-default). - skip-dirs: - - cover - # default is true. Enables skipping of directories: # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ skip-dirs-use-default: true @@ -36,9 +29,6 @@ run: # output configuration options output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number - # print lines of code with issue, default is true print-issued-lines: true @@ -53,84 +43,52 @@ linters-settings: check-shadowing: false disable-all: false - tagliatelle: - case: - rules: - json: goCamel - yaml: goCamel - + golint: + min-confidence: 1.0 gocritic: # Which checks should be enabled; can't be combined with 'disabled-checks'; # See https://go-critic.github.io/overview#checks-overview # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` # By default list of stable checks is used. - enabled-checks: - - argOrder # Diagnostic options - - badCond - - caseOrder - - dupArg - - dupBranchBody - - dupCase - - dupSubExpr - - nilValReturn - - offBy1 - - weakCond - - boolExprSimplify # Style options here and below. - - builtinShadow - - emptyFallthrough - - hexLiteral - - underef - - equalFold + enabled-tags: + - diagnostic + - style + disabled-checks: + # diagnostic + - commentedOutCode + - uncheckedInlineErr + + # style + - exitAfterDefer + - ifElseChain + - importShadow + - octalLiteral + - paramTypeCombine + - ptrToRefParam + - stringsCompare + - tooManyResultsChecker + - typeDefFirst + - typeUnparen + - unlabelStmt + - unnamedResult + - whyNoLint revive: ignore-generated-header: true rules: - - name: blank-imports - disabled: false - - name: bool-literal-in-expr - disabled: false - - name: confusing-results - disabled: false - - name: constant-logical-expr - disabled: false - - name: context-as-argument - disabled: false - - name: exported - disabled: false - - name: errorf - disabled: false - - name: if-return - disabled: false - - name: increment-decrement - disabled: false - - name: modifies-value-receiver - disabled: false - - name: optimize-operands-order - disabled: false - - name: range-val-in-closure - disabled: false - - name: struct-tag - disabled: false - - name: superfluous-else - disabled: false - - name: time-equal - disabled: false - - name: unexported-naming - disabled: false - - name: unexported-return - disabled: false - - name: unnecessary-stmt - disabled: false - - name: unreachable-code - disabled: false - name: package-comments disabled: true + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + linters: disable-all: true fast: false enable: - - tagliatelle - gocritic - gofmt - revive @@ -138,6 +96,9 @@ linters: - misspell - typecheck - whitespace + - tagliatelle + - unused + - unparam issues: # Maximum issues count per one linter. Set to 0 to disable. Default is 50. diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 8d45a72..f630e9f 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -63,8 +63,7 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) defer rows.Close() for rows.Next() { - var siacoin types.SiacoinElement - err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), decode(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) + siacoin, err := scanSiacoinElement(rows) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 7d96a4c..a8e6010 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -29,11 +29,6 @@ func scanStateElement(s scanner) (se types.StateElement, err error) { return } -func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { - err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) - return -} - func scanAddress(s scanner) (ab addressRef, err error) { err = s.Scan(&ab.ID, decode(&ab.Balance.Siacoins), decode(&ab.Balance.ImmatureSiacoins), &ab.Balance.Siafunds) return diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 1892c57..ff80a38 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -158,7 +158,7 @@ func TestReorg(t *testing.T) { for i := 0; i < 5; i++ { blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height = state.Index.Height + 1 + state.Index.Height++ } if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) @@ -252,7 +252,7 @@ func TestReorg(t *testing.T) { for i := 0; i < 10; i++ { blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height = state.Index.Height + 1 + state.Index.Height++ } if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) @@ -451,7 +451,7 @@ func TestEphemeralBalance(t *testing.T) { for i := 0; i < 2; i++ { blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height = state.Index.Height + 1 + state.Index.Height++ } if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index e50fda5..2ee3a11 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -97,9 +97,9 @@ func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { Tx: dbtx, log: log, } - if err = fn(tx); err != nil { + if err := fn(tx); err != nil { return err - } else if err = tx.Commit(); err != nil { + } else if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } return nil diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 7f76eda..23f826d 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -11,6 +11,11 @@ import ( "go.sia.tech/walletd/wallet" ) +func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + return +} + func insertAddress(tx *txn, addr types.Address) (id int64, err error) { const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address @@ -83,7 +88,7 @@ func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet. events = append(events, event) eventIDs = append(eventIDs, eventID) } - if err = rows.Err(); err != nil { + if err := rows.Err(); err != nil { return nil, nil, err } return @@ -300,8 +305,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins defer rows.Close() for rows.Next() { - var siacoin types.SiacoinElement - err := rows.Scan(decode(&siacoin.ID), &siacoin.LeafIndex, decodeSlice[types.Hash256](&siacoin.MerkleProof), decode(&siacoin.SiacoinOutput.Value), decode(&siacoin.SiacoinOutput.Address), &siacoin.MaturityHeight) + siacoin, err := scanSiacoinElement(rows) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } From af713dcaff9923d89115cfc0fbd51972954a53bb Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 22 Mar 2024 12:24:38 -0700 Subject: [PATCH 125/630] sqlite: fix siacoin element scan --- persist/sqlite/addresses.go | 4 ++-- persist/sqlite/consensus.go | 4 ++-- persist/sqlite/wallet.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index f630e9f..b7ae3fe 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -50,7 +50,7 @@ func (s *Store) AddressEvents(address types.Address, offset, limit int) (events // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE sa.sia_address=$1 @@ -94,7 +94,7 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) var siafund types.SiafundElement err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) if err != nil { - return fmt.Errorf("failed to scan siacoin element: %w", err) + return fmt.Errorf("failed to scan siafund element: %w", err) } siafunds = append(siafunds, siafund) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index a8e6010..3dfcb24 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -146,7 +146,7 @@ WHERE maturity_height=$1` var value types.Currency if err := rows.Scan(&addressID, decode(&value)); err != nil { - return fmt.Errorf("failed to scan siacoin elements: %w", err) + return fmt.Errorf("failed to scan siacoin balance: %w", err) } balanceDelta[addressID] = balanceDelta[addressID].Add(value) } @@ -201,7 +201,7 @@ func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { var value types.Currency if err := rows.Scan(&addressID, decode(&value)); err != nil { - return fmt.Errorf("failed to scan siacoin elements: %w", err) + return fmt.Errorf("failed to scan siacoin balance: %w", err) } balanceDelta[addressID] = balanceDelta[addressID].Add(value) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 23f826d..7d33b1e 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -292,7 +292,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins return err } - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siacoin_value, sa.sia_address, se.maturity_height + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) @@ -340,7 +340,7 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds var siafund types.SiafundElement err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) if err != nil { - return fmt.Errorf("failed to scan siacoin element: %w", err) + return fmt.Errorf("failed to scan siafund element: %w", err) } siafunds = append(siafunds, siafund) } From 45a92cc0a83bc88dd9dffd370bb20593ca2d9e26 Mon Sep 17 00:00:00 2001 From: Christopher Schinnerl Date: Mon, 25 Mar 2024 13:28:15 +0100 Subject: [PATCH 126/630] Update project-add.yml --- .github/workflows/project-add.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index 3304fc0..8b93101 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -10,12 +10,5 @@ on: jobs: add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.5.0 - with: - # You can target a project in a different organization - # to the issue - project-url: https://github.com/orgs/SiaFoundation/projects/5 - github-token: ${{ secrets.PAT_ADD_TO_PROJECT }} + uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master + secrets: inherit From 0582f3336b5ecfcc5d7d35813f66a4018d1bbbc3 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 11 Mar 2024 15:54:27 -0400 Subject: [PATCH 127/630] all: Refactor for new subscription API --- api/api_test.go | 27 +++++++++++++ go.mod | 2 +- go.sum | 4 +- persist/sqlite/consensus.go | 36 ++++++++--------- persist/sqlite/consensus_test.go | 59 +++++++++++++++++++--------- persist/sqlite/store.go | 2 +- persist/sqlite/wallet_test.go | 11 +----- wallet/manager.go | 66 +++++++++++++++++++++++++++----- wallet/update.go | 4 +- 9 files changed, 147 insertions(+), 64 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 9edea87..f1dd741 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -248,6 +248,17 @@ func TestWallet(t *testing.T) { t.Fatal(err) } + waitForBlock := func() { + for i := 0; i < 1000; i++ { + time.Sleep(10 * time.Millisecond) + tip, _ := ws.LastCommittedIndex() + if tip == cm.Tip() { + return + } + } + t.Fatal("timed out waiting for block") + } + sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) defer shutdown() @@ -333,6 +344,7 @@ func TestWallet(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } + waitForBlock() // get new balance balance, err = wc.Balance() @@ -372,6 +384,7 @@ func TestWallet(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } + waitForBlock() // get new balance balance, err = wc.Balance() @@ -400,6 +413,7 @@ func TestWallet(t *testing.T) { t.Fatal(err) } } + waitForBlock() // get new balance balance, err = wc.Balance() @@ -675,8 +689,19 @@ func TestV2(t *testing.T) { } return cm.AddBlocks([]types.Block{b}) } + waitForBlock := func() { + for i := 0; i < 1000; i++ { + time.Sleep(10 * time.Millisecond) + tip, _ := ws.LastCommittedIndex() + if tip == cm.Tip() { + return + } + } + t.Fatal("timed out waiting for block") + } checkBalances := func(p, s types.Currency) { t.Helper() + waitForBlock() if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) } else if !primaryBalance.Siacoins.Equals(p) { @@ -690,6 +715,7 @@ func TestV2(t *testing.T) { } sendV1 := func() error { t.Helper() + waitForBlock() // which wallet is sending? key := primaryPrivateKey @@ -736,6 +762,7 @@ func TestV2(t *testing.T) { } sendV2 := func() error { t.Helper() + waitForBlock() // which wallet is sending? key := primaryPrivateKey diff --git a/go.mod b/go.mod index ebc464b..1b8b89d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21.6 require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.2.1 - go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8 + go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.17.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 151ee65..3b8d15c 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= -go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8 h1:d7AYumkRcNZSgzQne/pP2bAqF9SL071Wqqj0YouZk5g= -go.sia.tech/coreutils v0.0.4-0.20240308153335-c2b088520ec8/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= +go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 h1:5AuiglkLdoBenrg41cJXJ4wTxkVTo85Asj9SPljnmiE= +go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 3dfcb24..427a067 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -573,33 +573,29 @@ func (ut *updateTx) RevertEvents(index types.ChainIndex) error { } // ProcessChainApplyUpdate implements chain.Subscriber -func (s *Store) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { +func (s *Store) ProcessChainApplyUpdate(cau chain.ApplyUpdate) error { s.updates = append(s.updates, cau) log := s.log.Named("ProcessChainApplyUpdate").With(zap.Stringer("index", cau.State.Index)) log.Debug("received update") - if mayCommit { - log.Debug("committing updates", zap.Int("n", len(s.updates))) - return s.transaction(func(tx *txn) error { - utx := &updateTx{ - tx: tx, - relevantAddresses: make(map[types.Address]bool), - } - - if err := wallet.ApplyChainUpdates(utx, s.updates); err != nil { - return fmt.Errorf("failed to apply updates: %w", err) - } else if err := setLastCommittedIndex(tx, cau.State.Index); err != nil { - return fmt.Errorf("failed to set last committed index: %w", err) - } - s.updates = nil - return nil - }) - } + log.Debug("committing updates", zap.Int("n", len(s.updates))) + return s.transaction(func(tx *txn) error { + utx := &updateTx{ + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } - return nil + if err := wallet.ApplyChainUpdates(utx, s.updates); err != nil { + return fmt.Errorf("failed to apply updates: %w", err) + } else if err := setLastCommittedIndex(tx, cau.State.Index); err != nil { + return fmt.Errorf("failed to set last committed index: %w", err) + } + s.updates = nil + return nil + }) } // ProcessChainRevertUpdate implements chain.Subscriber -func (s *Store) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { +func (s *Store) ProcessChainRevertUpdate(cru chain.RevertUpdate) error { log := s.log.Named("ProcessChainRevertUpdate").With(zap.Stringer("index", cru.State.Index)) // update hasn't been committed yet diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index ff80a38..fafd567 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -76,6 +76,31 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty return b } +func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { + index, err := db.LastCommittedIndex() + if err != nil { + t.Fatal(err) + } + for index != cm.Tip() { + crus, caus, err := cm.UpdatesSince(index, 1000) + if err != nil { + t.Fatal(err) + } + for _, cru := range crus { + if err := db.ProcessChainRevertUpdate(cru); err != nil { + t.Fatal("failed to process revert update:", err) + } + index = cru.State.Index + } + for _, cau := range caus { + if err := db.ProcessChainApplyUpdate(cau); err != nil { + t.Fatal("failed to process apply update:", err) + } + index = cau.State.Index + } + } +} + func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -94,20 +119,14 @@ func TestReorg(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network(addr) + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { t.Fatal(err) } - defer store.Close() - cm := chain.NewManager(store, genesisState) - if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { - t.Fatal(err) - } - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -121,6 +140,7 @@ func TestReorg(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was received balance, err := db.AddressBalance(addr) @@ -163,6 +183,7 @@ func TestReorg(t *testing.T) { if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was reverted balance, err = db.AddressBalance(addr) @@ -194,6 +215,7 @@ func TestReorg(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was received balance, err = db.AddressBalance(addr) @@ -235,6 +257,7 @@ func TestReorg(t *testing.T) { prevState = cm.TipState() } } + syncDB(t, db, cm) // check that the balance was updated balance, err = db.AddressBalance(addr) @@ -257,6 +280,7 @@ func TestReorg(t *testing.T) { if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the balance is correct balance, err = db.AddressBalance(addr) @@ -299,20 +323,15 @@ func TestEphemeralBalance(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network(addr) + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { t.Fatal(err) } - defer store.Close() cm := chain.NewManager(store, genesisState) - if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { - t.Fatal(err) - } - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -328,6 +347,7 @@ func TestEphemeralBalance(t *testing.T) { if err := cm.AddBlocks([]types.Block{block}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was received balance, err := db.AddressBalance(addr) @@ -355,6 +375,7 @@ func TestEphemeralBalance(t *testing.T) { t.Fatal(err) } } + syncDB(t, db, cm) // create a transaction that spends the matured payout utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) @@ -418,6 +439,7 @@ func TestEphemeralBalance(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(revertState, txnset, types.VoidAddress)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was spent balance, err = db.AddressBalance(addr) @@ -456,6 +478,7 @@ func TestEphemeralBalance(t *testing.T) { if err := cm.AddBlocks(blocks); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the transaction was reverted balance, err = db.AddressBalance(addr) @@ -494,20 +517,15 @@ func TestV2(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV2Network(addr) + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { t.Fatal(err) } - defer store.Close() cm := chain.NewManager(store, genesisState) - if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { - t.Fatal(err) - } - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -520,6 +538,7 @@ func TestV2(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was received balance, err := db.AddressBalance(addr) @@ -546,6 +565,7 @@ func TestV2(t *testing.T) { t.Fatal(err) } } + syncDB(t, db, cm) // create a v2 transaction that spends the matured payout utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) @@ -572,6 +592,7 @@ func TestV2(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the change was received balance, err = db.AddressBalance(addr) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 2ee3a11..dff4ed9 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -20,7 +20,7 @@ type ( db *sql.DB log *zap.Logger - updates []*chain.ApplyUpdate + updates []chain.ApplyUpdate } ) diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index 20db201..b976d95 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -132,14 +132,9 @@ func TestResubscribe(t *testing.T) { if err != nil { t.Fatal(err) } - defer store.Close() cm := chain.NewManager(store, genesisState) - if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { - t.Fatal(err) - } - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -153,6 +148,7 @@ func TestResubscribe(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } + syncDB(t, db, cm) // check that the payout was received balance, err := db.WalletBalance(w.ID) @@ -184,11 +180,6 @@ func TestResubscribe(t *testing.T) { t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) } - cm.RemoveSubscriber(db) - if err := cm.AddSubscriber(db, types.ChainIndex{}); err != nil { - t.Fatal(err) - } - // check that the balance, events, and utxos did not change // check that the payout was received balance, err = db.WalletBalance(w.ID) diff --git a/wallet/manager.go b/wallet/manager.go index 8c34b27..7720f21 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -14,15 +14,17 @@ import ( type ( // A ChainManager manages the consensus state ChainManager interface { - AddSubscriber(chain.Subscriber, types.ChainIndex) error - RemoveSubscriber(chain.Subscriber) - + Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) + + OnReorg(fn func(types.ChainIndex)) (cancel func()) + UpdatesSince(index types.ChainIndex, max int) (rus []chain.RevertUpdate, aus []chain.ApplyUpdate, err error) } // A Store is a persistent store of wallet data. Store interface { - chain.Subscriber + ProcessChainApplyUpdate(cau chain.ApplyUpdate) error + ProcessChainRevertUpdate(cru chain.RevertUpdate) error WalletEvents(walletID ID, offset, limit int) ([]Event, error) AddWallet(Wallet) (Wallet, error) @@ -53,8 +55,9 @@ type ( store Store log *zap.Logger - mu sync.Mutex - used map[types.Hash256]bool + mu sync.Mutex + used map[types.Hash256]bool + unsubscribe func() } ) @@ -158,8 +161,32 @@ func (m *Manager) Subscribe(startHeight uint64) error { return errors.New("invalid height") } } - m.chain.RemoveSubscriber(m.store) - return m.chain.AddSubscriber(m.store, index) + m.mu.Lock() + defer m.mu.Unlock() + // TODO: is this right? won't it result in duplicate state? + return syncStore(m.store, m.chain, index) +} + +func syncStore(store Store, cm ChainManager, index types.ChainIndex) error { + for index != cm.Tip() { + crus, caus, err := cm.UpdatesSince(index, 1000) + if err != nil { + return fmt.Errorf("failed to subscribe to chain manager: %w", err) + } + for _, cru := range crus { + if err := store.ProcessChainRevertUpdate(cru); err != nil { + return fmt.Errorf("failed to process revert update: %w", err) + } + index = cru.State.Index + } + for _, cau := range caus { + if err := store.ProcessChainApplyUpdate(cau); err != nil { + return fmt.Errorf("failed to process apply update: %w", err) + } + index = cau.State.Index + } + } + return nil } // NewManager creates a new wallet manager. @@ -173,8 +200,29 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) lastTip, err := store.LastCommittedIndex() if err != nil { return nil, fmt.Errorf("failed to get last committed index: %w", err) - } else if err := cm.AddSubscriber(store, lastTip); err != nil { + } + if err := syncStore(store, cm, lastTip); err != nil { return nil, fmt.Errorf("failed to subscribe to chain manager: %w", err) } + + reorgChan := make(chan types.ChainIndex, 1) + go func() { + for range reorgChan { + m.mu.Lock() + lastTip, err := store.LastCommittedIndex() + if err != nil { + log.Error("failed to get last committed index", zap.Error(err)) + } else if err := syncStore(store, cm, lastTip); err != nil { + log.Error("failed to sync store", zap.Error(err)) + } + m.mu.Unlock() + } + }() + m.unsubscribe = cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgChan <- index: + default: + } + }) return m, nil } diff --git a/wallet/update.go b/wallet/update.go index 2890d3c..a726a2a 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -49,7 +49,7 @@ type ( ) // ApplyChainUpdates atomically applies a set of chain updates to a store -func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { +func ApplyChainUpdates(tx ApplyTx, updates []chain.ApplyUpdate) error { for _, cau := range updates { // update the immature balance of each relevant address if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { @@ -175,7 +175,7 @@ func ApplyChainUpdates(tx ApplyTx, updates []*chain.ApplyUpdate) error { } // RevertChainUpdate atomically reverts a chain update from a store -func RevertChainUpdate(tx RevertTx, cru *chain.RevertUpdate) error { +func RevertChainUpdate(tx RevertTx, cru chain.RevertUpdate) error { // determine which siacoin and siafund elements are ephemeral // // note: I thought we could use LeafIndex == EphemeralLeafIndex, but From eb6421ac8b13ea5550bf799c270aa293ae646641 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 25 Mar 2024 09:40:21 -0700 Subject: [PATCH 128/630] api: fix test addresses --- api/api_test.go | 79 ++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index f1dd741..592ffe2 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -53,6 +53,17 @@ func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Cl return c, func() { l.Close() } } +func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { + for i := 0; i < 1000; i++ { + time.Sleep(10 * time.Millisecond) + tip, _ := ws.LastCommittedIndex() + if tip == cm.Tip() { + return + } + } + tb.Fatal("timed out waiting for block") +} + func TestWalletAdd(t *testing.T) { log := zaptest.NewLogger(t) @@ -248,17 +259,6 @@ func TestWallet(t *testing.T) { t.Fatal(err) } - waitForBlock := func() { - for i := 0; i < 1000; i++ { - time.Sleep(10 * time.Millisecond) - tip, _ := ws.LastCommittedIndex() - if tip == cm.Tip() { - return - } - } - t.Fatal("timed out waiting for block") - } - sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) defer shutdown() @@ -344,16 +344,16 @@ func TestWallet(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } - waitForBlock() + waitForBlock(t, cm, ws) // get new balance balance, err = wc.Balance() if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) + t.Fatal("balance should be 1 SC, got", balance.Siacoins) } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } // transaction should appear in history @@ -361,14 +361,14 @@ func TestWallet(t *testing.T) { if err != nil { t.Fatal(err) } else if len(events) == 0 { - t.Error("transaction should appear in history") + t.Fatal("transaction should appear in history") } outputs, err := wc.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { - t.Error("should have two UTXOs, got", len(outputs)) + t.Fatal("should have two UTXOs, got", len(outputs)) } // mine a block to add an immature balance @@ -384,16 +384,16 @@ func TestWallet(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } - waitForBlock() + waitForBlock(t, cm, ws) // get new balance balance, err = wc.Balance() if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) + t.Fatal("balance should be 1 SC, got", balance.Siacoins) } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { - t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + t.Fatalf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) } // mine enough blocks for the miner payout to mature @@ -413,16 +413,16 @@ func TestWallet(t *testing.T) { t.Fatal(err) } } - waitForBlock() + waitForBlock(t, cm, ws) // get new balance balance, err = wc.Balance() if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(expectedBalance) { - t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + t.Fatalf("balance should be %d, got %d", expectedBalance, balance.Siacoins) } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } } @@ -540,15 +540,16 @@ func TestAddresses(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } + waitForBlock(t, cm, ws) // get new balance balance, err = c.AddressBalance(addr.Address) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) + t.Fatal("balance should be 1 SC, got", balance.Siacoins) } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } // transaction should appear in history @@ -556,14 +557,14 @@ func TestAddresses(t *testing.T) { if err != nil { t.Fatal(err) } else if len(events) == 0 { - t.Error("transaction should appear in history") + t.Fatal("transaction should appear in history") } outputs, err := c.AddressSiacoinOutputs(addr.Address, 0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { - t.Error("should have two UTXOs, got", len(outputs)) + t.Fatal("should have two UTXOs, got", len(outputs)) } // mine a block to add an immature balance @@ -579,15 +580,16 @@ func TestAddresses(t *testing.T) { if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } + waitForBlock(t, cm, ws) // get new balance balance, err = c.AddressBalance(addr.Address) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) + t.Fatal("balance should be 1 SC, got", balance.Siacoins) } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { - t.Errorf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + t.Fatalf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) } // mine enough blocks for the miner payout to mature @@ -607,15 +609,16 @@ func TestAddresses(t *testing.T) { t.Fatal(err) } } + waitForBlock(t, cm, ws) // get new balance balance, err = c.AddressBalance(addr.Address) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(expectedBalance) { - t.Errorf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + t.Fatalf("balance should be %d, got %d", expectedBalance, balance.Siacoins) } else if !balance.ImmatureSiacoins.IsZero() { - t.Error("immature balance should be 0 SC, got", balance.ImmatureSiacoins) + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } } @@ -689,19 +692,9 @@ func TestV2(t *testing.T) { } return cm.AddBlocks([]types.Block{b}) } - waitForBlock := func() { - for i := 0; i < 1000; i++ { - time.Sleep(10 * time.Millisecond) - tip, _ := ws.LastCommittedIndex() - if tip == cm.Tip() { - return - } - } - t.Fatal("timed out waiting for block") - } checkBalances := func(p, s types.Currency) { t.Helper() - waitForBlock() + waitForBlock(t, cm, ws) if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) } else if !primaryBalance.Siacoins.Equals(p) { @@ -715,7 +708,7 @@ func TestV2(t *testing.T) { } sendV1 := func() error { t.Helper() - waitForBlock() + waitForBlock(t, cm, ws) // which wallet is sending? key := primaryPrivateKey @@ -762,7 +755,7 @@ func TestV2(t *testing.T) { } sendV2 := func() error { t.Helper() - waitForBlock() + waitForBlock(t, cm, ws) // which wallet is sending? key := primaryPrivateKey From 4bf72a099855423cf6282e1aea206b0fac0a5ba3 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 25 Mar 2024 10:38:26 -0700 Subject: [PATCH 129/630] api: TestP2P waitForBlock --- api/api_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index 592ffe2..9088354 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1118,6 +1118,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } } + waitForBlock(t, cm1, store1) // now send coins back with a v2 transaction if err := sendV2(); err != nil { t.Fatal(err) @@ -1133,6 +1134,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } } + waitForBlock(t, cm1, store1) // v1 transactions should no longer work if err := sendV1(); err == nil { t.Fatal("expected v1 txn to be rejected") From 4699ff8a17c20b021475dce374d30b8e8993deed Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 25 Mar 2024 11:16:33 -0700 Subject: [PATCH 130/630] sqlite: fix Annotate SQL scan destination --- api/api_test.go | 44 ++++++++++++++++++++++++++++++++++++++-- persist/sqlite/wallet.go | 2 +- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 9088354..08811dc 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -233,6 +233,14 @@ func TestWalletAdd(t *testing.T) { func TestWallet(t *testing.T) { log := zaptest.NewLogger(t) + // create syncer + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create chain manager n, genesisBlock := testNetwork() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) @@ -241,26 +249,37 @@ func TestWallet(t *testing.T) { Address: giftAddress, } - // create wallets dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { t.Fatal(err) } cm := chain.NewManager(dbstore, tipState) + // create the sqlite store ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } defer ws.Close() + // create the syncer + s := syncer.New(syncerListener, cm, ws, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + // create the wallet manager wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) if err != nil { t.Fatal(err) } + // create seed address vault sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) - c, shutdown := runServer(cm, nil, wm) + + // run server + c, shutdown := runServer(cm, s, wm) defer shutdown() w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { @@ -331,6 +350,26 @@ func TestWallet(t *testing.T) { sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] + // broadcast the transaction to the transaction pool + if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { + t.Fatal(err) + } + + // shouldn't have any events yet + events, err = wc.Events(0, -1) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatal("event history should be empty") + } + + tpool, err := wc.PoolTransactions() + if err != nil { + t.Fatal(err) + } else if len(tpool) != 1 { + t.Fatal("txpool should have one transaction") + } + cs := cm.TipState() b := types.Block{ ParentID: cs.Index.ID, @@ -338,6 +377,7 @@ func TestWallet(t *testing.T) { MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, Transactions: []types.Transaction{txn}, } + for b.ID().CmpWork(cs.ChildTarget) < 0 { b.Nonce += cs.NonceFactor() } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 7d33b1e..7d0caf3 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -407,7 +407,7 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` // addresses into memory. ownsAddress := func(address types.Address) bool { var dbID int64 - err := stmt.QueryRow(id, encode(address)).Scan(dbID) + err := stmt.QueryRow(id, encode(address)).Scan(&dbID) if err != nil && !errors.Is(err, sql.ErrNoRows) { panic(err) // database error } From 70fed3e9591079d2de5bddce08204e37f328979e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 27 Mar 2024 09:25:25 -0700 Subject: [PATCH 131/630] api: more waitForBlocks --- api/api_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index 08811dc..1fac5d1 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1084,6 +1084,7 @@ func TestP2P(t *testing.T) { return err } checkBalances(pbal, sbal) + waitForBlock(t, cm1, store1) return nil } sendV2 := func() error { @@ -1134,6 +1135,7 @@ func TestP2P(t *testing.T) { } else if err := addBlock(); err != nil { return err } + waitForBlock(t, cm1, store1) checkBalances(pbal, sbal) return nil } From b7e808dbd0a443e67952a3b0c9abf34e1c54b977 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 27 Mar 2024 13:44:30 -0700 Subject: [PATCH 132/630] sqlite,wallet: batch updates into a single transaction --- persist/sqlite/consensus.go | 42 +----- persist/sqlite/consensus_test.go | 15 +- persist/sqlite/store.go | 3 - wallet/manager.go | 18 +-- wallet/update.go | 241 ++++++++++++++++--------------- 5 files changed, 132 insertions(+), 187 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 427a067..799c3a1 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -10,7 +10,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/wallet" - "go.uber.org/zap" ) type updateTx struct { @@ -573,49 +572,16 @@ func (ut *updateTx) RevertEvents(index types.ChainIndex) error { } // ProcessChainApplyUpdate implements chain.Subscriber -func (s *Store) ProcessChainApplyUpdate(cau chain.ApplyUpdate) error { - s.updates = append(s.updates, cau) - log := s.log.Named("ProcessChainApplyUpdate").With(zap.Stringer("index", cau.State.Index)) - log.Debug("received update") - log.Debug("committing updates", zap.Int("n", len(s.updates))) +func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { return s.transaction(func(tx *txn) error { utx := &updateTx{ tx: tx, relevantAddresses: make(map[types.Address]bool), } - if err := wallet.ApplyChainUpdates(utx, s.updates); err != nil { - return fmt.Errorf("failed to apply updates: %w", err) - } else if err := setLastCommittedIndex(tx, cau.State.Index); err != nil { - return fmt.Errorf("failed to set last committed index: %w", err) - } - s.updates = nil - return nil - }) -} - -// ProcessChainRevertUpdate implements chain.Subscriber -func (s *Store) ProcessChainRevertUpdate(cru chain.RevertUpdate) error { - log := s.log.Named("ProcessChainRevertUpdate").With(zap.Stringer("index", cru.State.Index)) - - // update hasn't been committed yet - if len(s.updates) > 0 && s.updates[len(s.updates)-1].Block.ID() == cru.Block.ID() { - log.Debug("removed uncommitted update") - s.updates = s.updates[:len(s.updates)-1] - return nil - } - - log.Debug("reverting update") - // update has been committed, revert it - return s.transaction(func(tx *txn) error { - utx := &updateTx{ - tx: tx, - relevantAddresses: make(map[types.Address]bool), - } - - if err := wallet.RevertChainUpdate(utx, cru); err != nil { - return fmt.Errorf("failed to revert update: %w", err) - } else if err := setLastCommittedIndex(tx, cru.State.Index); err != nil { + if err := wallet.UpdateChainState(utx, reverted, applied); err != nil { + return fmt.Errorf("failed to update chain state: %w", err) + } else if err := setLastCommittedIndex(tx, applied[len(applied)-1].State.Index); err != nil { return fmt.Errorf("failed to set last committed index: %w", err) } return nil diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index fafd567..eb2ec7b 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -85,19 +85,10 @@ func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { crus, caus, err := cm.UpdatesSince(index, 1000) if err != nil { t.Fatal(err) + } else if err := db.UpdateChainState(crus, caus); err != nil { + t.Fatal(err) } - for _, cru := range crus { - if err := db.ProcessChainRevertUpdate(cru); err != nil { - t.Fatal("failed to process revert update:", err) - } - index = cru.State.Index - } - for _, cau := range caus { - if err := db.ProcessChainApplyUpdate(cau); err != nil { - t.Fatal("failed to process apply update:", err) - } - index = cau.State.Index - } + index = caus[len(caus)-1].State.Index } } diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index dff4ed9..39f03fe 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "go.sia.tech/coreutils/chain" "go.uber.org/zap" "lukechampine.com/frand" ) @@ -19,8 +18,6 @@ type ( Store struct { db *sql.DB log *zap.Logger - - updates []chain.ApplyUpdate } ) diff --git a/wallet/manager.go b/wallet/manager.go index 7720f21..308c519 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -23,8 +23,7 @@ type ( // A Store is a persistent store of wallet data. Store interface { - ProcessChainApplyUpdate(cau chain.ApplyUpdate) error - ProcessChainRevertUpdate(cru chain.RevertUpdate) error + UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error WalletEvents(walletID ID, offset, limit int) ([]Event, error) AddWallet(Wallet) (Wallet, error) @@ -172,19 +171,10 @@ func syncStore(store Store, cm ChainManager, index types.ChainIndex) error { crus, caus, err := cm.UpdatesSince(index, 1000) if err != nil { return fmt.Errorf("failed to subscribe to chain manager: %w", err) + } else if err := store.UpdateChainState(crus, caus); err != nil { + return fmt.Errorf("failed to update chain state: %w", err) } - for _, cru := range crus { - if err := store.ProcessChainRevertUpdate(cru); err != nil { - return fmt.Errorf("failed to process revert update: %w", err) - } - index = cru.State.Index - } - for _, cau := range caus { - if err := store.ProcessChainApplyUpdate(cau); err != nil { - return fmt.Errorf("failed to process apply update: %w", err) - } - index = cau.State.Index - } + index = caus[len(caus)-1].State.Index } return nil } diff --git a/wallet/update.go b/wallet/update.go index a726a2a..52e3f43 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -29,153 +29,141 @@ type ( RemoveSiafundElements([]types.SiafundElement, types.ChainIndex) error AddressRelevant(types.Address) (bool, error) - } - - // An ApplyTx atomically applies a set of updates to a store. - ApplyTx interface { - UpdateTx ApplyMatureSiacoinBalance(types.ChainIndex) error AddEvents([]Event) error - } - - // RevertTx atomically reverts an update from a store. - RevertTx interface { - UpdateTx RevertMatureSiacoinBalance(types.ChainIndex) error RevertEvents(index types.ChainIndex) error } ) -// ApplyChainUpdates atomically applies a set of chain updates to a store -func ApplyChainUpdates(tx ApplyTx, updates []chain.ApplyUpdate) error { - for _, cau := range updates { - // update the immature balance of each relevant address - if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { - return fmt.Errorf("failed to get matured siacoin elements: %w", err) - } +// applyChainUpdate atomically applies a chain update to a store +func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { + // update the immature balance of each relevant address + if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { + return fmt.Errorf("failed to get matured siacoin elements: %w", err) + } - // determine which siacoin and siafund elements are ephemeral - // - // note: I thought we could use LeafIndex == EphemeralLeafIndex, but - // it seems to be set before the subscriber is called. - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cau.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } + // determine which siacoin and siafund elements are ephemeral + // + // note: I thought we could use LeafIndex == EphemeralLeafIndex, but + // it seems to be set before the subscriber is called. + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cau.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true } - - // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement - cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { - return - } - - relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) - if err != nil { - panic(err) - } else if !relevant { - return - } - - if spent { - spentSiacoinElements = append(spentSiacoinElements, se) - } else { - newSiacoinElements = append(newSiacoinElements, se) - } - }) - - if err := tx.AddSiacoinElements(newSiacoinElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to add siacoin elements: %w", err) - } else if err := tx.RemoveSiacoinElements(spentSiacoinElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to remove siacoin elements: %w", err) + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] } - - var newSiafundElements, spentSiafundElements []types.SiafundElement - cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { - return - } - - relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) - if err != nil { - panic(err) - } else if !relevant { - return - } - - if spent { - spentSiafundElements = append(spentSiafundElements, se) - } else { - newSiafundElements = append(newSiafundElements, se) - } - }) - - if err := tx.AddSiafundElements(newSiafundElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to add siafund elements: %w", err) - } else if err := tx.RemoveSiafundElements(spentSiafundElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to remove siafund elements: %w", err) + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true } - - // add events - relevant := func(addr types.Address) bool { - relevant, err := tx.AddressRelevant(addr) - if err != nil { - panic(fmt.Errorf("failed to check if address is relevant: %w", err)) - } - return relevant + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] } - if err := tx.AddEvents(AppliedEvents(cau.State, cau.Block, cau, relevant)); err != nil { - return fmt.Errorf("failed to add events: %w", err) + } + + // add new siacoin elements to the store + var newSiacoinElements, spentSiacoinElements []types.SiacoinElement + cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { + if ephemeral[se.ID] { + return } - // fetch all siacoin and siafund state elements - siacoinStateElements, err := tx.SiacoinStateElements() + relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) if err != nil { - return fmt.Errorf("failed to get siacoin state elements: %w", err) + panic(err) + } else if !relevant { + return } - // update siacoin element proofs - for i := range siacoinStateElements { - cau.UpdateElementProof(&siacoinStateElements[i]) + if spent { + spentSiacoinElements = append(spentSiacoinElements, se) + } else { + newSiacoinElements = append(newSiacoinElements, se) } + }) + + if err := tx.AddSiacoinElements(newSiacoinElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } else if err := tx.RemoveSiacoinElements(spentSiacoinElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) + } - if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) + var newSiafundElements, spentSiafundElements []types.SiafundElement + cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { + if ephemeral[se.ID] { + return } - siafundStateElements, err := tx.SiafundStateElements() + relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) if err != nil { - return fmt.Errorf("failed to get siafund state elements: %w", err) + panic(err) + } else if !relevant { + return } - // update siafund element proofs - for i := range siafundStateElements { - cau.UpdateElementProof(&siafundStateElements[i]) + if spent { + spentSiafundElements = append(spentSiafundElements, se) + } else { + newSiafundElements = append(newSiafundElements, se) } + }) - if err := tx.UpdateSiafundStateElements(siafundStateElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) + if err := tx.AddSiafundElements(newSiafundElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } else if err := tx.RemoveSiafundElements(spentSiafundElements, cau.State.Index); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + + // add events + relevant := func(addr types.Address) bool { + relevant, err := tx.AddressRelevant(addr) + if err != nil { + panic(fmt.Errorf("failed to check if address is relevant: %w", err)) } + return relevant + } + if err := tx.AddEvents(AppliedEvents(cau.State, cau.Block, cau, relevant)); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + + // fetch all siacoin and siafund state elements + siacoinStateElements, err := tx.SiacoinStateElements() + if err != nil { + return fmt.Errorf("failed to get siacoin state elements: %w", err) + } + + // update siacoin element proofs + for i := range siacoinStateElements { + cau.UpdateElementProof(&siacoinStateElements[i]) + } + + if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) + } + + siafundStateElements, err := tx.SiafundStateElements() + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } + + // update siafund element proofs + for i := range siafundStateElements { + cau.UpdateElementProof(&siafundStateElements[i]) + } + + if err := tx.UpdateSiafundStateElements(siafundStateElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) } return nil } -// RevertChainUpdate atomically reverts a chain update from a store -func RevertChainUpdate(tx RevertTx, cru chain.RevertUpdate) error { +// revertChainUpdate atomically reverts a chain update from a store +func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { // determine which siacoin and siafund elements are ephemeral // // note: I thought we could use LeafIndex == EphemeralLeafIndex, but @@ -197,12 +185,6 @@ func RevertChainUpdate(tx RevertTx, cru chain.RevertUpdate) error { } } - // revert the immature balance of each relevant address - revertedIndex := types.ChainIndex{ - Height: cru.State.Index.Height + 1, - ID: cru.Block.ID(), - } - var removedSiacoinElements, addedSiacoinElements []types.SiacoinElement cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if ephemeral[se.ID] { @@ -291,3 +273,22 @@ func RevertChainUpdate(tx RevertTx, cru chain.RevertUpdate) error { // revert events return tx.RevertEvents(revertedIndex) } + +func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { + for _, cru := range reverted { + revertedIndex := types.ChainIndex{ + ID: cru.Block.ID(), + Height: cru.State.Index.Height + 1, + } + if err := revertChainUpdate(tx, cru, revertedIndex); err != nil { + return fmt.Errorf("failed to revert chain update %q: %w", revertedIndex, err) + } + } + + for _, cau := range applied { + if err := applyChainUpdate(tx, cau); err != nil { + return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) + } + } + return nil +} From 91d1df63f4df74476b78e957d9724c34ad1b1726 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 27 Mar 2024 13:58:48 -0700 Subject: [PATCH 133/630] api: waitForBlock processing in both stores --- api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 1fac5d1..a5e59ea 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1021,6 +1021,8 @@ func TestP2P(t *testing.T) { } checkBalances := func(p, s types.Currency) { t.Helper() + waitForBlock(t, cm1, store1) + waitForBlock(t, cm2, store2) if primaryBalance, err := primary.Balance(); err != nil { t.Fatal(err) } else if !primaryBalance.Siacoins.Equals(p) { @@ -1084,7 +1086,6 @@ func TestP2P(t *testing.T) { return err } checkBalances(pbal, sbal) - waitForBlock(t, cm1, store1) return nil } sendV2 := func() error { @@ -1135,7 +1136,6 @@ func TestP2P(t *testing.T) { } else if err := addBlock(); err != nil { return err } - waitForBlock(t, cm1, store1) checkBalances(pbal, sbal) return nil } From d3a8e89441b4997cb3cfc4535cc14b5c950d56ef Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 1 Apr 2024 16:50:06 -0700 Subject: [PATCH 134/630] ci: update workflows --- .github/actions/test/action.yml | 4 ++-- .github/workflows/main.yml | 4 ++-- .github/workflows/publish.yml | 24 ++++++++++++------------ .github/workflows/ui.yml | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 5754f0e..4a0a3e3 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: git config --global core.autocrlf false - name: Lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v4 with: skip-cache: true # - name: Analyze @@ -17,6 +17,6 @@ runs: # analyzers: | # go.sia.tech/jape.Analyzer - name: Test - uses: n8maninger/action-golang-test@v1 + uses: n8maninger/action-golang-test@v2 with: args: "-race;-tags=testing netgo" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d07b53f..dcb05b4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,8 +21,8 @@ jobs: steps: - name: Configure git run: git config --global core.autocrlf false # required on Windows - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1b3cb03..41e5f50 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,15 +18,15 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v3 - - uses: docker/setup-qemu-action@v2 - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/metadata-action@v4 + - uses: docker/metadata-action@v5 name: generate tags id: meta with: @@ -35,7 +35,7 @@ jobs: type=ref,event=branch type=sha,prefix= type=semver,pattern={{version}} - - uses: docker/build-push-action@v4 + - uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 @@ -44,8 +44,8 @@ jobs: build-linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: 'stable' - name: Setup @@ -83,8 +83,8 @@ jobs: build-mac: runs-on: macos-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: 'stable' - name: Setup @@ -167,8 +167,8 @@ jobs: build-windows: runs-on: windows-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: 'stable' - name: Setup diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index dcb530a..2d7896d 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: '1.21' @@ -60,7 +60,7 @@ jobs: go mod tidy - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 if: env.GO_TAG != 'null' with: token: ${{ secrets.GITHUB_TOKEN }} From 7c07efad57780b86458cb69e724a0b0380cd64b2 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 1 Apr 2024 16:50:16 -0700 Subject: [PATCH 135/630] sqlite, wallet: fix resubscribe panic --- persist/sqlite/consensus.go | 68 +++++++++++--- persist/sqlite/init.sql | 5 +- persist/sqlite/store.go | 10 +-- wallet/manager.go | 22 ++--- wallet/wallet_test.go | 171 ++++++++++++++++++++++++++++++++++++ 5 files changed, 248 insertions(+), 28 deletions(-) create mode 100644 wallet/wallet_test.go diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 799c3a1..0a54941 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -13,8 +13,7 @@ import ( ) type updateTx struct { - tx *txn - + tx *txn relevantAddresses map[types.Address]bool } @@ -130,25 +129,38 @@ func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, } func (ut *updateTx) ApplyMatureSiacoinBalance(index types.ChainIndex) error { - const query = `SELECT se.address_id, se.siacoin_value + const query = `SELECT id, se.address_id, se.siacoin_value FROM siacoin_elements se -WHERE maturity_height=$1` +WHERE maturity_height=$1 AND matured=false` rows, err := ut.tx.Query(query, index.Height) if err != nil { return fmt.Errorf("failed to query siacoin elements: %w", err) } defer rows.Close() + var matured []types.SiacoinOutputID balanceDelta := make(map[int64]types.Currency) for rows.Next() { + var outputID types.SiacoinOutputID var addressID int64 var value types.Currency - if err := rows.Scan(&addressID, decode(&value)); err != nil { + if err := rows.Scan(decode(&outputID), &addressID, decode(&value)); err != nil { return fmt.Errorf("failed to scan siacoin balance: %w", err) } balanceDelta[addressID] = balanceDelta[addressID].Add(value) + matured = append(matured, outputID) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) + } + + updateMaturedStmt, err := ut.tx.Prepare(`UPDATE siacoin_elements SET matured=true WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) } + defer updateMaturedStmt.Close() getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) if err != nil { @@ -168,7 +180,6 @@ WHERE maturity_height=$1` if err != nil { return fmt.Errorf("failed to get address balance: %w", err) } - balance = balance.Add(delta) immatureBalance = immatureBalance.Sub(delta) @@ -181,29 +192,53 @@ WHERE maturity_height=$1` return fmt.Errorf("expected 1 row affected, got %v", n) } } + + for _, id := range matured { + res, err := updateMaturedStmt.Exec(encode(id)) + if err != nil { + return fmt.Errorf("failed to update matured: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } return nil } func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { - const query = `SELECT se.address_id, se.siacoin_value + const query = `SELECT se.id, se.address_id, se.siacoin_value FROM siacoin_elements se - WHERE maturity_height=$1` + WHERE maturity_height=$1 AND matured=true` rows, err := ut.tx.Query(query, index.Height) if err != nil { return fmt.Errorf("failed to query siacoin elements: %w", err) } defer rows.Close() + var matured []types.SiacoinOutputID balanceDelta := make(map[int64]types.Currency) for rows.Next() { + var outputID types.SiacoinOutputID var addressID int64 var value types.Currency - if err := rows.Scan(&addressID, decode(&value)); err != nil { + if err := rows.Scan(decode(&outputID), &addressID, decode(&value)); err != nil { return fmt.Errorf("failed to scan siacoin balance: %w", err) } balanceDelta[addressID] = balanceDelta[addressID].Add(value) + matured = append(matured, outputID) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) + } + + updateMaturedStmt, err := ut.tx.Prepare(`UPDATE siacoin_elements SET matured=false WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) } + defer updateMaturedStmt.Close() getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) if err != nil { @@ -236,6 +271,17 @@ func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { return fmt.Errorf("expected 1 row affected, got %v", n) } } + + for _, id := range matured { + res, err := updateMaturedStmt.Exec(encode(id)) + if err != nil { + return fmt.Errorf("failed to update matured: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } return nil } @@ -251,7 +297,7 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty defer addrStmt.Close() // ignore elements already in the database. - insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -267,7 +313,7 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } var dummyID types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID).Scan(decode(&dummyID)) + err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0).Scan(decode(&dummyID)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 6a9ee54..77715fc 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -18,10 +18,11 @@ CREATE TABLE siacoin_elements ( merkle_proof BLOB NOT NULL, leaf_index INTEGER NOT NULL, maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ - address_id INTEGER NOT NULL REFERENCES sia_addresses (id) + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL /* tracks whether the value has been added to the address balance */ ); CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); -CREATE INDEX siacoin_elements_maturity_height ON siacoin_elements (maturity_height); +CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 39f03fe..417ee19 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -21,6 +21,11 @@ type ( } ) +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + // transaction executes a function within a database transaction. If the // function returns an error, the transaction is rolled back. Otherwise, the // transaction is committed. If the transaction fails due to a busy error, it is @@ -55,11 +60,6 @@ func (s *Store) transaction(fn func(*txn) error) error { return fmt.Errorf("transaction failed (attempt %d): %w", attempt, err) } -// Close closes the underlying database. -func (s *Store) Close() error { - return s.db.Close() -} - func sqliteFilepath(fp string) string { params := []string{ fmt.Sprintf("_busy_timeout=%d", busyTimeout), diff --git a/wallet/manager.go b/wallet/manager.go index 308c519..51fbc1e 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -191,12 +191,20 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) if err != nil { return nil, fmt.Errorf("failed to get last committed index: %w", err) } - if err := syncStore(store, cm, lastTip); err != nil { - return nil, fmt.Errorf("failed to subscribe to chain manager: %w", err) - } - reorgChan := make(chan types.ChainIndex, 1) go func() { + if err := syncStore(store, cm, lastTip); err != nil { + log.Fatal("failed to subscribe to chain manager", zap.Error(err)) + } + + reorgChan := make(chan types.ChainIndex, 1) + m.unsubscribe = cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgChan <- index: + default: + } + }) + for range reorgChan { m.mu.Lock() lastTip, err := store.LastCommittedIndex() @@ -208,11 +216,5 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) m.mu.Unlock() } }() - m.unsubscribe = cm.OnReorg(func(index types.ChainIndex) { - select { - case reorgChan <- index: - default: - } - }) return m, nil } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go new file mode 100644 index 0000000..b86f26e --- /dev/null +++ b/wallet/wallet_test.go @@ -0,0 +1,171 @@ +package wallet_test + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" +) + +func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { + for i := 0; i < 1000; i++ { + time.Sleep(10 * time.Millisecond) + tip, _ := ws.LastCommittedIndex() + if tip == cm.Tip() { + return + } + } + tb.Fatal("timed out waiting for block") +} + +func TestResubscribe(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testutil.Network() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + checkBalance := func(siacoin, immature types.Currency, siafund uint64) error { + waitForBlock(t, cm, db) + + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if b.Siafunds != siafund { + return fmt.Errorf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } + return nil + } + + // check that the wallet has no balance + if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } + + expectedBalance1 := cm.TipState().BlockReward() + // mine a block to fund the first address + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr)}); err != nil { + t.Fatal(err) + } + + // mine a block to fund the second address + expectedBalance2 := cm.TipState().BlockReward() + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr2)}); err != nil { + t.Fatal(err) + } + + // check that the wallet has one immature payout + if err := checkBalance(types.ZeroCurrency, expectedBalance1, 0); err != nil { + t.Fatal(err) + } + + // mine until the first payout matures + for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } + + // resubscribe the wallet + if err := wm.Subscribe(0); err != nil { + t.Fatal(err) + } + + // check that the wallet balance did not change + if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } + + // add the second address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } else if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } + + // resubscribe + if err := wm.Subscribe(0); err != nil { + t.Fatal(err) + } + + if err := checkBalance(expectedBalance1, expectedBalance2, 0); err != nil { + t.Fatal(err) + } + + // mine a block to mature the second payout + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } + + // sanity check + if err := wm.Subscribe(0); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency, 0); err != nil { + t.Fatal(err) + } +} From 87d866ed26ce14884124f7298fe95799f3f16ae6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 8 Mar 2024 13:17:02 -0800 Subject: [PATCH 136/630] cmd: remove unused commands, fix miner logic --- cmd/walletd/main.go | 39 ++------------- cmd/walletd/miner.go | 108 +++++++++++++++++++++++++++++++++++++++++ cmd/walletd/node.go | 11 +++-- cmd/walletd/testnet.go | 100 -------------------------------------- 4 files changed, 120 insertions(+), 138 deletions(-) create mode 100644 cmd/walletd/miner.go diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index acedcd8..181035e 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -92,28 +92,7 @@ Generates a secure testnet seed. mineUsage = `Usage: walletd mine -Runs a testnet CPU miner. -` - balanceUsage = `Usage: - walletd balance - -Displays testnet balance. -` - sendUsage = `Usage: - walletd send [flags] [amount] [address] - -Sends a simple testnet transaction. -` - txnsUsage = `Usage: - walletd txns - -Lists testnet transactions and miner rewards. -` - txpoolUsage = `Usage: - walletd txpool - -Lists unconfirmed testnet transactions in the txpool. -Note that only transactions relevant to the wallet are shown. +Runs a CPU miner. Not intended for production use. ` ) @@ -121,7 +100,7 @@ func main() { log.SetFlags(0) var gatewayAddr, apiAddr, dir, network, seed string - var upnp, v2 bool + var upnp, bootstrap bool var minerAddrStr string var minerBlocks int @@ -133,17 +112,13 @@ func main() { rootCmd.StringVar(&dir, "dir", ".", "directory to store node state in") rootCmd.StringVar(&network, "network", "mainnet", "network to connect to") rootCmd.BoolVar(&upnp, "upnp", true, "attempt to forward ports and discover IP with UPnP") + rootCmd.BoolVar(&bootstrap, "bootstrap", true, "attempt to bootstrap the network") rootCmd.StringVar(&seed, "seed", "", "testnet seed") versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) mineCmd := flagg.New("mine", mineUsage) mineCmd.IntVar(&minerBlocks, "n", -1, "mine this many blocks. If negative, mine indefinitely") mineCmd.StringVar(&minerAddrStr, "addr", "", "address to send block rewards to (required)") - balanceCmd := flagg.New("balance", balanceUsage) - sendCmd := flagg.New("send", sendUsage) - sendCmd.BoolVar(&v2, "v2", false, "send a v2 transaction") - txnsCmd := flagg.New("txns", txnsUsage) - txpoolCmd := flagg.New("txpool", txpoolUsage) cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, @@ -151,10 +126,6 @@ func main() { {Cmd: versionCmd}, {Cmd: seedCmd}, {Cmd: mineCmd}, - {Cmd: balanceCmd}, - {Cmd: sendCmd}, - {Cmd: txnsCmd}, - {Cmd: txpoolCmd}, }, }) @@ -195,7 +166,7 @@ func main() { // redirect stdlib log to zap zap.RedirectStdLog(logger.Named("stdlib")) - n, err := newNode(gatewayAddr, dir, network, upnp, logger) + n, err := newNode(gatewayAddr, dir, network, upnp, bootstrap, logger) if err != nil { log.Fatal(err) } @@ -241,6 +212,6 @@ func main() { } c := api.NewClient("http://"+apiAddr+"/api", getAPIPassword()) - runTestnetMiner(c, minerAddr, minerBlocks) + runCPUMiner(c, minerAddr, minerBlocks) } } diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go new file mode 100644 index 0000000..bbe6336 --- /dev/null +++ b/cmd/walletd/miner.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/binary" + "fmt" + "log" + "math/big" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/walletd/api" + "lukechampine.com/frand" +) + +func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { + buf := make([]byte, 32+8+8+32) + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) + if b.V2 != nil { + copy(buf[:32], "sia/id/block|") + copy(buf[48:], b.V2.Commitment[:]) + } else { + root := b.MerkleRoot() + copy(buf[:32], b.ParentID[:]) + copy(buf[48:], root[:]) + } + factor := cs.NonceFactor() + startBlock := time.Now() + for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { + b.Nonce += factor + hashes++ + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + if time.Since(startBlock) > 10*time.Second { + return hashes, false + } + } + return hashes, true +} + +func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { + log.Println("Started mining into", minerAddr) + start := time.Now() + + var hashes float64 + var blocks uint64 + var last types.ChainIndex +outer: + for i := 0; ; i++ { + if n >= 0 && i >= n { + return + } + elapsed := time.Since(start) + cs, err := c.ConsensusTipState() + check("Couldn't get consensus tip state:", err) + if cs.Index == last { + fmt.Println("Tip now", cs.Index) + last = cs.Index + } + n := big.NewInt(int64(hashes)) + n.Mul(n, big.NewInt(int64(24*time.Hour))) + d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) + d.Mul(d, big.NewInt(int64(1+elapsed))) + r, _ := new(big.Rat).SetFrac(n, d).Float64() + fmt.Printf("\rMining block %4v...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", cs.Index.Height+1, hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) + + txns, v2txns, err := c.TxpoolTransactions() + check("Couldn't get txpool transactions:", err) + b := types.Block{ + ParentID: cs.Index.ID, + Nonce: cs.NonceFactor() * frand.Uint64n(100), + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: cs.BlockReward()}}, + Transactions: txns, + } + for _, txn := range txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) + } + for _, txn := range v2txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) + } + if len(v2txns) > 0 || cs.Index.Height+1 >= cs.Network.HardforkV2.RequireHeight { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + Transactions: v2txns, + } + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + } + h, ok := mineBlock(cs, &b) + hashes += float64(h) + if !ok { + continue outer + } + blocks++ + index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} + tip, err := c.ConsensusTip() + check("Couldn't get consensus tip:", err) + if tip != cs.Index { + fmt.Printf("\nMined %v but tip changed, starting over\n", index) + } else if err := c.SyncerBroadcastBlock(b); err != nil { + fmt.Printf("\nMined invalid block: %v\n", err) + } else if b.V2 == nil { + fmt.Printf("\nFound v1 block %v\n", index) + } else { + fmt.Printf("\nFound v2 block %v\n", index) + } + } +} diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index b2a7554..83ece3d 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -99,7 +99,7 @@ func (n *node) Close() error { return n.store.Close() } -func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logger) (*node, error) { +func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, log *zap.Logger) (*node, error) { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -166,11 +166,14 @@ func newNode(addr, dir string, chainNetwork string, useUPNP bool, log *zap.Logge return nil, fmt.Errorf("failed to open wallet database: %w", err) } - for _, peer := range bootstrapPeers { - if err := store.AddPeer(peer); err != nil { - return nil, fmt.Errorf("failed to add bootstrap peer '%s': %w", peer, err) + if useBootstrap { + for _, peer := range bootstrapPeers { + if err := store.AddPeer(peer); err != nil { + return nil, fmt.Errorf("failed to add bootstrap peer '%s': %w", peer, err) + } } } + header := gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go index 64ff63c..dc9c706 100644 --- a/cmd/walletd/testnet.go +++ b/cmd/walletd/testnet.go @@ -1,16 +1,10 @@ package main import ( - "encoding/binary" - "fmt" - "log" - "math/big" "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" - "go.sia.tech/walletd/api" - "lukechampine.com/frand" ) // TestnetAnagami returns the chain parameters and genesis block for the "Anagami" @@ -63,97 +57,3 @@ func TestnetAnagami() (*consensus.Network, types.Block) { return n, b } - -func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { - buf := make([]byte, 32+8+8+32) - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) - if b.V2 != nil { - copy(buf[:32], "sia/id/block|") - copy(buf[48:], b.V2.Commitment[:]) - } else { - root := b.MerkleRoot() - copy(buf[:32], b.ParentID[:]) - copy(buf[48:], root[:]) - } - factor := cs.NonceFactor() - startBlock := time.Now() - for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { - b.Nonce += factor - hashes++ - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - if time.Since(startBlock) > 10*time.Second { - return hashes, false - } - } - return hashes, true -} - -func runTestnetMiner(c *api.Client, minerAddr types.Address, n int) { - log.Println("Started mining into", minerAddr) - start := time.Now() - - var hashes float64 - var blocks uint64 - var last types.ChainIndex -outer: - for i := 0; ; i++ { - if n <= 0 && i >= n { - return - } - elapsed := time.Since(start) - cs, err := c.ConsensusTipState() - check("Couldn't get consensus tip state:", err) - if cs.Index == last { - fmt.Println("Tip now", cs.Index) - last = cs.Index - } - n := big.NewInt(int64(hashes)) - n.Mul(n, big.NewInt(int64(24*time.Hour))) - d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) - d.Mul(d, big.NewInt(int64(1+elapsed))) - r, _ := new(big.Rat).SetFrac(n, d).Float64() - fmt.Printf("\rMining block %4v...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", cs.Index.Height+1, hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) - - txns, v2txns, err := c.TxpoolTransactions() - check("Couldn't get txpool transactions:", err) - b := types.Block{ - ParentID: cs.Index.ID, - Nonce: cs.NonceFactor() * frand.Uint64n(100), - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: cs.BlockReward()}}, - Transactions: txns, - } - for _, txn := range txns { - b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) - } - for _, txn := range v2txns { - b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) - } - if len(v2txns) > 0 || cs.Index.Height+1 >= cs.Network.HardforkV2.RequireHeight { - b.V2 = &types.V2BlockData{ - Height: cs.Index.Height + 1, - Transactions: v2txns, - } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - } - h, ok := mineBlock(cs, &b) - hashes += float64(h) - if !ok { - continue outer - } - blocks++ - index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} - tip, err := c.ConsensusTip() - check("Couldn't get consensus tip:", err) - if tip != cs.Index { - fmt.Printf("\nMined %v but tip changed, starting over\n", index) - } else if err := c.SyncerBroadcastBlock(b); err != nil { - fmt.Printf("\nMined invalid block: %v\n", err) - } else if b.V2 == nil { - fmt.Printf("\nFound v1 block %v\n", index) - } else { - fmt.Printf("\nFound v2 block %v\n", index) - } - } -} From b96d0639129438ab66ba2490a0f0a832634614d4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 8 Mar 2024 13:27:31 -0800 Subject: [PATCH 137/630] cmd: make seed consistent with renterd and hostd --- cmd/walletd/main.go | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 181035e..e08dfd3 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -7,16 +7,14 @@ import ( "os" "os/signal" "runtime/debug" - "strings" "go.sia.tech/core/types" + cwallet "go.sia.tech/coreutils/wallet" "go.sia.tech/walletd/api" - "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/term" "lukechampine.com/flagg" - "lukechampine.com/frand" ) var commit = "?" @@ -71,14 +69,9 @@ Run 'walletd' with no arguments to start the blockchain node and API server. Actions: version print walletd version + seed generate a recovery phrase + mine run CPU miner` -Testnet Actions: - seed generate a seed - mine run CPU miner - balance view wallet balance - send send a simple transaction - txns view transaction history -` versionUsage = `Usage: walletd version @@ -87,7 +80,7 @@ Prints the version of the walletd binary. seedUsage = `Usage: walletd seed -Generates a secure testnet seed. +Generates a secure BIP-39 recovery phrase. ` mineUsage = `Usage: walletd mine @@ -193,13 +186,15 @@ func main() { cmd.Usage() return } - seed := frand.Bytes(8) - var entropy [32]byte - copy(entropy[:], seed) - addr := types.StandardUnlockHash(wallet.NewSeedFromEntropy(&entropy).PublicKey(0)) - fmt.Printf("Seed: %x\n", seed) - fmt.Printf("Address: %v\n", strings.TrimPrefix(addr.String(), "addr:")) + recoveryPhrase := cwallet.NewSeedPhrase() + var seed [32]byte + if err := cwallet.SeedFromPhrase(&seed, recoveryPhrase); err != nil { + log.Fatal(err) + } + addr := types.StandardUnlockHash(cwallet.KeyFromSeed(&seed, 0).PublicKey()) + fmt.Println("Recovery Phrase:", recoveryPhrase) + fmt.Println("Address", addr) case mineCmd: if len(cmd.Args()) != 0 { cmd.Usage() From d2e782855031c42ae69fd2168295468b0c396189 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 1 Apr 2024 17:07:56 -0700 Subject: [PATCH 138/630] cmd: use coreutils.FindBlockNonce --- cmd/walletd/miner.go | 51 ++++++++------------------------------------ 1 file changed, 9 insertions(+), 42 deletions(-) diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index bbe6336..157387d 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -1,54 +1,26 @@ package main import ( - "encoding/binary" "fmt" "log" "math/big" "time" - "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils" "go.sia.tech/walletd/api" "lukechampine.com/frand" ) -func mineBlock(cs consensus.State, b *types.Block) (hashes int, found bool) { - buf := make([]byte, 32+8+8+32) - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) - if b.V2 != nil { - copy(buf[:32], "sia/id/block|") - copy(buf[48:], b.V2.Commitment[:]) - } else { - root := b.MerkleRoot() - copy(buf[:32], b.ParentID[:]) - copy(buf[48:], root[:]) - } - factor := cs.NonceFactor() - startBlock := time.Now() - for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { - b.Nonce += factor - hashes++ - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - if time.Since(startBlock) > 10*time.Second { - return hashes, false - } - } - return hashes, true -} - func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { log.Println("Started mining into", minerAddr) start := time.Now() - var hashes float64 - var blocks uint64 var last types.ChainIndex -outer: - for i := 0; ; i++ { - if n >= 0 && i >= n { - return + var blocksFound int + for { + if n > 0 && blocksFound >= n { + break } elapsed := time.Since(start) cs, err := c.ConsensusTipState() @@ -57,12 +29,9 @@ outer: fmt.Println("Tip now", cs.Index) last = cs.Index } - n := big.NewInt(int64(hashes)) - n.Mul(n, big.NewInt(int64(24*time.Hour))) d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) d.Mul(d, big.NewInt(int64(1+elapsed))) - r, _ := new(big.Rat).SetFrac(n, d).Float64() - fmt.Printf("\rMining block %4v...(%.2f kH/s, %.2f blocks/day (expected: %.2f), difficulty %v)", cs.Index.Height+1, hashes/elapsed.Seconds()/1000, float64(blocks)*float64(24*time.Hour)/float64(elapsed), r, cs.Difficulty) + fmt.Printf("\rMining block %4v...(%.2f blocks/day), difficulty %v)", cs.Index.Height+1, float64(blocksFound)*float64(24*time.Hour)/float64(elapsed), cs.Difficulty) txns, v2txns, err := c.TxpoolTransactions() check("Couldn't get txpool transactions:", err) @@ -86,12 +55,10 @@ outer: } b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) } - h, ok := mineBlock(cs, &b) - hashes += float64(h) - if !ok { - continue outer + if !coreutils.FindBlockNonce(cs, &b, time.Minute) { + continue } - blocks++ + blocksFound++ index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} tip, err := c.ConsensusTip() check("Couldn't get consensus tip:", err) From dfc076e185cf626bf769ccf2b231c2b26594a289 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 06:51:05 -0700 Subject: [PATCH 139/630] address comments --- cmd/walletd/miner.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index 157387d..a531fcd 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -16,19 +16,14 @@ func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { log.Println("Started mining into", minerAddr) start := time.Now() - var last types.ChainIndex var blocksFound int for { - if n > 0 && blocksFound >= n { + if n >= 0 && blocksFound >= n { break } elapsed := time.Since(start) cs, err := c.ConsensusTipState() check("Couldn't get consensus tip state:", err) - if cs.Index == last { - fmt.Println("Tip now", cs.Index) - last = cs.Index - } d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) d.Mul(d, big.NewInt(int64(1+elapsed))) fmt.Printf("\rMining block %4v...(%.2f blocks/day), difficulty %v)", cs.Index.Height+1, float64(blocksFound)*float64(24*time.Hour)/float64(elapsed), cs.Difficulty) From 5da40f9f30e424812ab3dfee11ef537071908452 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 12:19:33 -0700 Subject: [PATCH 140/630] build: add build info --- build/build.go | 21 +++++++++ build/gen.go | 114 +++++++++++++++++++++++++++++++++++++++++++++++++ build/meta.go | 7 +++ 3 files changed, 142 insertions(+) create mode 100644 build/build.go create mode 100644 build/gen.go create mode 100644 build/meta.go diff --git a/build/build.go b/build/build.go new file mode 100644 index 0000000..119b32f --- /dev/null +++ b/build/build.go @@ -0,0 +1,21 @@ +// Package build contains build-time information. +package build + +//go:generate go run gen.go + +import "time" + +// Commit returns the commit hash of walletd +func Commit() string { + return commit +} + +// Version returns the version of walletd +func Version() string { + return version +} + +// Time returns the time at which the binary was built. +func Time() time.Time { + return time.Unix(buildTime, 0) +} diff --git a/build/gen.go b/build/gen.go new file mode 100644 index 0000000..3bf7dff --- /dev/null +++ b/build/gen.go @@ -0,0 +1,114 @@ +//go:build ignore + +// This script generates meta.go which contains version info for the walletd binary. It can be run with `go generate`. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "strings" + "text/template" + "time" +) + +const logFormat = `{%n "commit": "%H",%n "shortCommit": "%h",%n "timestamp": "%cD",%n "tag": "%(describe:tags=true)"%n}` + +type ( + gitTime time.Time + + gitMeta struct { + Commit string `json:"commit"` + ShortCommit string `json:"shortCommit"` + Timestamp gitTime `json:"timestamp"` + Tag string `json:"tag"` + } +) + +var buildTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. +// This file was generated by go generate at {{ .RunTime }}. +package build + +const ( + commit = "{{ .Commit }}" + version = "{{ .Version }}" + buildTime = {{ .UnixTimestamp }} +) +`)) + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (t *gitTime) UnmarshalJSON(buf []byte) error { + timeFormats := []string{ + time.RFC1123Z, + "Mon, 2 Jan 2006 15:04:05 -0700", + "2006-01-02 15:04:05 -0700", + time.UnixDate, + time.ANSIC, + time.RFC3339, + time.RFC1123, + } + + for _, format := range timeFormats { + parsed, err := time.Parse(format, strings.Trim(string(buf), `"`)) + if err == nil { + *t = gitTime(parsed) + return nil + } + } + return errors.New("failed to parse time") +} + +func getGitMeta() (meta gitMeta, _ error) { + cmd := exec.Command("git", "log", "-1", "--pretty=format:"+logFormat+"") + buf, err := cmd.Output() + if err != nil { + if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 { + return gitMeta{}, fmt.Errorf("command failed: %w", errors.New(string(err.Stderr))) + } + return gitMeta{}, fmt.Errorf("failed to execute command: %w", err) + } else if err := json.Unmarshal(buf, &meta); err != nil { + return gitMeta{}, fmt.Errorf("failed to unmarshal json: %w", err) + } + return +} + +func main() { + meta, err := getGitMeta() + if err != nil { + log.Fatalln(err) + } + + commit := meta.ShortCommit + version := meta.Tag + if len(version) == 0 { + // no version, use commit and current time for development + version = commit + meta.Timestamp = gitTime(time.Now()) + } + + f, err := os.Create("meta.go") + if err != nil { + log.Fatalln(err) + } + defer f.Close() + + err = buildTemplate.Execute(f, struct { + Commit string + Version string + UnixTimestamp int64 + + RunTime string + }{ + Commit: commit, + Version: version, + UnixTimestamp: time.Time(meta.Timestamp).Unix(), + + RunTime: time.Now().Format(time.RFC3339), + }) + if err != nil { + log.Fatalln(err) + } +} diff --git a/build/meta.go b/build/meta.go new file mode 100644 index 0000000..05dd9fc --- /dev/null +++ b/build/meta.go @@ -0,0 +1,7 @@ +package build + +const ( + commit = "?" + version = "?" + buildTime = 0 +) From 85dbe5db7fbc97e311609d2842ce48ff940cf35f Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 12:19:40 -0700 Subject: [PATCH 141/630] api: add state endpoint --- api/api.go | 10 ++++++++ api/client.go | 6 +++++ api/server.go | 64 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/api/api.go b/api/api.go index 6f86e19..d3f0fd1 100644 --- a/api/api.go +++ b/api/api.go @@ -8,6 +8,16 @@ import ( "go.sia.tech/walletd/wallet" ) +// A StateResponse returns information about the current state of the walletd +// daemon. +type StateResponse struct { + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + BuildTime time.Time `json:"buildTime"` + StartTime time.Time `json:"startTime"` +} + // A GatewayPeer is a currently-connected peer. type GatewayPeer struct { Addr string `json:"addr"` diff --git a/api/client.go b/api/client.go index 1e614d3..d9f5a99 100644 --- a/api/client.go +++ b/api/client.go @@ -16,6 +16,12 @@ type Client struct { n *consensus.Network // for ConsensusTipState } +// State returns information about the current state of the walletd daemon. +func (c *Client) State() (resp StateResponse, err error) { + err = c.c.GET("/state", &resp) + return +} + // TxpoolBroadcast broadcasts a set of transaction to the network. func (c *Client) TxpoolBroadcast(txns []types.Transaction, v2txns []types.V2Transaction) (err error) { err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) diff --git a/api/server.go b/api/server.go index 007a23c..d202ba4 100644 --- a/api/server.go +++ b/api/server.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "reflect" + "runtime" "sync" "time" @@ -15,6 +16,7 @@ import ( "go.sia.tech/core/gateway" "go.sia.tech/core/types" "go.sia.tech/coreutils/syncer" + "go.sia.tech/walletd/build" "go.sia.tech/walletd/wallet" ) @@ -70,6 +72,8 @@ type ( } ) +var startTime = time.Now() + type server struct { cm ChainManager s Syncer @@ -80,6 +84,16 @@ type server struct { used map[types.Hash256]bool } +func (s *server) stateHandler(jc jape.Context) { + jc.Encode(StateResponse{ + Version: build.Version(), + Commit: build.Commit(), + OS: runtime.GOOS, + BuildTime: build.Time(), + StartTime: startTime, + }) +} + func (s *server) consensusNetworkHandler(jc jape.Context) { jc.Encode(*s.cm.TipState().Network) } @@ -628,36 +642,38 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { used: make(map[types.Hash256]bool), } return jape.Mux(map[string]jape.Handler{ - "GET /consensus/network": srv.consensusNetworkHandler, - "GET /consensus/tip": srv.consensusTipHandler, - "GET /consensus/tipstate": srv.consensusTipStateHandler, + "GET /state": srv.stateHandler, + + "GET /consensus/network": srv.consensusNetworkHandler, + "GET /consensus/tip": srv.consensusTipHandler, + "GET /consensus/tipstate": srv.consensusTipStateHandler, - "GET /syncer/peers": srv.syncerPeersHandler, - "POST /syncer/connect": srv.syncerConnectHandler, - "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, + "GET /syncer/peers": srv.syncerPeersHandler, + "POST /syncer/connect": srv.syncerConnectHandler, + "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, - "GET /txpool/transactions": srv.txpoolTransactionsHandler, - "GET /txpool/fee": srv.txpoolFeeHandler, - "POST /txpool/broadcast": srv.txpoolBroadcastHandler, + "GET /txpool/transactions": srv.txpoolTransactionsHandler, + "GET /txpool/fee": srv.txpoolFeeHandler, + "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - "POST /resubscribe": srv.resubscribeHandler, + "POST /resubscribe": srv.resubscribeHandler, - "GET /wallets": srv.walletsHandler, + "GET /wallets": srv.walletsHandler, "POST /wallets": srv.walletsHandlerPOST, - "POST /wallets/:id": srv.walletsIDHandlerPOST, - "DELETE /wallets/:id": srv.walletsIDHandlerDELETE, - "PUT /wallets/:id/addresses": srv.walletsAddressHandlerPUT, + "POST /wallets/:id": srv.walletsIDHandlerPOST, + "DELETE /wallets/:id": srv.walletsIDHandlerDELETE, + "PUT /wallets/:id/addresses": srv.walletsAddressHandlerPUT, "DELETE /wallets/:id/addresses/:addr": srv.walletsAddressHandlerDELETE, - "GET /wallets/:id/addresses": srv.walletsAddressesHandlerGET, - "GET /wallets/:id/balance": srv.walletsBalanceHandler, - "GET /wallets/:id/events": srv.walletsEventsHandler, - "GET /wallets/:id/txpool": srv.walletsTxpoolHandler, - "GET /wallets/:id/outputs/siacoin": srv.walletsOutputsSiacoinHandler, - "GET /wallets/:id/outputs/siafund": srv.walletsOutputsSiafundHandler, - "POST /wallets/:id/reserve": srv.walletsReserveHandler, - "POST /wallets/:id/release": srv.walletsReleaseHandler, - "POST /wallets/:id/fund": srv.walletsFundHandler, - "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, + "GET /wallets/:id/addresses": srv.walletsAddressesHandlerGET, + "GET /wallets/:id/balance": srv.walletsBalanceHandler, + "GET /wallets/:id/events": srv.walletsEventsHandler, + "GET /wallets/:id/txpool": srv.walletsTxpoolHandler, + "GET /wallets/:id/outputs/siacoin": srv.walletsOutputsSiacoinHandler, + "GET /wallets/:id/outputs/siafund": srv.walletsOutputsSiafundHandler, + "POST /wallets/:id/reserve": srv.walletsReserveHandler, + "POST /wallets/:id/release": srv.walletsReleaseHandler, + "POST /wallets/:id/fund": srv.walletsFundHandler, + "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, "GET /addresses/:addr/balance": srv.addressesAddrBalanceHandler, "GET /addresses/:addr/events": srv.addressesAddrEventsHandler, From 9ad3e3ff1af5efaedf06abf1a1e8d8c96919581d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 12:23:24 -0700 Subject: [PATCH 142/630] docker: add go generate step --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index ac7f0db..e803063 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,17 @@ FROM docker.io/library/golang:1.21 AS builder WORKDIR /walletd +# Install dependencies +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source COPY . . # Enable CGO for sqlite3 support ENV CGO_ENABLED=1 +RUN go generate ./... RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd FROM docker.io/library/alpine:3 From 77b901ddf72408e3b4f93f29a7b57abf2ad45ea6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 12:23:49 -0700 Subject: [PATCH 143/630] cmd: use build package for version info --- cmd/walletd/main.go | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index e08dfd3..ed1709d 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -6,41 +6,17 @@ import ( "net" "os" "os/signal" - "runtime/debug" "go.sia.tech/core/types" cwallet "go.sia.tech/coreutils/wallet" "go.sia.tech/walletd/api" + "go.sia.tech/walletd/build" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/term" "lukechampine.com/flagg" ) -var commit = "?" -var timestamp = "?" - -func init() { - info, ok := debug.ReadBuildInfo() - if !ok { - return - } - modified := false - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - commit = setting.Value[:8] - case "vcs.time": - timestamp = setting.Value - case "vcs.modified": - modified = setting.Value == "true" - } - } - if modified { - commit += " (modified)" - } -} - func check(context string, err error) { if err != nil { log.Fatalf("%v: %v", context, err) @@ -122,7 +98,7 @@ func main() { }, }) - log.Println("walletd v0.1.0") + log.Println("walletd", build.Version()) switch cmd { case rootCmd: if len(cmd.Args()) != 0 { @@ -178,9 +154,8 @@ func main() { cmd.Usage() return } - log.Println("Commit Hash:", commit) - log.Println("Commit Date:", timestamp) - + log.Println("Commit Hash:", build.Commit()) + log.Println("Commit Date:", build.Time()) case seedCmd: if len(cmd.Args()) != 0 { cmd.Usage() From 7afa241f584d4c0714a136b5668f9defc404b0d4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 13:11:11 -0700 Subject: [PATCH 144/630] sqlite: move some peer fields into memory only --- cmd/walletd/node.go | 8 ++- persist/sqlite/init.sql | 5 +- persist/sqlite/peers.go | 122 +++++++++++++++++++++++------------ persist/sqlite/peers_test.go | 30 ++++++--- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 83ece3d..831cba4 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -174,12 +174,18 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, } } + ps, err := sqlite.NewPeerStore(store) + if err != nil { + return nil, fmt.Errorf("failed to create peer store: %w", err) + } + header := gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - s := syncer.New(l, cm, store, header, syncer.WithLogger(log.Named("syncer"))) + + s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) wm, err := wallet.NewManager(cm, store, log.Named("wallet")) if err != nil { return nil, fmt.Errorf("failed to create wallet manager: %w", err) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 77715fc..3f399e5 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -75,10 +75,7 @@ CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); CREATE TABLE syncer_peers ( peer_address TEXT PRIMARY KEY NOT NULL, - first_seen INTEGER NOT NULL, - last_connect INTEGER NOT NULL, - synced_blocks INTEGER NOT NULL, - sync_duration INTEGER NOT NULL + first_seen INTEGER NOT NULL ); CREATE TABLE syncer_bans ( diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index 3ba38ac..a626908 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -7,32 +7,102 @@ import ( "net" "strconv" "strings" + "sync" "time" "go.sia.tech/coreutils/syncer" "go.uber.org/zap" ) -func scanPeerInfo(s scanner) (pi syncer.PeerInfo, err error) { - err = s.Scan(&pi.Address, decode(&pi.FirstSeen), decode(&pi.LastConnect), &pi.SyncedBlocks, &pi.SyncDuration) - return +// A PeerStore stores information about peers. +type PeerStore struct { + s *Store + + // session-specific peer info is stored in memory to reduce write load + // on the database + mu sync.Mutex + peerInfo map[string]syncer.PeerInfo +} + +// AddPeer adds the given peer to the store. +func (ps *PeerStore) AddPeer(peer string) error { + ps.mu.Lock() + defer ps.mu.Unlock() + ps.peerInfo[peer] = syncer.PeerInfo{ + Address: peer, + FirstSeen: time.Now(), + } + return ps.s.AddPeer(peer) +} + +// Peers returns the addresses of all known peers. +func (ps *PeerStore) Peers() ([]syncer.PeerInfo, error) { + ps.mu.Lock() + defer ps.mu.Unlock() + + // copy the map to a slice + peers := make([]syncer.PeerInfo, 0, len(ps.peerInfo)) + for _, pi := range ps.peerInfo { + peers = append(peers, pi) + } + return peers, nil +} + +// UpdatePeerInfo updates the information for the given peer. +func (ps *PeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) error { + ps.mu.Lock() + defer ps.mu.Unlock() + if pi, ok := ps.peerInfo[peer]; !ok { + return syncer.ErrPeerNotFound + } else { + fn(&pi) + ps.peerInfo[peer] = pi + } + return nil +} + +// Ban temporarily bans the given peer. +func (ps *PeerStore) Ban(peer string, duration time.Duration, reason string) error { + return ps.s.Ban(peer, duration, reason) +} + +// Banned returns true if the peer is banned. +func (ps *PeerStore) Banned(peer string) (bool, error) { + return ps.s.Banned(peer) } -func getPeerInfo(tx *txn, peer string) (syncer.PeerInfo, error) { - const query = `SELECT peer_address, first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers WHERE peer_address=$1` - return scanPeerInfo(tx.QueryRow(query, peer)) +// PeerInfo returns the information for the given peer. +func (ps *PeerStore) PeerInfo(peer string) (syncer.PeerInfo, error) { + ps.mu.Lock() + defer ps.mu.Unlock() + if pi, ok := ps.peerInfo[peer]; ok { + return pi, nil + } + return syncer.PeerInfo{}, syncer.ErrPeerNotFound +} + +// NewPeerStore creates a new peer store using the given store. +func NewPeerStore(s *Store) (syncer.PeerStore, error) { + ps := &PeerStore{s: s, peerInfo: make(map[string]syncer.PeerInfo)} + peers, err := s.Peers() + if err != nil { + return nil, fmt.Errorf("failed to load peers: %w", err) + } + for _, pi := range peers { + ps.peerInfo[pi.Address] = pi + } + return ps, nil } -func (s *Store) updatePeerInfo(tx *txn, peer string, info syncer.PeerInfo) error { - const query = `UPDATE syncer_peers SET first_seen=$1, last_connect=$2, synced_blocks=$3, sync_duration=$4 WHERE peer_address=$5 RETURNING peer_address` - err := tx.QueryRow(query, encode(info.FirstSeen), encode(info.LastConnect), info.SyncedBlocks, info.SyncDuration, peer).Scan(&peer) - return err +func scanPeerInfo(s scanner) (pi syncer.PeerInfo, err error) { + err = s.Scan(&pi.Address, decode(&pi.FirstSeen)) + return } // AddPeer adds the given peer to the store. func (s *Store) AddPeer(peer string) error { return s.transaction(func(tx *txn) error { - const query = `INSERT INTO syncer_peers (peer_address, first_seen, last_connect, synced_blocks, sync_duration) VALUES ($1, $2, 0, 0, 0) ON CONFLICT (peer_address) DO NOTHING` + const query = `INSERT INTO syncer_peers (peer_address, first_seen) VALUES ($1, $2) ON CONFLICT (peer_address) DO NOTHING` _, err := tx.Exec(query, peer, encode(time.Now())) return err }) @@ -41,7 +111,7 @@ func (s *Store) AddPeer(peer string) error { // Peers returns the addresses of all known peers. func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { err := s.transaction(func(tx *txn) error { - const query = `SELECT peer_address, first_seen, last_connect, synced_blocks, sync_duration FROM syncer_peers` + const query = `SELECT peer_address, first_seen FROM syncer_peers` rows, err := tx.Query(query) if err != nil { return err @@ -59,34 +129,6 @@ func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { return peers, err } -// UpdatePeerInfo updates the info for the given peer. -func (s *Store) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) error { - return s.transaction(func(tx *txn) error { - info, err := getPeerInfo(tx, peer) - if err != nil { - return fmt.Errorf("failed to get peer info: %w", err) - } - fn(&info) - return s.updatePeerInfo(tx, peer, info) - }) -} - -// PeerInfo returns the info for the given peer. -func (s *Store) PeerInfo(peer string) (syncer.PeerInfo, error) { - var info syncer.PeerInfo - var err error - err = s.transaction(func(tx *txn) error { - info, err = getPeerInfo(tx, peer) - return err - }) - if errors.Is(err, sql.ErrNoRows) { - return syncer.PeerInfo{}, syncer.ErrPeerNotFound - } else if err != nil { - return syncer.PeerInfo{}, err - } - return info, nil -} - // normalizePeer normalizes a peer address to a CIDR subnet. func normalizePeer(peer string) (string, error) { host, _, err := net.SplitHostPort(peer) diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 62fe6ad..135ad26 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -18,9 +18,14 @@ func TestAddPeer(t *testing.T) { } defer db.Close() + ps, err := NewPeerStore(db) + if err != nil { + t.Fatal(err) + } + const peer = "1.2.3.4:9981" - if err := db.AddPeer(peer); err != nil { + if err := ps.AddPeer(peer); err != nil { t.Fatal(err) } @@ -28,7 +33,7 @@ func TestAddPeer(t *testing.T) { syncedBlocks := uint64(15) syncDuration := 5 * time.Second - err = db.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { + err = ps.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { info.LastConnect = lastConnect info.SyncedBlocks = syncedBlocks info.SyncDuration = syncDuration @@ -37,7 +42,7 @@ func TestAddPeer(t *testing.T) { t.Fatal(err) } - info, err := db.PeerInfo(peer) + info, err := ps.PeerInfo(peer) if err != nil { t.Fatal(err) } @@ -52,7 +57,7 @@ func TestAddPeer(t *testing.T) { t.Errorf("expected SyncDuration = %s; got %s", syncDuration, info.SyncDuration) } - peers, err := db.Peers() + peers, err := ps.Peers() if err != nil { t.Fatal(err) } else if len(peers) != 1 { @@ -78,23 +83,28 @@ func TestBanPeer(t *testing.T) { } defer db.Close() + ps, err := NewPeerStore(db) + if err != nil { + t.Fatal(err) + } + const peer = "1.2.3.4" - if banned, err := db.Banned(peer); err != nil || banned { + if banned, err := ps.Banned(peer); err != nil || banned { t.Fatal("expected peer to not be banned", err) } // ban the peer - db.Ban(peer, time.Second, "test") + ps.Ban(peer, time.Second, "test") - if banned, err := db.Banned(peer); err != nil || !banned { + if banned, err := ps.Banned(peer); err != nil || !banned { t.Fatal("expected peer to be banned", err) } // wait for the ban to expire time.Sleep(time.Second) - if banned, err := db.Banned(peer); err != nil || banned { + if banned, err := ps.Banned(peer); err != nil || banned { t.Fatal("expected peer to not be banned", err) } @@ -105,8 +115,8 @@ func TestBanPeer(t *testing.T) { } t.Log("banning", subnet) - db.Ban(subnet.String(), time.Second, "test") - if banned, err := db.Banned(peer); err != nil || !banned { + ps.Ban(subnet.String(), time.Second, "test") + if banned, err := ps.Banned(peer); err != nil || !banned { t.Fatal("expected peer to be banned", err) } } From c9794afdd4d636da931f3dd5f83d2e19a33093d8 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 13:34:40 -0700 Subject: [PATCH 145/630] api, wallet: add scan progress to api --- api/api.go | 7 +++++ api/api_test.go | 35 ++++++++++++++++------ api/client.go | 13 ++++++--- api/server.go | 67 ++++++++++++++++++++++++++++++++++++++++--- wallet/manager.go | 19 +++++------- wallet/wallet_test.go | 10 +++---- 6 files changed, 118 insertions(+), 33 deletions(-) diff --git a/api/api.go b/api/api.go index d3f0fd1..9972477 100644 --- a/api/api.go +++ b/api/api.go @@ -93,3 +93,10 @@ type SeedSignRequest struct { Transaction types.Transaction `json:"transaction"` Keys []uint64 `json:"keys"` } + +type RescanResponse struct { + StartIndex types.ChainIndex `json:"startIndex"` + Index types.ChainIndex `json:"index"` + StartTime time.Time `json:"startTime"` + Error *string `json:"error,omitempty"` +} diff --git a/api/api_test.go b/api/api_test.go index a5e59ea..d28083a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -262,8 +262,13 @@ func TestWallet(t *testing.T) { } defer ws.Close() + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + // create the syncer - s := syncer.New(syncerListener, cm, ws, gateway.Header{ + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerListener.Addr().String(), @@ -288,9 +293,10 @@ func TestWallet(t *testing.T) { t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) } wc := c.Wallet(w.ID) - if err := c.Resubscribe(0); err != nil { + if err := c.Rescan(0); err != nil { t.Fatal(err) } + waitForBlock(t, cm, ws) balance, err := wc.Balance() if err != nil { @@ -505,9 +511,10 @@ func TestAddresses(t *testing.T) { t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) } wc := c.Wallet(w.ID) - if err := c.Resubscribe(0); err != nil { + if err := c.Rescan(0); err != nil { t.Fatal(err) } + waitForBlock(t, cm, ws) balance, err := wc.Balance() if err != nil { @@ -707,9 +714,13 @@ func TestV2(t *testing.T) { if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { t.Fatal(err) } - if err := c.Resubscribe(0); err != nil { + + // rescan is a helper function that waits for the wallet to finish + // rescanning + if err := c.Rescan(0); err != nil { t.Fatal(err) } + waitForBlock(t, cm, ws) // define some helper functions addBlock := func(txns []types.Transaction, v2txns []types.V2Transaction) error { @@ -901,6 +912,12 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer store1.Close() + + peerStore, err := sqlite.NewPeerStore(store1) + if err != nil { + t.Fatal(err) + } + wm1, err := wallet.NewManager(cm1, store1, log1.Named("wallet")) if err != nil { t.Fatal(err) @@ -910,7 +927,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer l1.Close() - s1 := syncer.New(l1, cm1, store1, gateway.Header{ + s1 := syncer.New(l1, cm1, peerStore, gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l1.Addr().String(), @@ -926,9 +943,10 @@ func TestP2P(t *testing.T) { if err := primary.AddAddress(wallet.Address{Address: primaryAddress}); err != nil { t.Fatal(err) } - if err := c1.Resubscribe(0); err != nil { + if err := c1.Rescan(0); err != nil { t.Fatal(err) } + waitForBlock(t, cm1, store1) dbstore2, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { @@ -950,7 +968,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer l2.Close() - s2 := syncer.New(l2, cm2, store2, gateway.Header{ + s2 := syncer.New(l2, cm2, peerStore, gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l2.Addr().String(), @@ -967,9 +985,10 @@ func TestP2P(t *testing.T) { if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { t.Fatal(err) } - if err := c2.Resubscribe(0); err != nil { + if err := c2.Rescan(0); err != nil { t.Fatal(err) } + waitForBlock(t, cm2, store2) // define some helper functions addBlock := func() error { diff --git a/api/client.go b/api/client.go index d9f5a99..8e6931c 100644 --- a/api/client.go +++ b/api/client.go @@ -116,10 +116,15 @@ func (c *Client) Wallet(id wallet.ID) *WalletClient { return &WalletClient{c: c.c, id: id} } -// Resubscribe subscribes the wallet to consensus updates, starting at the -// specified height. -func (c *Client) Resubscribe(height uint64) (err error) { - err = c.c.POST("/resubscribe", height, nil) +// ScanStatus returns the current state of wallet scanning. +func (c *Client) ScanStatus() (resp RescanResponse, err error) { + err = c.c.GET("/rescan", &resp) + return +} + +// Rescan rescans the blockchain starting from the specified height. +func (c *Client) Rescan(height uint64) (err error) { + err = c.c.POST("/rescan", height, nil) return } diff --git a/api/server.go b/api/server.go index d202ba4..8a18224 100644 --- a/api/server.go +++ b/api/server.go @@ -23,6 +23,7 @@ import ( type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { + BestIndex(height uint64) (types.ChainIndex, bool) TipState() consensus.State AddBlocks([]types.Block) error RecommendedFee() types.Currency @@ -47,7 +48,8 @@ type ( // A WalletManager manages wallets, keyed by name. WalletManager interface { - Subscribe(startHeight uint64) error + Tip() (types.ChainIndex, error) + Scan(index types.ChainIndex) error AddWallet(wallet.Wallet) (wallet.Wallet, error) UpdateWallet(wallet.Wallet) (wallet.Wallet, error) @@ -82,6 +84,10 @@ type server struct { // for walletsReserveHandler mu sync.Mutex used map[types.Hash256]bool + + scanMu sync.Mutex // for resubscribe + scanInProgress bool + scanInfo RescanResponse } func (s *server) stateHandler(jc jape.Context) { @@ -254,13 +260,65 @@ func (s *server) walletsIDHandlerDELETE(jc jape.Context) { } } -func (s *server) resubscribeHandler(jc jape.Context) { +func (s *server) rescanHandlerGET(jc jape.Context) { + index, err := s.wm.Tip() + if jc.Check("couldn't get tip", err) != nil { + return + } + + s.scanMu.Lock() + defer s.scanMu.Unlock() + if s.scanInfo.StartTime.IsZero() { + s.scanInfo.StartTime = startTime + } + s.scanInfo.Index = index + jc.Encode(s.scanInfo) +} + +func (s *server) rescanHandlerPOST(jc jape.Context) { var height uint64 if jc.Decode(&height) != nil { return - } else if jc.Check("couldn't subscribe wallet", s.wm.Subscribe(height)) != nil { + } + + s.scanMu.Lock() + defer s.scanMu.Unlock() + + if s.scanInProgress { + jc.Error(errors.New("scan already in progress"), http.StatusConflict) return } + + var index types.ChainIndex + if height > 0 { + var ok bool + index, ok = s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + } + + s.scanInProgress = true + s.scanInfo = RescanResponse{ + StartIndex: index, + Index: index, + StartTime: time.Now(), + Error: nil, + } + + go func() { + err := s.wm.Scan(index) + + // update the scan state + s.scanMu.Lock() + defer s.scanMu.Unlock() + s.scanInProgress = false + if err != nil { + msg := err.Error() + s.scanInfo.Error = &msg + } + }() } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { @@ -656,7 +714,8 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - "POST /resubscribe": srv.resubscribeHandler, + "GET /rescan": srv.rescanHandlerGET, + "POST /rescan": srv.rescanHandlerPOST, "GET /wallets": srv.walletsHandler, "POST /wallets": srv.walletsHandlerPOST, diff --git a/wallet/manager.go b/wallet/manager.go index 51fbc1e..f053fea 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -1,7 +1,6 @@ package wallet import ( - "errors" "fmt" "sync" "time" @@ -60,6 +59,11 @@ type ( } ) +// Tip returns the last scanned chain index of the manager. +func (m *Manager) Tip() (types.ChainIndex, error) { + return m.store.LastCommittedIndex() +} + // AddWallet adds the given wallet. func (m *Manager) AddWallet(w Wallet) (Wallet, error) { return m.store.AddWallet(w) @@ -150,19 +154,10 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { return nil } -// Subscribe resubscribes the indexer starting at the given height. -func (m *Manager) Subscribe(startHeight uint64) error { - var index types.ChainIndex - if startHeight > 0 { - var ok bool - index, ok = m.chain.BestIndex(startHeight - 1) - if !ok { - return errors.New("invalid height") - } - } +// Scan rescans the chain starting from the given index. +func (m *Manager) Scan(index types.ChainIndex) error { m.mu.Lock() defer m.mu.Unlock() - // TODO: is this right? won't it result in duplicate state? return syncStore(m.store, m.chain, index) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index b86f26e..c5a2163 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -123,8 +123,8 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - // resubscribe the wallet - if err := wm.Subscribe(0); err != nil { + // scan for changes + if err := wm.Scan(types.ChainIndex{}); err != nil { t.Fatal(err) } @@ -140,8 +140,8 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - // resubscribe - if err := wm.Subscribe(0); err != nil { + // scan for changes + if err := wm.Scan(types.ChainIndex{}); err != nil { t.Fatal(err) } @@ -160,7 +160,7 @@ func TestResubscribe(t *testing.T) { } // sanity check - if err := wm.Subscribe(0); err != nil { + if err := wm.Scan(types.ChainIndex{}); err != nil { t.Fatal(err) } From 74634c7532d8f1eb4ce1e59e9550b13f9064f4cd Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 2 Apr 2024 13:41:45 -0700 Subject: [PATCH 146/630] cmd: close node --- cmd/walletd/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index ed1709d..3e1c8bf 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -139,6 +139,8 @@ func main() { if err != nil { log.Fatal(err) } + defer n.Close() + log.Println("p2p: Listening on", n.s.Addr()) stop := n.Start() log.Println("api: Listening on", l.Addr()) From b994800766aa3946150c0bb0e2e800b4e2603f78 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 3 Apr 2024 00:33:29 -0700 Subject: [PATCH 147/630] api: address comments --- api/api_test.go | 2 -- api/server.go | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index d28083a..d2a0012 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -715,8 +715,6 @@ func TestV2(t *testing.T) { t.Fatal(err) } - // rescan is a helper function that waits for the wallet to finish - // rescanning if err := c.Rescan(0); err != nil { t.Fatal(err) } diff --git a/api/server.go b/api/server.go index 8a18224..48498c0 100644 --- a/api/server.go +++ b/api/server.go @@ -74,9 +74,9 @@ type ( } ) -var startTime = time.Now() - type server struct { + startTime time.Time + cm ChainManager s Syncer wm WalletManager @@ -96,7 +96,7 @@ func (s *server) stateHandler(jc jape.Context) { Commit: build.Commit(), OS: runtime.GOOS, BuildTime: build.Time(), - StartTime: startTime, + StartTime: s.startTime, }) } @@ -269,7 +269,7 @@ func (s *server) rescanHandlerGET(jc jape.Context) { s.scanMu.Lock() defer s.scanMu.Unlock() if s.scanInfo.StartTime.IsZero() { - s.scanInfo.StartTime = startTime + s.scanInfo.StartTime = s.startTime } s.scanInfo.Index = index jc.Encode(s.scanInfo) @@ -694,6 +694,8 @@ func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { // NewServer returns an HTTP handler that serves the walletd API. func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { srv := server{ + startTime: time.Now(), + cm: cm, s: s, wm: wm, From 1e1be97c5476e6b025a4c59052ab629975328321 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Wed, 3 Apr 2024 11:02:12 -0400 Subject: [PATCH 148/630] ci: use repository dispatch and shared ui action --- .github/workflows/ui.yml | 69 +++++----------------------------------- 1 file changed, 8 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 2d7896d..a91153b 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -1,9 +1,8 @@ -name: Update UI +name: Update UI and open PR on: - # Run daily - schedule: - - cron: '0 0 * * *' + repository_dispatch: + types: [update-ui] # Enable manual trigger workflow_dispatch: @@ -11,61 +10,9 @@ jobs: update-ui: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - - - name: Check for new walletd tag in SiaFoundation/web - id: check-tag - env: - GH_TOKEN: ${{ github.token }} - run: | - # Fetch tags with pagination - TAGS_JSON=$(gh api --paginate repos/SiaFoundation/web/tags) - - # Extract tags that start with "walletd/", sort them in version order, and pick the highest version - LATEST_WALLETD_GO_TAG=$(echo "$TAGS_JSON" | jq -r '.[] | select(.name | startswith("walletd/")).name' | sort -Vr | head -n 1) - LATEST_WALLETD_VERSION=$(echo "$LATEST_WALLETD_GO_TAG" | sed 's/walletd\///') - - echo "Latest walletd tag is $LATEST_WALLETD_GO_TAG" - echo "GO_TAG=$LATEST_WALLETD_GO_TAG" >> $GITHUB_ENV - echo "VERSION=$LATEST_WALLETD_VERSION" >> $GITHUB_ENV - - - name: Fetch release notes for the release - id: release-notes - env: - GH_TOKEN: ${{ github.token }} - if: env.GO_TAG != 'null' - run: | - RELEASE_TAG_FORMATTED=$(echo "$GO_TAG" | sed 's/\/v/@/') - RELEASES_JSON=$(gh api --paginate repos/SiaFoundation/web/releases) - - RELEASE_NOTES=$(echo "$RELEASES_JSON" | jq -r --arg TAG_NAME "$RELEASE_TAG_FORMATTED" '.[] | select(.name == $TAG_NAME).body') - echo "Release notes for $RELEASE_TAG_FORMATTED: $RELEASE_NOTES" - echo "RELEASE_NOTES<> $GITHUB_ENV - echo "$RELEASE_NOTES" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - - name: Update go.mod with latest module - if: env.GO_TAG != 'null' - run: | - GO_MODULE_FORMATTED=$(echo "$GO_TAG" | sed 's/\//@/') - echo "Updating go.mod to use $GO_MODULE_FORMATTED" - go clean -modcache - go get go.sia.tech/web/$GO_MODULE_FORMATTED - go mod tidy - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 - if: env.GO_TAG != 'null' + - name: Update UI and open PR + uses: SiaFoundation/workflows/.github/actions/ui-update@master with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "ui: ${{ env.VERSION }}" - title: "ui: ${{ env.VERSION }}" - body: ${{ env.RELEASE_NOTES }} - branch: "ui/update" - delete-branch: true + moduleName: 'walletd' + goVersion: '1.21' + token: ${{ secrets.GITHUB_TOKEN }} From 7dba738be0adbf0d169b500b8e91b5610327f935 Mon Sep 17 00:00:00 2001 From: alexfreska Date: Thu, 4 Apr 2024 12:03:08 +0000 Subject: [PATCH 149/630] ui: v0.18.0 --- go.mod | 8 +++++--- go.sum | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 1b8b89d..4430c55 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,15 @@ module go.sia.tech/walletd -go 1.21.6 +go 1.21.7 + +toolchain go1.21.8 require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.2.1 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.17.0 + go.sia.tech/web/walletd v0.18.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.18.0 lukechampine.com/flagg v1.1.1 @@ -20,7 +22,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.sia.tech/mux v1.2.0 // indirect - go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect + go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.21.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/go.sum b/go.sum index 3b8d15c..827bf40 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,10 @@ go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+ go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= -go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= -go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.17.0 h1:8k/m1L50LIylw1HYLlTuc3e4bYlx//qZ8xG4C/YNeA0= -go.sia.tech/web/walletd v0.17.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= +go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b h1:nwfLGAR0sjN/zb9QW0xWeNR8MjdtJl6KKZqPo2Amz3U= +go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= +go.sia.tech/web/walletd v0.18.0 h1:Sc04C9dTA7pWvC6AOfdQ5scd3zMNJb8Oc3vHqLAdY54= +go.sia.tech/web/walletd v0.18.0/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From f5cafa851d5408c1d79abe3a21c2d4b06a915ed6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 4 Apr 2024 11:32:19 -0700 Subject: [PATCH 150/630] add git ignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ccf8ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +bin/ +walletd.yml +.DS_Store +.vscode/ \ No newline at end of file From 54a8bb95c6cf60b8766f40f09e15c1786ab1870d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 4 Apr 2024 11:32:47 -0700 Subject: [PATCH 151/630] sqlite,wallet: fix incorrect table when removing siafunds, add test --- persist/sqlite/consensus.go | 6 +- persist/sqlite/sql.go | 2 +- wallet/wallet_test.go | 175 ++++++++++++++++++++++++++++++++---- 3 files changed, 166 insertions(+), 17 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 0a54941..74b8049 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -477,13 +477,17 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index types.ChainIndex) error { + if len(elements) == 0 { + return nil + } + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) + stmt, err := ut.tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index fb253d1..c9f990f 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -13,7 +13,7 @@ import ( const ( longQueryDuration = 10 * time.Millisecond - longTxnDuration = 10 * time.Millisecond + longTxnDuration = time.Second // reduce syncing spam ) type ( diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index c5a2163..2f174e9 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -41,7 +41,13 @@ func TestResubscribe(t *testing.T) { } defer bdb.Close() + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -55,10 +61,6 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - // mine a single payout to the wallet - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -73,9 +75,12 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - checkBalance := func(siacoin, immature types.Currency, siafund uint64) error { + checkBalance := func(siacoin, immature types.Currency) error { waitForBlock(t, cm, db) + // note: the siafund balance is currently hardcoded to the number of + // siafunds in genesis. If we ever modify this test to also spend + // siafunds, this will need to be updated. b, err := wm.WalletBalance(w.ID) if err != nil { return fmt.Errorf("failed to check balance: %w", err) @@ -83,14 +88,14 @@ func TestResubscribe(t *testing.T) { return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) } else if !b.ImmatureSiacoins.Equals(immature) { return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) - } else if b.Siafunds != siafund { - return fmt.Errorf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } else if b.Siafunds != network.GenesisState().SiafundCount() { + return fmt.Errorf("expected siafund balance %v, got %v", network.GenesisState().SiafundCount(), b.Siafunds) } return nil } // check that the wallet has no balance - if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency, 0); err != nil { + if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { t.Fatal(err) } @@ -107,7 +112,7 @@ func TestResubscribe(t *testing.T) { } // check that the wallet has one immature payout - if err := checkBalance(types.ZeroCurrency, expectedBalance1, 0); err != nil { + if err := checkBalance(types.ZeroCurrency, expectedBalance1); err != nil { t.Fatal(err) } @@ -119,7 +124,7 @@ func TestResubscribe(t *testing.T) { } // check that the wallet balance has matured - if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { t.Fatal(err) } @@ -129,14 +134,14 @@ func TestResubscribe(t *testing.T) { } // check that the wallet balance did not change - if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { t.Fatal(err) } // add the second address to the wallet if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) - } else if err := checkBalance(expectedBalance1, types.ZeroCurrency, 0); err != nil { + } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { t.Fatal(err) } @@ -145,7 +150,7 @@ func TestResubscribe(t *testing.T) { t.Fatal(err) } - if err := checkBalance(expectedBalance1, expectedBalance2, 0); err != nil { + if err := checkBalance(expectedBalance1, expectedBalance2); err != nil { t.Fatal(err) } @@ -155,7 +160,7 @@ func TestResubscribe(t *testing.T) { } // check that the wallet balance has matured - if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency, 0); err != nil { + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { t.Fatal(err) } @@ -165,7 +170,147 @@ func TestResubscribe(t *testing.T) { } // check that the wallet balance has matured - if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency, 0); err != nil { + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } +} + +func TestSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + checkBalance := func(walletID wallet.ID, siafunds uint64) error { + waitForBlock(t, cm, db) + + b, err := wm.WalletBalance(walletID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if b.Siafunds != siafunds { + return fmt.Errorf("expected siafund balance %v, got %v", siafunds, b.Siafunds) + } + return nil + } + + if err := wm.Scan(types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, network.GenesisState().SiafundCount()); err != nil { + t.Fatal(err) + } + + // split the siafunds between the two addresses + sendAmount := network.GenesisState().SiafundCount() / 2 + parentID := genesisBlock.Transactions[0].SiafundOutputID(0) + txn := types.Transaction{ + SiafundInputs: []types.SiafundInput{ + { + ParentID: parentID, + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr2, Value: sendAmount}, + {Address: addr1, Value: sendAmount}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(parentID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + state := cm.TipState() + sigHash := state.WholeSigHash(txn, txn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } else if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // rescan for sanity check + if err := wm.Scan(types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // add a second wallet + w2, err := wm.AddWallet(wallet.Wallet{Name: "test2"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // wallet should have no balance since it hasn't been scanned + if err := checkBalance(w2.ID, 0); err != nil { + t.Fatal(err) + } + + // rescan for the second wallet + if err := wm.Scan(types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w2.ID, sendAmount); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // add the first address to the second wallet + if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + // rescan shouldn't be necessary since the address was already scanned + if err := checkBalance(w2.ID, network.GenesisState().SiafundCount()); err != nil { t.Fatal(err) } } From 4bdc535a3c13b2c140769e8c8cc6228e2f096700 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:07:56 +0000 Subject: [PATCH 152/630] build(deps): bump go.sia.tech/core from 0.2.1 to 0.2.2 Bumps [go.sia.tech/core](https://github.com/SiaFoundation/core) from 0.2.1 to 0.2.2. - [Commits](https://github.com/SiaFoundation/core/compare/v0.2.1...v0.2.2) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4430c55..21db151 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.21.8 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.1 + go.sia.tech/core v0.2.2 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.18.0 diff --git a/go.sum b/go.sum index 827bf40..3940bca 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.sia.tech/core v0.2.1 h1:CqmMd+T5rAhC+Py3NxfvGtvsj/GgwIqQHHVrdts/LqY= -go.sia.tech/core v0.2.1/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= +go.sia.tech/core v0.2.2 h1:33RJrt08o7KyUOY4tITH6ECmRq1lhtapqc/SncIF/2A= +go.sia.tech/core v0.2.2/go.mod h1:Zk7HaybEPgkPC1p6e6tTQr8PIeZClTgNcLNGYDLQJeE= go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 h1:5AuiglkLdoBenrg41cJXJ4wTxkVTo85Asj9SPljnmiE= go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= From b0200503a9b130650f3f200bafc8f0d69fa45e9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:08:00 +0000 Subject: [PATCH 153/630] build(deps): bump golang.org/x/term from 0.18.0 to 0.19.0 Bumps [golang.org/x/term](https://github.com/golang/term) from 0.18.0 to 0.19.0. - [Commits](https://github.com/golang/term/compare/v0.18.0...v0.19.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4430c55..c25f76a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.18.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.18.0 + golang.org/x/term v0.19.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 @@ -25,6 +25,6 @@ require ( go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 827bf40..2f9cbc4 100644 --- a/go.sum +++ b/go.sum @@ -37,10 +37,10 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From dbdb3dc21c29a8ec0cf11cc49d27eee13bb3e63e Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Tue, 16 Apr 2024 18:02:25 +0000 Subject: [PATCH 154/630] ui: v0.19.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c174c4b..52e72d6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.2 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.18.0 + go.sia.tech/web/walletd v0.19.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.19.0 lukechampine.com/flagg v1.1.1 diff --git a/go.sum b/go.sum index e8d6e51..f8a87c2 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b h1:nwfLGAR0sjN/zb9QW0xWeNR8MjdtJl6KKZqPo2Amz3U= go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.18.0 h1:Sc04C9dTA7pWvC6AOfdQ5scd3zMNJb8Oc3vHqLAdY54= -go.sia.tech/web/walletd v0.18.0/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= +go.sia.tech/web/walletd v0.19.0 h1:lDsLTCGCKi9QNqsBGwf+9oaipD2o0ss8x6CrFqLXo30= +go.sia.tech/web/walletd v0.19.0/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From ded21ac048462c52eebc22ef27e37efaf6e4016e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Apr 2024 12:44:34 -0700 Subject: [PATCH 155/630] api,cmd,sqlite,wallet: revert orphaned blocks during rescan --- api/server.go | 4 +- cmd/walletd/main.go | 1 - cmd/walletd/node.go | 1 + internal/threadgroup/threadgroup.go | 96 +++++++++++ internal/threadgroup/threadgroup_test.go | 89 ++++++++++ persist/sqlite/addresses.go | 3 +- persist/sqlite/consensus.go | 199 ++++++++++++++++++++++- persist/sqlite/init.sql | 48 +++--- persist/sqlite/wallet.go | 12 +- wallet/manager.go | 65 ++++++-- wallet/update.go | 7 + 11 files changed, 479 insertions(+), 46 deletions(-) create mode 100644 internal/threadgroup/threadgroup.go create mode 100644 internal/threadgroup/threadgroup_test.go diff --git a/api/server.go b/api/server.go index 48498c0..3778993 100644 --- a/api/server.go +++ b/api/server.go @@ -49,7 +49,7 @@ type ( // A WalletManager manages wallets, keyed by name. WalletManager interface { Tip() (types.ChainIndex, error) - Scan(index types.ChainIndex) error + Scan(_ context.Context, index types.ChainIndex) error AddWallet(wallet.Wallet) (wallet.Wallet, error) UpdateWallet(wallet.Wallet) (wallet.Wallet, error) @@ -308,7 +308,7 @@ func (s *server) rescanHandlerPOST(jc jape.Context) { } go func() { - err := s.wm.Scan(index) + err := s.wm.Scan(context.Background(), index) // update the scan state s.scanMu.Lock() diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 3e1c8bf..cab41ed 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -150,7 +150,6 @@ func main() { <-signalCh log.Println("Shutting down...") stop() - case versionCmd: if len(cmd.Args()) != 0 { cmd.Usage() diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 831cba4..1900d77 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -95,6 +95,7 @@ type node struct { // Close shuts down the node and closes its database. func (n *node) Close() error { + n.wm.Close() n.chainStore.Close() return n.store.Close() } diff --git a/internal/threadgroup/threadgroup.go b/internal/threadgroup/threadgroup.go new file mode 100644 index 0000000..8688474 --- /dev/null +++ b/internal/threadgroup/threadgroup.go @@ -0,0 +1,96 @@ +package threadgroup + +import ( + "context" + "errors" + "sync" +) + +type ( + // A ThreadGroup provides synchronization between a module and its + // goroutines to enable clean shutdowns + ThreadGroup struct { + mu sync.Mutex + wg sync.WaitGroup + closed chan struct{} + } +) + +// ErrClosed is returned when the threadgroup has already been stopped +var ErrClosed = errors.New("threadgroup closed") + +// Done returns a channel that will be closed when the threadgroup is stopped +func (tg *ThreadGroup) Done() <-chan struct{} { + return tg.closed +} + +// Add adds a new thread to the group, done must be called to signal that the +// thread is done. Returns ErrClosed if the threadgroup is already closed. +func (tg *ThreadGroup) Add() (func(), error) { + tg.mu.Lock() + defer tg.mu.Unlock() + select { + case <-tg.closed: + return nil, ErrClosed + default: + } + tg.wg.Add(1) + return func() { tg.wg.Done() }, nil +} + +// WithContext returns a copy of the parent context. The returned context will +// be cancelled if the parent context is cancelled or if the threadgroup is +// stopped. +func (tg *ThreadGroup) WithContext(parent context.Context) (context.Context, context.CancelFunc) { + // wrap the parent context in a cancellable context + ctx, cancel := context.WithCancel(parent) + // start a goroutine to wait for either the parent context being cancelled + // or the threagroup being stopped + go func() { + select { + case <-ctx.Done(): + break + case <-tg.closed: + break + } + // threadgroup or parent context cancelled, cancel the child context + cancel() + }() + return ctx, cancel +} + +// AddContext adds a new thread to the group and returns a copy of the parent +// context. It is a convenience function combining Add and WithContext. +func (tg *ThreadGroup) AddContext(parent context.Context) (context.Context, context.CancelFunc, error) { + // try to add to the group + done, err := tg.Add() + if err != nil { + return nil, nil, err + } + + ctx, cancel := tg.WithContext(parent) + var once sync.Once + return ctx, func() { + cancel() + once.Do(done) + }, nil +} + +// Stop stops accepting new threads and waits for all existing threads to close +func (tg *ThreadGroup) Stop() { + tg.mu.Lock() + select { + case <-tg.closed: + default: + close(tg.closed) + } + tg.mu.Unlock() + tg.wg.Wait() +} + +// New creates a new threadgroup +func New() *ThreadGroup { + return &ThreadGroup{ + closed: make(chan struct{}), + } +} diff --git a/internal/threadgroup/threadgroup_test.go b/internal/threadgroup/threadgroup_test.go new file mode 100644 index 0000000..ca8a964 --- /dev/null +++ b/internal/threadgroup/threadgroup_test.go @@ -0,0 +1,89 @@ +package threadgroup + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestThreadgroup(t *testing.T) { + tg := New() + + for i := 0; i < 10; i++ { + done, err := tg.Add() + if err != nil { + t.Fatal(err) + } + time.AfterFunc(100*time.Millisecond, done) + } + start := time.Now() + tg.Stop() + if time.Since(start) < 100*time.Millisecond { + t.Fatal("expected stop to wait for all threads to complete") + } + + _, err := tg.Add() + if !errors.Is(err, ErrClosed) { + t.Fatalf("expected ErrClosed, got %v", err) + } +} + +func TestThreadgroupContext(t *testing.T) { + tg := New() + + t.Run("context cancel", func(t *testing.T) { + ctx, cancel, err := tg.AddContext(context.Background()) + if err != nil { + t.Fatal(err) + } + defer cancel() + + time.AfterFunc(100*time.Millisecond, cancel) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatalf("expected Canceled, got %v", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("expected context to be cancelled") + } + }) + + t.Run("parent cancel", func(t *testing.T) { + parentCtx, parentCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer parentCancel() + + ctx, cancel, err := tg.AddContext(parentCtx) + if err != nil { + t.Fatal(err) + } + defer cancel() + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("expected DeadlineExceeded, got %v", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("expected context to be cancelled") + } + }) + + t.Run("stop", func(t *testing.T) { + for i := 0; i < 10; i++ { + _, cancel, err := tg.AddContext(context.Background()) + if err != nil { + t.Fatal(err) + } + time.AfterFunc(100*time.Millisecond, cancel) + } + + start := time.Now() + tg.Stop() + if time.Since(start) < 100*time.Millisecond { + t.Fatal("expected threadgroup to wait until all threads complete") + } + }) +} diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index b7ae3fe..66973e7 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -91,8 +91,7 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) defer rows.Close() for rows.Next() { - var siafund types.SiafundElement - err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) + siafund, err := scanSiafundElement(rows) if err != nil { return fmt.Errorf("failed to scan siafund element: %w", err) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 74b8049..3d34e9c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -10,6 +10,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" ) type updateTx struct { @@ -296,8 +297,14 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } defer addrStmt.Close() + indexStmt, err := insertIndexStmt(ut.tx) + if err != nil { + return fmt.Errorf("failed to prepare index statement: %w", err) + } + defer indexStmt.Close() + // ignore elements already in the database. - insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -305,6 +312,12 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { + var chainIndexID int64 + err := indexStmt.QueryRow(index.Height, encode(index.ID)).Scan(&chainIndexID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) @@ -313,7 +326,7 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } var dummyID types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0).Scan(decode(&dummyID)) + err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, chainIndexID).Scan(decode(&dummyID)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { @@ -422,13 +435,19 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty return nil } + indexStmt, err := insertIndexStmt(ut.tx) + if err != nil { + return fmt.Errorf("failed to prepare index statement: %w", err) + } + defer indexStmt.Close() + addrStmt, err := insertAddressStatement(ut.tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -436,6 +455,11 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty balanceChanges := make(map[int64]uint64) for _, se := range elements { + var chainIndexID int64 + if err := indexStmt.QueryRow(index.Height, encode(index.ID)).Scan(&chainIndexID); err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) @@ -444,7 +468,7 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } var dummy types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID).Scan(decode(&dummy)) + err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, chainIndexID).Scan(decode(&dummy)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { @@ -621,6 +645,164 @@ func (ut *updateTx) RevertEvents(index types.ChainIndex) error { return err } +func (ut *updateTx) getOrphanedSiacoinBalance(indexID int64) (map[int64]wallet.Balance, error) { + const query = `SELECT address_id, siacoin_value, matured +FROM siacoin_elements +WHERE chain_index_id=$1` + rows, err := ut.tx.Query(query, indexID) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]wallet.Balance) + for rows.Next() { + var addrID int64 + var value types.Currency + var matured bool + + if err := rows.Scan(&addrID, decode(&value), &matured); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + + balance := balances[addrID] + if matured { + balance.Siacoins = balance.Siacoins.Add(value) + } else { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(value) + } + balances[addrID] = balance + } + return balances, rows.Err() +} + +func (ut *updateTx) getOrphanedSiafundBalance(indexID int64) (map[int64]uint64, error) { + const query = `SELECT address_id, siafund_value +FROM siafund_elements +WHERE chain_index_id=$1` + rows, err := ut.tx.Query(query, indexID) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]uint64) + for rows.Next() { + var addrID int64 + var value uint64 + + if err := rows.Scan(&addrID, &value); err != nil { + return nil, fmt.Errorf("failed to scan siafund element: %w", err) + } + balances[addrID] += value + } + return balances, rows.Err() +} + +func (ut *updateTx) getOrphanedIndexes(index types.ChainIndex) (orphaned []int64, err error) { + rows, err := ut.tx.Query(`SELECT id FROM chain_indices WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query orphans: %w", err) + } + defer rows.Close() + + for rows.Next() { + var indexID int64 + if err := rows.Scan(&indexID); err != nil { + return nil, fmt.Errorf("failed to scan orphan: %w", err) + } + orphaned = append(orphaned, indexID) + } + return orphaned, rows.Err() +} + +// RevertOrphans reverts any chain indices that were orphaned by the given index +func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.BlockID, err error) { + log := ut.tx.log.Named("RevertOrphans").With(zap.Uint64("height", index.Height), zap.Stringer("applied", index.ID)) + + orphaned, err := ut.getOrphanedIndexes(index) + if err != nil { + return nil, fmt.Errorf("failed to get orphaned indexes: %w", err) + } + + if len(orphaned) == 0 { + return nil, nil + } + + var revertedBalance map[int64]wallet.Balance + for _, id := range orphaned { + // revert siacoin balances + siacoins, err := ut.getOrphanedSiacoinBalance(id) + if err != nil { + return nil, fmt.Errorf("failed to get orphaned siacoin elements: %w", err) + } + + // revert siafund balances + siafunds, err := ut.getOrphanedSiafundBalance(id) + if err != nil { + return nil, fmt.Errorf("failed to get orphaned siafund elements: %w", err) + } + + for addr, balance := range siafunds { + b := siacoins[addr] + b.Siafunds = balance + siacoins[addr] = b + } + revertedBalance = siacoins + } + + getBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return nil, fmt.Errorf("failed to prepare balance statement: %w", err) + } + defer getBalanceStmt.Close() + + updateBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE id=$4`) + if err != nil { + return nil, fmt.Errorf("failed to prepare update statement: %w", err) + } + + for addrID, balance := range revertedBalance { + var existing wallet.Balance + err := getBalanceStmt.QueryRow(addrID).Scan(decode(&existing.Siacoins), decode(&existing.ImmatureSiacoins), &existing.Siafunds) + if err != nil { + return nil, fmt.Errorf("failed to get balance: %w", err) + } + + existing.Siacoins = existing.Siacoins.Sub(balance.Siacoins) + existing.ImmatureSiacoins = existing.ImmatureSiacoins.Sub(balance.ImmatureSiacoins) + if existing.Siafunds < balance.Siafunds { + panic("siafund balance cannot be negative") + } + existing.Siafunds -= balance.Siafunds + + res, err := updateBalanceStmt.Exec(encode(existing.Siacoins), encode(existing.ImmatureSiacoins), existing.Siafunds, addrID) + if err != nil { + return nil, fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return nil, fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return nil, fmt.Errorf("expected 1 row affected, got %v", n) + } + } + + rows, err := ut.tx.Query(`DELETE FROM chain_indices WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query orphans: %w", err) + } + defer rows.Close() + + for rows.Next() { + var orphan types.BlockID + if err := rows.Scan(decode(&orphan)); err != nil { + return nil, fmt.Errorf("failed to scan orphan: %w", err) + } + reverted = append(reverted, orphan) + log.Debug("reverted orphan", zap.Stringer("orphan", orphan)) + } + return reverted, rows.Err() +} + // ProcessChainApplyUpdate implements chain.Subscriber func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { return s.transaction(func(tx *txn) error { @@ -644,15 +826,24 @@ func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { return } +// ResetLastIndex resets the last indexed tip to trigger a full rescan. +func (s *Store) ResetLastIndex() error { + _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(types.ChainIndex{})) + return err + +} + func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(index)) return err } func insertAddressStatement(tx *txn) (*stmt, error) { + // the on conflict is effectively a no-op, but enables us to return the id of the existing address return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) } func insertIndexStmt(tx *txn) (*stmt, error) { + // the on conflict is effectively a no-op, but enables us to return the id of the existing index return tx.Prepare(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`) } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 3f399e5..293073d 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -19,10 +19,12 @@ CREATE TABLE siacoin_elements ( leaf_index INTEGER NOT NULL, maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - matured BOOLEAN NOT NULL /* tracks whether the value has been added to the address balance */ + matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE ); CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index ON siacoin_elements (chain_index_id); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -30,9 +32,30 @@ CREATE TABLE siafund_elements ( merkle_proof BLOB NOT NULL, leaf_index INTEGER NOT NULL, siafund_value INTEGER NOT NULL, - address_id INTEGER NOT NULL REFERENCES sia_addresses (id) + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE ); CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index ON siafund_elements (chain_index_id); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + event_id BLOB UNIQUE NOT NULL, + index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, + maturity_height INTEGER NOT NULL, + date_created INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_data BLOB NOT NULL +); +CREATE INDEX events_index_id ON events (index_id); + +CREATE TABLE event_addresses ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + PRIMARY KEY (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); CREATE TABLE wallets ( id INTEGER PRIMARY KEY, @@ -54,25 +77,6 @@ CREATE TABLE wallet_addresses ( CREATE INDEX wallet_addresses_wallet_id ON wallet_addresses (wallet_id); CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); -CREATE TABLE events ( - id INTEGER PRIMARY KEY, - event_id BLOB UNIQUE NOT NULL, - index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, - maturity_height INTEGER NOT NULL, - date_created INTEGER NOT NULL, - event_type TEXT NOT NULL, - event_data BLOB NOT NULL -); - - -CREATE TABLE event_addresses ( - event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, - address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - PRIMARY KEY (event_id, address_id) -); -CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); -CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); - CREATE TABLE syncer_peers ( peer_address TEXT PRIMARY KEY NOT NULL, first_seen INTEGER NOT NULL @@ -88,5 +92,5 @@ CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row db_version INTEGER NOT NULL, -- used for migrations - last_indexed_tip BLOB -- the last chain index that was processed + last_indexed_tip BLOB NOT NULL -- the last chain index that was processed ); diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 7d0caf3..3ef42de 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -16,6 +16,11 @@ func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { return } +func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { + s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + return +} + func insertAddress(tx *txn, addr types.Address) (id int64, err error) { const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address @@ -142,8 +147,8 @@ func (s *Store) WalletEvents(id wallet.ID, offset, limit int) (events []wallet.E // AddWallet adds a wallet to the database. func (s *Store) AddWallet(w wallet.Wallet) (wallet.Wallet, error) { - w.DateCreated = time.Now() - w.LastUpdated = time.Now() + w.DateCreated = time.Now().Truncate(time.Second) + w.LastUpdated = time.Now().Truncate(time.Second) err := s.transaction(func(tx *txn) error { const query = `INSERT INTO wallets (friendly_name, description, date_created, last_updated, extra_data) VALUES ($1, $2, $3, $4, $5) RETURNING id` @@ -337,8 +342,7 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds defer rows.Close() for rows.Next() { - var siafund types.SiafundElement - err := rows.Scan(decode(&siafund.ID), &siafund.LeafIndex, decodeSlice(&siafund.MerkleProof), &siafund.SiafundOutput.Value, decode(&siafund.ClaimStart), decode(&siafund.SiafundOutput.Address)) + siafund, err := scanSiafundElement(rows) if err != nil { return fmt.Errorf("failed to scan siafund element: %w", err) } diff --git a/wallet/manager.go b/wallet/manager.go index f053fea..4fd3404 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -1,12 +1,14 @@ package wallet import ( + "context" "fmt" "sync" "time" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/internal/threadgroup" "go.uber.org/zap" ) @@ -52,10 +54,10 @@ type ( chain ChainManager store Store log *zap.Logger + tg *threadgroup.ThreadGroup - mu sync.Mutex - used map[types.Hash256]bool - unsubscribe func() + mu sync.Mutex + used map[types.Hash256]bool } ) @@ -154,15 +156,33 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { return nil } -// Scan rescans the chain starting from the given index. -func (m *Manager) Scan(index types.ChainIndex) error { +// Scan rescans the chain starting from the given index. The scan will complete +// when the chain manager reaches the current tip or the context is canceled. +func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { + ctx, cancel, err := m.tg.AddContext(ctx) + if err != nil { + return err + } + defer cancel() + m.mu.Lock() defer m.mu.Unlock() - return syncStore(m.store, m.chain, index) + return syncStore(ctx, m.store, m.chain, index) } -func syncStore(store Store, cm ChainManager, index types.ChainIndex) error { +// Close closes the wallet manager. +func (m *Manager) Close() error { + m.tg.Stop() + return nil +} + +func syncStore(ctx context.Context, store Store, cm ChainManager, index types.ChainIndex) error { for index != cm.Tip() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } crus, caus, err := cm.UpdatesSince(index, 1000) if err != nil { return fmt.Errorf("failed to subscribe to chain manager: %w", err) @@ -180,6 +200,7 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) chain: cm, store: store, log: log, + tg: threadgroup.New(), } lastTip, err := store.LastCommittedIndex() @@ -188,24 +209,46 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) } go func() { - if err := syncStore(store, cm, lastTip); err != nil { + ctx, cancel, err := m.tg.AddContext(context.Background()) + if err != nil { + log.Panic("failed to add to threadgroup", zap.Error(err)) + } + defer cancel() + + if err := syncStore(ctx, store, cm, lastTip); err != nil { log.Fatal("failed to subscribe to chain manager", zap.Error(err)) } reorgChan := make(chan types.ChainIndex, 1) - m.unsubscribe = cm.OnReorg(func(index types.ChainIndex) { + unsubscribe := cm.OnReorg(func(index types.ChainIndex) { select { case reorgChan <- index: default: } }) + defer unsubscribe() + + for { + select { + case <-ctx.Done(): + return + case <-reorgChan: + } - for range reorgChan { m.mu.Lock() + // check that the context was not canceled while waiting for the + // lock + select { + case <-ctx.Done(): + return + default: + } + + // update the store lastTip, err := store.LastCommittedIndex() if err != nil { log.Error("failed to get last committed index", zap.Error(err)) - } else if err := syncStore(store, cm, lastTip); err != nil { + } else if err := syncStore(ctx, store, cm, lastTip); err != nil { log.Error("failed to sync store", zap.Error(err)) } m.mu.Unlock() diff --git a/wallet/update.go b/wallet/update.go index 52e3f43..909e0ef 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -35,11 +35,18 @@ type ( RevertMatureSiacoinBalance(types.ChainIndex) error RevertEvents(index types.ChainIndex) error + + RevertOrphans(types.ChainIndex) (reverted []types.BlockID, err error) } ) // applyChainUpdate atomically applies a chain update to a store func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { + // revert any orphaned chain indices + if _, err := tx.RevertOrphans(cau.State.Index); err != nil { + return fmt.Errorf("failed to revert orphans: %w", err) + } + // update the immature balance of each relevant address if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { return fmt.Errorf("failed to get matured siacoin elements: %w", err) From 23f51750f77154142cf692f2fc00f91b55b71664 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Apr 2024 12:44:55 -0700 Subject: [PATCH 156/630] sqlite,wallet: move tests out of store, add orphaned blocks test --- persist/sqlite/consensus_test.go | 607 ---------------------- persist/sqlite/wallet_test.go | 213 -------- wallet/wallet_test.go | 864 ++++++++++++++++++++++++++++++- 3 files changed, 858 insertions(+), 826 deletions(-) delete mode 100644 persist/sqlite/consensus_test.go delete mode 100644 persist/sqlite/wallet_test.go diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go deleted file mode 100644 index eb2ec7b..0000000 --- a/persist/sqlite/consensus_test.go +++ /dev/null @@ -1,607 +0,0 @@ -package sqlite_test - -import ( - "path/filepath" - "testing" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" - "go.sia.tech/coreutils" - "go.sia.tech/coreutils/chain" - "go.sia.tech/walletd/persist/sqlite" - "go.sia.tech/walletd/wallet" - "go.uber.org/zap/zaptest" -) - -func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 1000 - n.HardforkV2.RequireHeight = 1000 - return n, genesisBlock -} - -func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 100 - n.HardforkV2.RequireHeight = 110 - return n, genesisBlock -} - -func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { - b := types.Block{ - ParentID: state.Index.ID, - Timestamp: types.CurrentTimestamp(), - Transactions: txns, - MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, - } - for b.ID().CmpWork(state.ChildTarget) < 0 { - b.Nonce += state.NonceFactor() - } - return b -} - -func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { - b := types.Block{ - ParentID: state.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, - - V2: &types.V2BlockData{ - Transactions: txns, - Height: state.Index.Height + 1, - }, - } - b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - for b.ID().CmpWork(state.ChildTarget) < 0 { - b.Nonce += state.NonceFactor() - } - return b -} - -func syncDB(t *testing.T, db *sqlite.Store, cm *chain.Manager) { - index, err := db.LastCommittedIndex() - if err != nil { - t.Fatal(err) - } - for index != cm.Tip() { - crus, caus, err := cm.UpdatesSince(index, 1000) - if err != nil { - t.Fatal(err) - } else if err := db.UpdateChainState(crus, caus); err != nil { - t.Fatal(err) - } - index = caus[len(caus)-1].State.Index - } -} - -func TestReorg(t *testing.T) { - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } - - expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was received - balance, err := db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err := db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // check that the utxo was created - utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } - - // mine to trigger a reorg - var blocks []types.Block - state := genesisState - for i := 0; i < 5; i++ { - blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) - state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was reverted - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.IsZero() { - t.Fatalf("expected 0, got %v", balance.ImmatureSiacoins) - } - - // check that the payout event was reverted - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 0 { - t.Fatalf("expected 0 events, got %v", len(events)) - } - - // check that the utxo was removed - utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 0 { - t.Fatalf("expected 0 outputs, got %v", len(utxos)) - } - - // mine a new payout - expectedPayout = cm.TipState().BlockReward() - maturityHeight = cm.TipState().MaturityHeight() - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was received - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // check that the utxo was created - utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } - - // mine until the payout matures - var prevState consensus.State - for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - if i == maturityHeight-5 { - prevState = cm.TipState() - } - } - syncDB(t, db, cm) - - // check that the balance was updated - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.IsZero() { - t.Fatalf("expected %v, got %v", types.ZeroCurrency, balance.ImmatureSiacoins) - } else if !balance.Siacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) - } - - // reorg the last few blocks to re-mature the payout - blocks = nil - state = prevState - for i := 0; i < 10; i++ { - blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) - state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the balance is correct - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.IsZero() { - t.Fatalf("expected %v, got %v", types.ZeroCurrency, balance.ImmatureSiacoins) - } else if !balance.Siacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) - } - - // check that only the single utxo still exists - utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } -} - -func TestEphemeralBalance(t *testing.T) { - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } - - expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() + 1 - block := mineBlock(cm.TipState(), nil, addr) - minerPayoutID := block.ID().MinerOutputID(0) - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{block}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was received - balance, err := db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err := db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } else if events[0].ID != types.Hash256(minerPayoutID) { - t.Fatalf("expected %v, got %v", minerPayoutID, events[0].ID) - } - - // mine until the payout matures - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - } - syncDB(t, db, cm) - - // create a transaction that spends the matured payout - utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } - - unlockConditions := types.StandardUnlockConditions(pk.PublicKey()) - parentTxn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{ - { - ParentID: types.SiacoinOutputID(utxos[0].ID), - UnlockConditions: unlockConditions, - }, - }, - SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr, Value: types.Siacoins(100)}, - {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100))}, - }, - Signatures: []types.TransactionSignature{ - { - ParentID: utxos[0].ID, - PublicKeyIndex: 0, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }, - }, - } - parentSigHash := cm.TipState().WholeSigHash(parentTxn, utxos[0].ID, 0, 0, nil) - parentSig := pk.SignHash(parentSigHash) - parentTxn.Signatures[0].Signature = parentSig[:] - - outputID := parentTxn.SiacoinOutputID(0) - txn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{ - { - ParentID: outputID, - UnlockConditions: unlockConditions, - }, - }, - SiacoinOutputs: []types.SiacoinOutput{ - {Address: types.VoidAddress, Value: types.Siacoins(100)}, - }, - Signatures: []types.TransactionSignature{ - { - ParentID: types.Hash256(outputID), - PublicKeyIndex: 0, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }, - }, - } - sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(outputID), 0, 0, nil) - sig := pk.SignHash(sigHash) - txn.Signatures[0].Signature = sig[:] - - txnset := []types.Transaction{parentTxn, txn} - - // broadcast the transactions - revertState := cm.TipState() - if err := cm.AddBlocks([]types.Block{mineBlock(revertState, txnset, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was spent - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.Siacoins.IsZero() { - t.Fatalf("expected 0, got %v", balance.Siacoins) - } - - // check that both transactions were added - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 3 { // 1 payout, 2 transactions - t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[2].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected miner payout event, got %v", events[2].Data.EventType()) - } else if events[1].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[1].Data.EventType()) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) - } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first - t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) - } else if events[0].ID != types.Hash256(txn.ID()) { // child txn second - t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) - } - - // trigger a reorg - var blocks []types.Block - state := revertState - for i := 0; i < 2; i++ { - blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) - state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the transaction was reverted - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.Siacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) - } - - // check that only the payout event remains - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } -} - -func TestV2(t *testing.T) { - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } - - expectedPayout := cm.TipState().BlockReward() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was received - balance, err := db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err := db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // mine until the payout matures - maturityHeight := cm.TipState().MaturityHeight() + 1 - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - } - syncDB(t, db, cm) - - // create a v2 transaction that spends the matured payout - utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } - - sce := utxos[0] - policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: policy}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{ - {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, - {Address: addr, Value: types.Siacoins(100)}, - }, - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} - - if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the change was received - balance, err = db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.Siacoins.Equals(types.Siacoins(100)) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a transaction event was recorded - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 2 { - t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) - } else if events[0].Relevant[0] != addr { - t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) - } -} diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go deleted file mode 100644 index b976d95..0000000 --- a/persist/sqlite/wallet_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package sqlite_test - -import ( - "encoding/json" - "path/filepath" - "testing" - - "go.sia.tech/core/types" - "go.sia.tech/coreutils" - "go.sia.tech/coreutils/chain" - "go.sia.tech/walletd/persist/sqlite" - "go.sia.tech/walletd/wallet" - "go.uber.org/zap/zaptest" -) - -func TestWalletAddresses(t *testing.T) { - log := zaptest.NewLogger(t) - db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - // Add a wallet - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } - - wallets, err := db.Wallets() - if err != nil { - t.Fatal(err) - } else if len(wallets) != 1 { - t.Fatal("expected 1 wallet, got", len(wallets)) - } else if wallets[0].ID != w.ID { - t.Fatal("unexpected wallet ID", wallets[0].ID) - } else if wallets[0].Name != "test" { - t.Fatal("unexpected wallet name", wallets[0].Name) - } else if wallets[0].Metadata != nil { - t.Fatal("unexpected metadata", wallets[0].Metadata) - } - - // Add an address - pk := types.GeneratePrivateKey() - spendPolicy := types.PolicyPublicKey(pk.PublicKey()) - address := spendPolicy.Address() - - addr := wallet.Address{ - Address: address, - SpendPolicy: &spendPolicy, - Description: "hello, world", - } - err = db.AddWalletAddress(w.ID, addr) - if err != nil { - t.Fatal(err) - } - - // Check that the address was added - addresses, err := db.WalletAddresses(w.ID) - if err != nil { - t.Fatal(err) - } else if len(addresses) != 1 { - t.Fatal("expected 1 address, got", len(addresses)) - } else if addresses[0].Address != address { - t.Fatal("unexpected address", addresses[0].Address) - } else if addresses[0].Description != "hello, world" { - t.Fatal("unexpected description", addresses[0].Description) - } else if *addresses[0].SpendPolicy != spendPolicy { - t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) - } - - // update the addresses metadata and description - addr.Description = "goodbye, world" - addr.Metadata = json.RawMessage(`{"foo": "bar"}`) - - if err := db.AddWalletAddress(w.ID, addr); err != nil { - t.Fatal(err) - } - - // Check that the address was added - addresses, err = db.WalletAddresses(w.ID) - if err != nil { - t.Fatal(err) - } else if len(addresses) != 1 { - t.Fatal("expected 1 address, got", len(addresses)) - } else if addresses[0].Address != address { - t.Fatal("unexpected address", addresses[0].Address) - } else if addresses[0].Description != "goodbye, world" { - t.Fatal("unexpected description", addresses[0].Description) - } else if *addresses[0].SpendPolicy != spendPolicy { - t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) - } else if string(addresses[0].Metadata) != `{"foo": "bar"}` { - t.Fatal("unexpected metadata", addresses[0].Metadata) - } - - // Remove the address - err = db.RemoveWalletAddress(w.ID, address) - if err != nil { - t.Fatal(err) - } - - // Check that the address was removed - addresses, err = db.WalletAddresses(w.ID) - if err != nil { - t.Fatal(err) - } else if len(addresses) != 0 { - t.Fatal("expected 0 addresses, got", len(addresses)) - } -} - -func TestResubscribe(t *testing.T) { - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - w, err := db.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } - - expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - - // check that the payout was received - balance, err := db.WalletBalance(w.ID) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err := db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // check that the utxo was created - utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } - - // check that the balance, events, and utxos did not change - // check that the payout was received - balance, err = db.WalletBalance(w.ID) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err = db.WalletEvents(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // check that the utxo was created - utxos, err = db.WalletSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } -} diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 2f174e9..8924639 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1,11 +1,15 @@ package wallet_test import ( + "bytes" + "context" + "encoding/json" "fmt" "path/filepath" "testing" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" @@ -26,6 +30,714 @@ func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { tb.Fatal("timed out waiting for block") } +func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 1000 + n.HardforkV2.RequireHeight = 1000 + return n, genesisBlock +} + +func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 100 + n.HardforkV2.RequireHeight = 110 + return n, genesisBlock +} + +func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + } + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + + V2: &types.V2BlockData{ + Transactions: txns, + Height: state.Index.Height + 1, + }, + } + b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func TestReorg(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } + + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the payout event was reverted + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // check that the utxo was removed + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } + + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight = cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + waitForBlock(t, cm, db) + + // check that the balance was updated + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that only the single utxo still exists + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } +} + +func TestEphemeralBalance(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + 1 + block := mineBlock(cm.TipState(), nil, addr) + minerPayoutID := block.ID().MinerOutputID(0) + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{block}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + balance, err := wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].ID != types.Hash256(minerPayoutID) { + t.Fatalf("expected %v, got %v", minerPayoutID, events[0].ID) + } + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // create a transaction that spends the matured payout + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + unlockConditions := types.StandardUnlockConditions(pk.PublicKey()) + parentTxn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(100)}, + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100))}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: utxos[0].ID, + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + parentSigHash := cm.TipState().WholeSigHash(parentTxn, utxos[0].ID, 0, 0, nil) + parentSig := pk.SignHash(parentSigHash) + parentTxn.Signatures[0].Signature = parentSig[:] + + outputID := parentTxn.SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: outputID, + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(100)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(outputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(outputID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + txnset := []types.Transaction{parentTxn, txn} + + // broadcast the transactions + revertState := cm.TipState() + if err := cm.AddBlocks([]types.Block{mineBlock(revertState, txnset, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was spent + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.IsZero() { + t.Fatalf("expected 0, got %v", balance.Siacoins) + } + + // check that both transactions were added + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 3 { // 1 payout, 2 transactions + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[2].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[2].Data.EventType()) + } else if events[1].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[1].Data.EventType()) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first + t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) + } else if events[0].ID != types.Hash256(txn.ID()) { // child txn second + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } + + // trigger a reorg + var blocks []types.Block + state := revertState + for i := 0; i < 2; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the transaction was reverted + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) + } + + // check that only the payout event remains + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } +} + +func TestWalletAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + // Add a wallet + w := wallet.Wallet{ + Name: "test", + Description: "hello, world!", + Metadata: json.RawMessage(`{"foo": "bar"}`), + } + w, err = wm.AddWallet(w) + if err != nil { + t.Fatal(err) + } + + wallets, err := wm.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != 1 { + t.Fatal("expected 1 wallet, got", len(wallets)) + } else if wallets[0].ID != w.ID { + t.Fatal("unexpected wallet ID", wallets[0].ID) + } else if wallets[0].Name != "test" { + t.Fatal("unexpected wallet name", wallets[0].Name) + } else if wallets[0].Description != "hello, world!" { + t.Fatal("unexpected description", wallets[0].Description) + } else if !bytes.Equal(wallets[0].Metadata, []byte(`{"foo": "bar"}`)) { + t.Fatal("unexpected metadata", wallets[0].Metadata) + } else if wallets[0].DateCreated.IsZero() || !wallets[0].DateCreated.Equal(w.DateCreated) { + t.Fatalf("expected creation date %s, got %s", w.DateCreated, wallets[0].DateCreated) + } else if wallets[0].LastUpdated.IsZero() || !wallets[0].LastUpdated.Equal(w.LastUpdated) { + t.Fatalf("expected last updated date %s, got %s", w.LastUpdated, wallets[0].LastUpdated) + } + + // Add an address + pk := types.GeneratePrivateKey() + spendPolicy := types.PolicyPublicKey(pk.PublicKey()) + address := spendPolicy.Address() + + addr := wallet.Address{ + Address: address, + SpendPolicy: &spendPolicy, + Description: "hello, world", + } + err = db.AddWalletAddress(w.ID, addr) + if err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err := db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "hello, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } + + // update the addresses metadata and description + addr.Description = "goodbye, world" + addr.Metadata = json.RawMessage(`{"foo": "bar"}`) + + if err := db.AddWalletAddress(w.ID, addr); err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err = db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "goodbye, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } else if string(addresses[0].Metadata) != `{"foo": "bar"}` { + t.Fatal("unexpected metadata", addresses[0].Metadata) + } + + // Remove the address + err = db.RemoveWalletAddress(w.ID, address) + if err != nil { + t.Fatal(err) + } + + // Check that the address was removed + addresses, err = db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 0 { + t.Fatal("expected 0 addresses, got", len(addresses)) + } +} + +func TestV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // mine until the payout matures + maturityHeight := cm.TipState().MaturityHeight() + 1 + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // create a v2 transaction that spends the matured payout + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, + {Address: addr, Value: types.Siacoins(100)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the change was received + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(100)) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a transaction event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Relevant[0] != addr { + t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) + } +} + func TestResubscribe(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() @@ -74,6 +786,10 @@ func TestResubscribe(t *testing.T) { if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } + // rescan to get the genesis Siafund state + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } checkBalance := func(siacoin, immature types.Currency) error { waitForBlock(t, cm, db) @@ -129,7 +845,7 @@ func TestResubscribe(t *testing.T) { } // scan for changes - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } @@ -146,7 +862,7 @@ func TestResubscribe(t *testing.T) { } // scan for changes - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } @@ -165,7 +881,7 @@ func TestResubscribe(t *testing.T) { } // sanity check - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } @@ -236,7 +952,7 @@ func TestSiafunds(t *testing.T) { return nil } - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } else if err := checkBalance(w1.ID, network.GenesisState().SiafundCount()); err != nil { t.Fatal(err) @@ -277,7 +993,7 @@ func TestSiafunds(t *testing.T) { } // rescan for sanity check - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } else if err := checkBalance(w1.ID, sendAmount); err != nil { t.Fatal(err) @@ -297,7 +1013,7 @@ func TestSiafunds(t *testing.T) { } // rescan for the second wallet - if err := wm.Scan(types.ChainIndex{}); err != nil { + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { t.Fatal(err) } else if err := checkBalance(w2.ID, sendAmount); err != nil { t.Fatal(err) @@ -314,3 +1030,139 @@ func TestSiafunds(t *testing.T) { t.Fatal(err) } } + +func TestOrphans(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } + + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // simulate an interrupted rescan by closing the wallet manager, resetting the + // last rescan index, and initializing a new wallet manager. + if err := wm.Close(); err != nil { + t.Fatal(err) + } else if err := db.ResetLastIndex(); err != nil { + t.Fatal(err) + } + + // mine to trigger a reorg. The underlying store must properly revert the + // orphaned blocks that will not be cleanly reverted since the rescan was + // interrupted. + var blocks []types.Block + state := genesisState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + wm, err = wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the payout event was reverted + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // check that the utxo was reverted + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 output, got %v", len(utxos)) + } +} From 13a409e7b2420b82e8e96102e3ac71f5ca39cbcc Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Apr 2024 12:50:15 -0700 Subject: [PATCH 157/630] sqlite: return err from scanSiafundElement --- persist/sqlite/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 3ef42de..2b654c7 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -17,7 +17,7 @@ func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { } func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { - s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) return } From 41ba07e286f828f2783e9a4a251489ccf1695533 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Apr 2024 12:50:42 -0700 Subject: [PATCH 158/630] sqlite: fix lint --- persist/sqlite/consensus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 3d34e9c..c048bdf 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -830,7 +830,6 @@ func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { func (s *Store) ResetLastIndex() error { _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(types.ChainIndex{})) return err - } func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { From d3951d9e214604315a60256ca608520311790864 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 17 Apr 2024 10:36:12 -0700 Subject: [PATCH 159/630] address review comments --- internal/threadgroup/threadgroup.go | 30 +++++++++++++++++------- internal/threadgroup/threadgroup_test.go | 6 ++--- persist/sqlite/consensus.go | 8 +++---- wallet/manager.go | 12 ++-------- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/internal/threadgroup/threadgroup.go b/internal/threadgroup/threadgroup.go index 8688474..d418264 100644 --- a/internal/threadgroup/threadgroup.go +++ b/internal/threadgroup/threadgroup.go @@ -1,3 +1,18 @@ +// Package threadgroup exposes a ThreadGroup object which can be used to +// facilitate clean shutdown. A ThreadGroup is similar to a sync.WaitGroup, +// but with two important additions: The ability to detect when shutdown has +// been initiated, and protections against adding more threads after shutdown +// has completed. +// +// ThreadGroup was designed with the following shutdown sequence in mind: +// +// 1. Call Stop, signaling that shutdown has begun. After Stop is called, no +// new goroutines should be created. +// +// 2. Wait for Stop to return. When Stop returns, all goroutines should have +// returned. +// +// 3. Free any resources used by the goroutines. package threadgroup import ( @@ -7,8 +22,8 @@ import ( ) type ( - // A ThreadGroup provides synchronization between a module and its - // goroutines to enable clean shutdowns + // A ThreadGroup is a sync.WaitGroup with additional functionality for + // facilitating clean shutdown. ThreadGroup struct { mu sync.Mutex wg sync.WaitGroup @@ -49,19 +64,16 @@ func (tg *ThreadGroup) WithContext(parent context.Context) (context.Context, con go func() { select { case <-ctx.Done(): - break case <-tg.closed: - break } - // threadgroup or parent context cancelled, cancel the child context - cancel() + cancel() // threadgroup is stopping or context cancelled, cancel the context }() return ctx, cancel } -// AddContext adds a new thread to the group and returns a copy of the parent +// AddWithContext adds a new thread to the group and returns a copy of the parent // context. It is a convenience function combining Add and WithContext. -func (tg *ThreadGroup) AddContext(parent context.Context) (context.Context, context.CancelFunc, error) { +func (tg *ThreadGroup) AddWithContext(parent context.Context) (context.Context, context.CancelFunc, error) { // try to add to the group done, err := tg.Add() if err != nil { @@ -72,6 +84,8 @@ func (tg *ThreadGroup) AddContext(parent context.Context) (context.Context, cont var once sync.Once return ctx, func() { cancel() + // it must be safe to call cancel multiple times, but it is not safe to + // call done multiple times since it's decrementing the waitgroup once.Do(done) }, nil } diff --git a/internal/threadgroup/threadgroup_test.go b/internal/threadgroup/threadgroup_test.go index ca8a964..27e57e3 100644 --- a/internal/threadgroup/threadgroup_test.go +++ b/internal/threadgroup/threadgroup_test.go @@ -33,7 +33,7 @@ func TestThreadgroupContext(t *testing.T) { tg := New() t.Run("context cancel", func(t *testing.T) { - ctx, cancel, err := tg.AddContext(context.Background()) + ctx, cancel, err := tg.AddWithContext(context.Background()) if err != nil { t.Fatal(err) } @@ -55,7 +55,7 @@ func TestThreadgroupContext(t *testing.T) { parentCtx, parentCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer parentCancel() - ctx, cancel, err := tg.AddContext(parentCtx) + ctx, cancel, err := tg.AddWithContext(parentCtx) if err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestThreadgroupContext(t *testing.T) { t.Run("stop", func(t *testing.T) { for i := 0; i < 10; i++ { - _, cancel, err := tg.AddContext(context.Background()) + _, cancel, err := tg.AddWithContext(context.Background()) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index c048bdf..629a80c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -645,7 +645,7 @@ func (ut *updateTx) RevertEvents(index types.ChainIndex) error { return err } -func (ut *updateTx) getOrphanedSiacoinBalance(indexID int64) (map[int64]wallet.Balance, error) { +func (ut *updateTx) orphanedSiacoinBalance(indexID int64) (map[int64]wallet.Balance, error) { const query = `SELECT address_id, siacoin_value, matured FROM siacoin_elements WHERE chain_index_id=$1` @@ -676,7 +676,7 @@ WHERE chain_index_id=$1` return balances, rows.Err() } -func (ut *updateTx) getOrphanedSiafundBalance(indexID int64) (map[int64]uint64, error) { +func (ut *updateTx) orphanedSiafundBalance(indexID int64) (map[int64]uint64, error) { const query = `SELECT address_id, siafund_value FROM siafund_elements WHERE chain_index_id=$1` @@ -732,13 +732,13 @@ func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.Bloc var revertedBalance map[int64]wallet.Balance for _, id := range orphaned { // revert siacoin balances - siacoins, err := ut.getOrphanedSiacoinBalance(id) + siacoins, err := ut.orphanedSiacoinBalance(id) if err != nil { return nil, fmt.Errorf("failed to get orphaned siacoin elements: %w", err) } // revert siafund balances - siafunds, err := ut.getOrphanedSiafundBalance(id) + siafunds, err := ut.orphanedSiafundBalance(id) if err != nil { return nil, fmt.Errorf("failed to get orphaned siafund elements: %w", err) } diff --git a/wallet/manager.go b/wallet/manager.go index 4fd3404..a57ca6f 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -159,7 +159,7 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { // Scan rescans the chain starting from the given index. The scan will complete // when the chain manager reaches the current tip or the context is canceled. func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { - ctx, cancel, err := m.tg.AddContext(ctx) + ctx, cancel, err := m.tg.AddWithContext(ctx) if err != nil { return err } @@ -209,7 +209,7 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) } go func() { - ctx, cancel, err := m.tg.AddContext(context.Background()) + ctx, cancel, err := m.tg.AddWithContext(context.Background()) if err != nil { log.Panic("failed to add to threadgroup", zap.Error(err)) } @@ -236,14 +236,6 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) } m.mu.Lock() - // check that the context was not canceled while waiting for the - // lock - select { - case <-ctx.Done(): - return - default: - } - // update the store lastTip, err := store.LastCommittedIndex() if err != nil { From fbc4b9eb2c843022a3de8a6ee1d17d94e3fa247e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 17 Apr 2024 20:42:50 -0700 Subject: [PATCH 160/630] api,wallet: close wallet manager --- api/api_test.go | 9 +++++++++ wallet/wallet_test.go | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index d2a0012..f254366 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -92,6 +92,7 @@ func TestWalletAdd(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() c, shutdown := runServer(cm, nil, wm) defer shutdown() @@ -279,6 +280,7 @@ func TestWallet(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() // create seed address vault sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) @@ -500,6 +502,7 @@ func TestAddresses(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) @@ -696,6 +699,8 @@ func TestV2(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() + c, shutdown := runServer(cm, nil, wm) defer shutdown() primaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) @@ -920,6 +925,8 @@ func TestP2P(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm1.Close() + l1, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) @@ -961,6 +968,8 @@ func TestP2P(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm2.Close() + l2, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 8924639..df505c6 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -123,6 +123,7 @@ func TestReorg(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -324,6 +325,7 @@ func TestEphemeralBalance(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -519,6 +521,7 @@ func TestWalletAddresses(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() // Add a wallet w := wallet.Wallet{ @@ -648,6 +651,7 @@ func TestV2(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -772,6 +776,7 @@ func TestResubscribe(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -925,6 +930,7 @@ func TestSiafunds(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -1061,6 +1067,7 @@ func TestOrphans(t *testing.T) { if err != nil { t.Fatal(err) } + defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { From 0546bd1421863bec01c45c9bbd1a2767638d2627 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Mon, 22 Apr 2024 22:16:22 +0000 Subject: [PATCH 161/630] ui: v0.19.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 52e72d6..bf6cff4 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.2 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.19.0 + go.sia.tech/web/walletd v0.19.1 go.uber.org/zap v1.27.0 golang.org/x/term v0.19.0 lukechampine.com/flagg v1.1.1 diff --git a/go.sum b/go.sum index f8a87c2..cb68adc 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b h1:nwfLGAR0sjN/zb9QW0xWeNR8MjdtJl6KKZqPo2Amz3U= go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.19.0 h1:lDsLTCGCKi9QNqsBGwf+9oaipD2o0ss8x6CrFqLXo30= -go.sia.tech/web/walletd v0.19.0/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= +go.sia.tech/web/walletd v0.19.1 h1:RvNO/S9bMJTMDJGkzauCK96BZ75Iri2j3k9VfzJkx5Y= +go.sia.tech/web/walletd v0.19.1/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 80f133703424bc0ca47605dd82e796e80b1d1d16 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 23 Apr 2024 11:17:10 +0200 Subject: [PATCH 162/630] persist: remove chain_indices table --- persist/sqlite/addresses.go | 3 +- persist/sqlite/consensus.go | 182 ++++++++++++++++-------------------- persist/sqlite/init.sql | 23 ++--- persist/sqlite/wallet.go | 3 +- wallet/update.go | 7 +- 5 files changed, 94 insertions(+), 124 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 66973e7..1891cea 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -19,9 +19,8 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e // AddressEvents returns the events of a single address. func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ev.height, ev.block_id, ev.event_type, ev.event_data FROM events ev - INNER JOIN chain_indices ci ON (ev.index_id = ci.id) INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) WHERE sa.sia_address = $1 diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 629a80c..9e71889 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -297,14 +297,8 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } defer addrStmt.Close() - indexStmt, err := insertIndexStmt(ut.tx) - if err != nil { - return fmt.Errorf("failed to prepare index statement: %w", err) - } - defer indexStmt.Close() - // ignore elements already in the database. - insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -312,12 +306,6 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - var chainIndexID int64 - err := indexStmt.QueryRow(index.Height, encode(index.ID)).Scan(&chainIndexID) - if err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) @@ -326,7 +314,7 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } var dummyID types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, chainIndexID).Scan(decode(&dummyID)) + err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, encode(index.ID), index.Height).Scan(decode(&dummyID)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { @@ -435,19 +423,13 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty return nil } - indexStmt, err := insertIndexStmt(ut.tx) - if err != nil { - return fmt.Errorf("failed to prepare index statement: %w", err) - } - defer indexStmt.Close() - addrStmt, err := insertAddressStatement(ut.tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -455,11 +437,6 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty balanceChanges := make(map[int64]uint64) for _, se := range elements { - var chainIndexID int64 - if err := indexStmt.QueryRow(index.Height, encode(index.ID)).Scan(&chainIndexID); err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) if err != nil { return fmt.Errorf("failed to query address: %w", err) @@ -468,7 +445,7 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } var dummy types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, chainIndexID).Scan(decode(&dummy)) + err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, encode(index.ID), index.Height).Scan(decode(&dummy)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { @@ -566,13 +543,7 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { return nil } - indexStmt, err := insertIndexStmt(ut.tx) - if err != nil { - return fmt.Errorf("failed to prepare index statement: %w", err) - } - defer indexStmt.Close() - - insertEventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, index_id, event_type, event_data) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (event_id) DO NOTHING RETURNING id`) + insertEventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (event_id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare event statement: %w", err) } @@ -593,19 +564,13 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { var buf bytes.Buffer enc := json.NewEncoder(&buf) for _, event := range events { - var chainIndexID int64 - err := indexStmt.QueryRow(event.Index.Height, encode(event.Index.ID)).Scan(&chainIndexID) - if err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } - buf.Reset() if err := enc.Encode(event.Data); err != nil { return fmt.Errorf("failed to encode event: %w", err) } var eventID int64 - err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), chainIndexID, event.Data.EventType(), buf.String()).Scan(&eventID) + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Data.EventType(), buf.String(), encode(event.Index.ID), event.Index.Height).Scan(&eventID) if errors.Is(err, sql.ErrNoRows) { continue // skip if the event already exists } else if err != nil { @@ -635,21 +600,20 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { return nil } -// RevertEvents reverts any events that were added by the index -func (ut *updateTx) RevertEvents(index types.ChainIndex) error { - var id int64 - err := ut.tx.QueryRow(`DELETE FROM chain_indices WHERE block_id=$1 AND height=$2 RETURNING id`, encode(index.ID), index.Height).Scan(&id) - if errors.Is(err, sql.ErrNoRows) { - return nil +// RevertIndex reverts all siacoin_elements, siafund_elements or events that were added by the index +func (ut *updateTx) RevertIndex(index types.ChainIndex) error { + if _, err := ut.tx.Exec(`DELETE FROM siacoin_elements WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { + return fmt.Errorf("failed to delete siacoin elements: %w", err) + } else if _, err := ut.tx.Exec(`DELETE FROM siafund_elements WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { + return fmt.Errorf("failed to delete siafund elements: %w", err) + } else if _, err := ut.tx.Exec(`DELETE FROM events WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { + return fmt.Errorf("failed to delete events: %w", err) } - return err + return nil } -func (ut *updateTx) orphanedSiacoinBalance(indexID int64) (map[int64]wallet.Balance, error) { - const query = `SELECT address_id, siacoin_value, matured -FROM siacoin_elements -WHERE chain_index_id=$1` - rows, err := ut.tx.Query(query, indexID) +func (ut *updateTx) orphanedSiacoinBalances(index types.ChainIndex) (map[int64]wallet.Balance, error) { + rows, err := ut.tx.Query(`SELECT address_id, siacoin_value, matured FROM siacoin_elements WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } @@ -676,11 +640,8 @@ WHERE chain_index_id=$1` return balances, rows.Err() } -func (ut *updateTx) orphanedSiafundBalance(indexID int64) (map[int64]uint64, error) { - const query = `SELECT address_id, siafund_value -FROM siafund_elements -WHERE chain_index_id=$1` - rows, err := ut.tx.Query(query, indexID) +func (ut *updateTx) orphanedSiafundBalances(index types.ChainIndex) (map[int64]uint64, error) { + rows, err := ut.tx.Query(`SELECT address_id, siafund_value FROM siafund_elements WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query siafund elements: %w", err) } @@ -699,56 +660,62 @@ WHERE chain_index_id=$1` return balances, rows.Err() } -func (ut *updateTx) getOrphanedIndexes(index types.ChainIndex) (orphaned []int64, err error) { - rows, err := ut.tx.Query(`SELECT id FROM chain_indices WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) +func (ut *updateTx) deleteOrphanedSiacoinElements(index types.ChainIndex) (reverted []types.BlockID, _ error) { + rows, err := ut.tx.Query(`DELETE FROM siacoin_elements WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query orphans: %w", err) + } + defer rows.Close() + + for rows.Next() { + var orphan types.BlockID + if err := rows.Scan(decode(&orphan)); err != nil { + return nil, fmt.Errorf("failed to scan orphan: %w", err) + } + reverted = append(reverted, orphan) + } + return reverted, rows.Err() +} + +func (ut *updateTx) deleteOrphanedSiafundElements(index types.ChainIndex) (reverted []types.BlockID, _ error) { + rows, err := ut.tx.Query(`DELETE FROM siafund_elements WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query orphans: %w", err) } defer rows.Close() for rows.Next() { - var indexID int64 - if err := rows.Scan(&indexID); err != nil { + var orphan types.BlockID + if err := rows.Scan(decode(&orphan)); err != nil { return nil, fmt.Errorf("failed to scan orphan: %w", err) } - orphaned = append(orphaned, indexID) + reverted = append(reverted, orphan) } - return orphaned, rows.Err() + return reverted, rows.Err() } // RevertOrphans reverts any chain indices that were orphaned by the given index func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.BlockID, err error) { log := ut.tx.log.Named("RevertOrphans").With(zap.Uint64("height", index.Height), zap.Stringer("applied", index.ID)) - orphaned, err := ut.getOrphanedIndexes(index) + // fetch orphaned siacoin balances + siacoins, err := ut.orphanedSiacoinBalances(index) if err != nil { - return nil, fmt.Errorf("failed to get orphaned indexes: %w", err) + return nil, fmt.Errorf("failed to get orphaned siacoin elements: %w", err) } - if len(orphaned) == 0 { - return nil, nil + // fetch orphaned siafund balances + siafunds, err := ut.orphanedSiafundBalances(index) + if err != nil { + return nil, fmt.Errorf("failed to get orphaned siafund elements: %w", err) } - var revertedBalance map[int64]wallet.Balance - for _, id := range orphaned { - // revert siacoin balances - siacoins, err := ut.orphanedSiacoinBalance(id) - if err != nil { - return nil, fmt.Errorf("failed to get orphaned siacoin elements: %w", err) - } - - // revert siafund balances - siafunds, err := ut.orphanedSiafundBalance(id) - if err != nil { - return nil, fmt.Errorf("failed to get orphaned siafund elements: %w", err) - } - - for addr, balance := range siafunds { - b := siacoins[addr] - b.Siafunds = balance - siacoins[addr] = b - } - revertedBalance = siacoins + // merge the balances + revertedBalance := siacoins + for addr, balance := range siafunds { + b := revertedBalance[addr] + b.Siafunds = balance + revertedBalance[addr] = b } getBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE id=$1`) @@ -786,21 +753,35 @@ func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.Bloc } } - rows, err := ut.tx.Query(`DELETE FROM chain_indices WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) + // delete events + if _, err := ut.tx.Exec(`DELETE FROM events WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)); err != nil { + return nil, err + } + + // delete orphaned siacoin elements + orphanedSCEs, err := ut.deleteOrphanedSiacoinElements(index) if err != nil { - return nil, fmt.Errorf("failed to query orphans: %w", err) + return nil, err } - defer rows.Close() + reverted = append(reverted, orphanedSCEs...) - for rows.Next() { - var orphan types.BlockID - if err := rows.Scan(decode(&orphan)); err != nil { - return nil, fmt.Errorf("failed to scan orphan: %w", err) + // delete orphaned siafund elements + orphanedSFEs, err := ut.deleteOrphanedSiafundElements(index) + if err != nil { + return nil, err + } + + // dedupe orphans + seen := make(map[types.BlockID]struct{}) + for _, orphan := range append(orphanedSCEs, orphanedSFEs...) { + if _, ok := seen[orphan]; !ok { + seen[orphan] = struct{}{} + reverted = append(reverted, orphan) + log.Debug("reverted orphan", zap.Stringer("orphan", orphan)) } - reverted = append(reverted, orphan) - log.Debug("reverted orphan", zap.Stringer("orphan", orphan)) } - return reverted, rows.Err() + + return reverted, nil } // ProcessChainApplyUpdate implements chain.Subscriber @@ -841,8 +822,3 @@ func insertAddressStatement(tx *txn) (*stmt, error) { // the on conflict is effectively a no-op, but enables us to return the id of the existing address return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) } - -func insertIndexStmt(tx *txn) (*stmt, error) { - // the on conflict is effectively a no-op, but enables us to return the id of the existing index - return tx.Prepare(`INSERT INTO chain_indices (height, block_id) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=EXCLUDED.height RETURNING id`) -} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 293073d..e316dfa 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -1,9 +1,3 @@ -CREATE TABLE chain_indices ( - id INTEGER PRIMARY KEY, - block_id BLOB UNIQUE NOT NULL, - height INTEGER UNIQUE NOT NULL -); - CREATE TABLE sia_addresses ( id INTEGER PRIMARY KEY, sia_address BLOB UNIQUE NOT NULL, @@ -20,11 +14,12 @@ CREATE TABLE siacoin_elements ( maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ address_id INTEGER NOT NULL REFERENCES sia_addresses (id), matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ - chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE + block_id BLOB NOT NULL, + height INTEGER NOT NULL ); CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); -CREATE INDEX siacoin_elements_chain_index ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_block_id_height ON siacoin_elements (block_id, height); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -33,21 +28,23 @@ CREATE TABLE siafund_elements ( leaf_index INTEGER NOT NULL, siafund_value INTEGER NOT NULL, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE + block_id BLOB NOT NULL, + height INTEGER NOT NULL ); CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); -CREATE INDEX siafund_elements_chain_index ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_block_id_height ON siafund_elements (block_id, height); CREATE TABLE events ( id INTEGER PRIMARY KEY, event_id BLOB UNIQUE NOT NULL, - index_id BLOB NOT NULL REFERENCES chain_indices (id) ON DELETE CASCADE, maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, event_type TEXT NOT NULL, - event_data BLOB NOT NULL + event_data BLOB NOT NULL, + block_id BLOB NOT NULL, + height INTEGER NOT NULL ); -CREATE INDEX events_index_id ON events (index_id); +CREATE INDEX events_block_id_height ON events (block_id, height); CREATE TABLE event_addresses ( event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 2b654c7..7b0e8eb 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -71,9 +71,8 @@ func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { } func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ev.height, ev.block_id, ev.event_type, ev.event_data FROM events ev - INNER JOIN chain_indices ci ON (ev.index_id = ci.id) WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` diff --git a/wallet/update.go b/wallet/update.go index 909e0ef..f0f1c12 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -33,9 +33,8 @@ type ( ApplyMatureSiacoinBalance(types.ChainIndex) error AddEvents([]Event) error + RevertIndex(index types.ChainIndex) error RevertMatureSiacoinBalance(types.ChainIndex) error - RevertEvents(index types.ChainIndex) error - RevertOrphans(types.ChainIndex) (reverted []types.BlockID, err error) } ) @@ -277,8 +276,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. return fmt.Errorf("failed to update siafund state elements: %w", err) } - // revert events - return tx.RevertEvents(revertedIndex) + // revert index + return tx.RevertIndex(revertedIndex) } func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { From 22ffeb5e833a8212df7feb53288b289655655648 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 24 Apr 2024 13:03:26 +0200 Subject: [PATCH 163/630] persist: remove append --- persist/sqlite/consensus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9e71889..70dcf55 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -763,7 +763,6 @@ func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.Bloc if err != nil { return nil, err } - reverted = append(reverted, orphanedSCEs...) // delete orphaned siafund elements orphanedSFEs, err := ut.deleteOrphanedSiafundElements(index) From b7f9ac172df908c6495e7ae9d14902bee26e227e Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 30 Apr 2024 19:56:43 -0400 Subject: [PATCH 164/630] close correct prepared statements --- persist/sqlite/consensus.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 70dcf55..f34e1d0 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -559,7 +559,7 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { if err != nil { return fmt.Errorf("failed to prepare relevant address statement: %w", err) } - defer addrStmt.Close() + defer relevantAddrStmt.Close() var buf bytes.Buffer enc := json.NewEncoder(&buf) @@ -728,6 +728,7 @@ func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.Bloc if err != nil { return nil, fmt.Errorf("failed to prepare update statement: %w", err) } + defer updateBalanceStmt.Close() for addrID, balance := range revertedBalance { var existing wallet.Balance From 6dddc713e38b2eb68648545fcb4bfdd72626df99 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 30 Apr 2024 15:31:14 -0700 Subject: [PATCH 165/630] cmd, sqlite, wallet: implement soft delete for siacoin elements --- .golangci.yml | 76 +++- api/api_test.go | 30 +- cmd/walletd/node.go | 5 +- persist/sqlite/addresses.go | 7 +- persist/sqlite/consensus.go | 735 ++++++++++++++++++++++++------- persist/sqlite/consensus_test.go | 309 +++++++++++++ persist/sqlite/consts_default.go | 2 + persist/sqlite/consts_testing.go | 2 + persist/sqlite/init.sql | 28 +- persist/sqlite/wallet.go | 7 +- wallet/manager.go | 31 +- wallet/update.go | 109 ++--- wallet/wallet_test.go | 131 ++++-- 13 files changed, 1105 insertions(+), 367 deletions(-) create mode 100644 persist/sqlite/consensus_test.go diff --git a/.golangci.yml b/.golangci.yml index d439ef1..050db11 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,16 +17,6 @@ run: # list of build tags, all linters use it. Default is empty list. build-tags: [] - # default is true. Enables skipping of directories: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs-use-default: true - - # which files to skip: they will be analyzed, but issues from them - # won't be reported. Default value is empty list, but there is - # no need to include all autogenerated files, we confidently recognize - # autogenerated files. If it's not please let us know. - skip-files: [] - # output configuration options output: # print lines of code with issue, default is true @@ -40,11 +30,14 @@ linters-settings: ## Enabled linters: govet: # report about shadowed variables - check-shadowing: false disable-all: false - golint: - min-confidence: 1.0 + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + gocritic: # Which checks should be enabled; can't be combined with 'disabled-checks'; @@ -56,13 +49,16 @@ linters-settings: - style disabled-checks: # diagnostic + - appendAssign - commentedOutCode - uncheckedInlineErr - # style + - httpNoBody - exitAfterDefer - ifElseChain - importShadow + - initClause + - nestingReduce - octalLiteral - paramTypeCombine - ptrToRefParam @@ -76,19 +72,56 @@ linters-settings: revive: ignore-generated-header: true rules: + - name: blank-imports + disabled: false + - name: bool-literal-in-expr + disabled: false + - name: confusing-naming + disabled: false + - name: confusing-results + disabled: false + - name: constant-logical-expr + disabled: false + - name: context-as-argument + disabled: false + - name: exported + disabled: false + - name: errorf + disabled: false + - name: if-return + disabled: false + - name: indent-error-flow + disabled: true + - name: increment-decrement + disabled: false + - name: modifies-value-receiver + disabled: true + - name: optimize-operands-order + disabled: false + - name: range-val-in-closure + disabled: false + - name: struct-tag + disabled: false + - name: superfluous-else + disabled: false + - name: time-equal + disabled: false + - name: unexported-naming + disabled: false + - name: unexported-return + disabled: false + - name: unnecessary-stmt + disabled: false + - name: unreachable-code + disabled: false - name: package-comments disabled: true - tagliatelle: - case: - rules: - json: goCamel - yaml: goCamel - linters: disable-all: true fast: false enable: + - tagliatelle - gocritic - gofmt - revive @@ -96,9 +129,6 @@ linters: - misspell - typecheck - whitespace - - tagliatelle - - unused - - unparam issues: # Maximum issues count per one linter. Set to 0 to disable. Default is 50. diff --git a/api/api_test.go b/api/api_test.go index f254366..d6e1de0 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -88,10 +88,7 @@ func TestWalletAdd(t *testing.T) { } defer ws.Close() - wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, ws, log.Named("wallet")) defer wm.Close() c, shutdown := runServer(cm, nil, wm) @@ -276,10 +273,7 @@ func TestWallet(t *testing.T) { }) // create the wallet manager - wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, ws, log.Named("wallet")) defer wm.Close() // create seed address vault @@ -498,10 +492,7 @@ func TestAddresses(t *testing.T) { } defer ws.Close() - wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, ws, log.Named("wallet")) defer wm.Close() sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) @@ -695,10 +686,7 @@ func TestV2(t *testing.T) { t.Fatal(err) } defer ws.Close() - wm, err := wallet.NewManager(cm, ws, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, ws, log.Named("wallet")) defer wm.Close() c, shutdown := runServer(cm, nil, wm) @@ -921,10 +909,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } - wm1, err := wallet.NewManager(cm1, store1, log1.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm1 := wallet.NewManager(cm1, store1, log1.Named("wallet")) defer wm1.Close() l1, err := net.Listen("tcp", ":0") @@ -964,10 +949,7 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer store2.Close() - wm2, err := wallet.NewManager(cm2, store2, log2.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm2 := wallet.NewManager(cm2, store2, log2.Named("wallet")) defer wm2.Close() l2, err := net.Listen("tcp", ":0") diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 1900d77..9eb9c7f 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -187,10 +187,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, } s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) - wm, err := wallet.NewManager(cm, store, log.Named("wallet")) - if err != nil { - return nil, fmt.Errorf("failed to create wallet manager: %w", err) - } + wm := wallet.NewManager(cm, store, log.Named("wallet")) return &node{ chainStore: bdb, diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 1891cea..b3fadd3 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -19,10 +19,11 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e // AddressEvents returns the events of a single address. func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ev.height, ev.block_id, ev.event_type, ev.event_data + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) + INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) WHERE sa.sia_address = $1 ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` @@ -52,7 +53,7 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address=$1 + WHERE sa.sia_address=$1 AND se.spent_index_id IS NULL LIMIT $2 OFFSET $3` rows, err := tx.Query(query, encode(address), limit, offset) @@ -80,7 +81,7 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address = $1 + WHERE sa.sia_address = $1 AND se.spent_index_id IS NULL LIMIT $2 OFFSET $3` rows, err := tx.Query(query, encode(address), limit, offset) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index f34e1d0..362c1bc 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -23,18 +23,8 @@ type addressRef struct { Balance wallet.Balance } -func scanStateElement(s scanner) (se types.StateElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) - return -} - -func scanAddress(s scanner) (ab addressRef, err error) { - err = s.Scan(&ab.ID, decode(&ab.Balance.Siacoins), decode(&ab.Balance.ImmatureSiacoins), &ab.Balance.Siafunds) - return -} - func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { - const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements WHERE spent_index_id IS NULL` rows, err := ut.tx.Query(query) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) @@ -53,6 +43,9 @@ func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { } func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) error { + log := ut.tx.log.Named("UpdateSiacoinStateElements") + log.Debug("updating siacoin state elements", zap.Int("count", len(elements))) + const query = `UPDATE siacoin_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` stmt, err := ut.tx.Prepare(query) if err != nil { @@ -66,12 +59,13 @@ func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) er if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } + log.Debug("updated element proof", zap.Stringer("id", se.ID), zap.Uint64("leafIndex", se.LeafIndex)) } return nil } func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { - const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements WHERE spent_index_id IS NULL` rows, err := ut.tx.Query(query) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) @@ -129,11 +123,131 @@ func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, return } -func (ut *updateTx) ApplyMatureSiacoinBalance(index types.ChainIndex) error { - const query = `SELECT id, se.address_id, se.siacoin_value -FROM siacoin_elements se -WHERE maturity_height=$1 AND matured=false` - rows, err := ut.tx.Query(query, index.Height) +func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState) error { + tx := ut.tx + log := tx.log.Named("ApplyIndex").With(zap.Stringer("blockID", index.ID), zap.Uint64("height", index.Height)) + + if err := revertOrphans(tx, index, log.Named("revertOrphans")); err != nil { + return fmt.Errorf("failed to revert orphans: %w", err) + } + + if err := applyMatureSiacoinBalance(tx, index, log.Named("applyMatureSiacoinBalance")); err != nil { + return fmt.Errorf("failed to apply mature siacoin balance: %w", err) + } + + var indexID int64 + if err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=height RETURNING id`, encode(index.ID), index.Height).Scan(&indexID); err != nil { + return fmt.Errorf("failed to insert chain index: %w", err) + } + + if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { + return fmt.Errorf("failed to spend siacoin elements: %w", err) + } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, log.Named("addSiacoinElements")); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } + + if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { + return fmt.Errorf("failed to spend siafund elements: %w", err) + } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } + + if err := addEvents(tx, state.Events, indexID); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + return nil +} + +func (ut *updateTx) RevertIndex(index types.ChainIndex, state wallet.RevertedState) error { + tx := ut.tx + + if err := revertSpentSiacoinElements(tx, state.UnspentSiacoinElements); err != nil { + return fmt.Errorf("failed to revert spent siacoin elements: %w", err) + } else if err := removeSiacoinElements(tx, state.DeletedSiacoinElements); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) + } + + if err := revertSpentSiafundElements(tx, state.UnspentSiafundElements); err != nil { + return fmt.Errorf("failed to revert spent siafund elements: %w", err) + } else if err := removeSiafundElements(tx, state.DeletedSiafundElements); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + + if err := revertEvents(tx, index); err != nil { + return fmt.Errorf("failed to revert events: %w", err) + } else if err := revertMatureSiacoinBalance(tx, index); err != nil { + return fmt.Errorf("failed to revert mature siacoin balance: %w", err) + } + return nil +} + +// ProcessChainApplyUpdate implements chain.Subscriber +func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { + log := s.log.Named("UpdateChainState").With(zap.Int("reverted", len(reverted)), zap.Int("applied", len(applied))) + return s.transaction(func(tx *txn) error { + utx := &updateTx{ + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } + + if err := wallet.UpdateChainState(utx, reverted, applied, log); err != nil { + return fmt.Errorf("failed to update chain state: %w", err) + } else if err := setLastCommittedIndex(tx, applied[len(applied)-1].State.Index); err != nil { + return fmt.Errorf("failed to set last committed index: %w", err) + } + + height := applied[len(applied)-1].State.Index.Height + + if height > spentElementRetentionBlocks { + pruneHeight := height - spentElementRetentionBlocks + + siacoins, err := pruneSpentSiacoinElements(tx, pruneHeight) + if err != nil { + return fmt.Errorf("failed to cleanup siacoin elements: %w", err) + } + + siafunds, err := pruneSpentSiafundElements(tx, pruneHeight) + if err != nil { + return fmt.Errorf("failed to cleanup siafund elements: %w", err) + } + + if len(siacoins) > 0 || len(siafunds) > 0 { + log.Debug("pruned elements", zap.Stringers("siacoins", siacoins), zap.Stringers("siafunds", siafunds), zap.Uint64("pruneHeight", pruneHeight)) + } + } + return nil + }) +} + +// LastCommittedIndex returns the last chain index that was committed. +func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { + err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index)) + return +} + +// ResetLastIndex resets the last indexed tip to trigger a full rescan. +func (s *Store) ResetLastIndex() error { + _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(types.ChainIndex{})) + return err +} + +func scanStateElement(s scanner) (se types.StateElement, err error) { + err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) + return +} + +func scanAddress(s scanner) (ab addressRef, err error) { + err = s.Scan(&ab.ID, decode(&ab.Balance.Siacoins), decode(&ab.Balance.ImmatureSiacoins), &ab.Balance.Siafunds) + return +} + +func applyMatureSiacoinBalance(tx *txn, index types.ChainIndex, log *zap.Logger) error { + log = log.With(zap.Uint64("maturityHeight", index.Height)) + log.Debug("applying mature siacoin balance") + const query = `SELECT id, address_id, siacoin_value +FROM siacoin_elements +WHERE maturity_height=$1 AND matured=false AND spent_index_id IS NULL` + rows, err := tx.Query(query, index.Height) if err != nil { return fmt.Errorf("failed to query siacoin elements: %w", err) } @@ -151,25 +265,26 @@ WHERE maturity_height=$1 AND matured=false` } balanceDelta[addressID] = balanceDelta[addressID].Add(value) matured = append(matured, outputID) + log.Debug("matured siacoin output", zap.Stringer("outputID", outputID), zap.Int64("addressID", addressID), zap.Stringer("value", value)) } if err := rows.Err(); err != nil { return fmt.Errorf("failed to scan siacoin elements: %w", err) } - updateMaturedStmt, err := ut.tx.Prepare(`UPDATE siacoin_elements SET matured=true WHERE id=$1`) + updateMaturedStmt, err := tx.Prepare(`UPDATE siacoin_elements SET matured=true WHERE id=$1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer updateMaturedStmt.Close() - getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + getAddressBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer getAddressBalanceStmt.Close() - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -207,11 +322,11 @@ WHERE maturity_height=$1 AND matured=false` return nil } -func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { +func revertMatureSiacoinBalance(tx *txn, index types.ChainIndex) error { const query = `SELECT se.id, se.address_id, se.siacoin_value FROM siacoin_elements se - WHERE maturity_height=$1 AND matured=true` - rows, err := ut.tx.Query(query, index.Height) + WHERE maturity_height=$1 AND matured=true AND spent_index_id IS NULL` + rows, err := tx.Query(query, index.Height) if err != nil { return fmt.Errorf("failed to query siacoin elements: %w", err) } @@ -235,19 +350,19 @@ func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { return fmt.Errorf("failed to scan siacoin elements: %w", err) } - updateMaturedStmt, err := ut.tx.Prepare(`UPDATE siacoin_elements SET matured=false WHERE id=$1`) + updateMaturedStmt, err := tx.Prepare(`UPDATE siacoin_elements SET matured=false WHERE id=$1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer updateMaturedStmt.Close() - getAddressBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + getAddressBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } defer getAddressBalanceStmt.Close() - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -286,19 +401,19 @@ func (ut *updateTx) RevertMatureSiacoinBalance(index types.ChainIndex) error { return nil } -func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index types.ChainIndex) error { +func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, log *zap.Logger) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(ut.tx) + addrStmt, err := insertAddressStatement(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() // ignore elements already in the database. - insertStmt, err := ut.tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -314,8 +429,9 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty } var dummyID types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, encode(index.ID), index.Height).Scan(decode(&dummyID)) + err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID).Scan(decode(&dummyID)) if errors.Is(err, sql.ErrNoRows) { + log.Debug("siacoin element already exists", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address)) continue // skip if the element already exists } else if err != nil { return fmt.Errorf("failed to execute statement: %w", err) @@ -323,10 +439,12 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty // update the balance if the element does not exist balance := balanceChanges[addrRef.ID] - if se.MaturityHeight <= index.Height { + if se.MaturityHeight == 0 { balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) + log.Debug("added siacoin output", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value)) } else { balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(se.SiacoinOutput.Value) + log.Debug("added immature siacoin output", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value), zap.Uint64("maturityHeight", se.MaturityHeight)) } balanceChanges[addrRef.ID] = balance } @@ -335,7 +453,7 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } @@ -354,18 +472,18 @@ func (ut *updateTx) AddSiacoinElements(elements []types.SiacoinElement, index ty return nil } -func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinElement, index types.ChainIndex) error { +func removeSiacoinElements(tx *txn, elements []types.SiacoinElement) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(ut.tx) + addrStmt, err := insertAddressStatement(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - stmt, err := ut.tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id`) + stmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id, matured`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -381,13 +499,14 @@ func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinElement, index } var dummy types.Hash256 - err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) + var matured bool + err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy), &matured) if err != nil { return fmt.Errorf("failed to delete element %q: %w", se.ID, err) } balance := balanceChanges[addrRef.ID] - if se.MaturityHeight < index.Height { + if matured { balance.Siacoins = balance.Siacoins.Sub(se.SiacoinOutput.Value) } else { balance.ImmatureSiacoins = balance.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) @@ -399,7 +518,7 @@ func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinElement, index return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } @@ -418,18 +537,140 @@ func (ut *updateTx) RemoveSiacoinElements(elements []types.SiacoinElement, index return nil } -func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index types.ChainIndex) error { +func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error { + if len(elements) == 0 { + return nil + } + + addrStmt, err := insertAddressStatement(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64) error { + if len(elements) == 0 { + return nil + } + + addrStmt, err := insertAddressStatement(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siacoins = balance.Siacoins.Sub(se.SiacoinOutput.Value) + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(ut.tx) + addrStmt, err := insertAddressStatement(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - insertStmt, err := ut.tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -445,7 +686,7 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty } var dummy types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, encode(index.ID), index.Height).Scan(decode(&dummy)) + err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID).Scan(decode(&dummy)) if errors.Is(err, sql.ErrNoRows) { continue // skip if the element already exists } else if err != nil { @@ -458,7 +699,7 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } @@ -477,18 +718,18 @@ func (ut *updateTx) AddSiafundElements(elements []types.SiafundElement, index ty return nil } -func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index types.ChainIndex) error { +func removeSiafundElements(tx *txn, elements []types.SiafundElement) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(ut.tx) + addrStmt, err := insertAddressStatement(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - stmt, err := ut.tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) + stmt, err := tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -519,7 +760,7 @@ func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index return nil } - updateAddressBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) if err != nil { return fmt.Errorf("failed to prepare update balance statement: %w", err) } @@ -538,24 +779,150 @@ func (ut *updateTx) RemoveSiafundElements(elements []types.SiafundElement, index return nil } -func (ut *updateTx) AddEvents(events []wallet.Event) error { +func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) error { + if len(elements) == 0 { + return nil + } + + addrStmt, err := insertAddressStatement(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + if balance.Siafunds < se.SiafundOutput.Value { + panic("siafund balance cannot be negative") + } + balance.Siafunds -= se.SiafundOutput.Value + + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance.Siafunds, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error { + if len(elements) == 0 { + return nil + } + + addrStmt, err := insertAddressStatement(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siafunds += se.SiafundOutput.Value + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance.Siafunds, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func addEvents(tx *txn, events []wallet.Event, indexID int64) error { if len(events) == 0 { return nil } - insertEventStmt, err := ut.tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, block_id, height) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (event_id) DO NOTHING RETURNING id`) + insertEventStmt, err := tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (event_id) DO NOTHING RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare event statement: %w", err) } defer insertEventStmt.Close() - addrStmt, err := ut.tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) + addrStmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() - relevantAddrStmt, err := ut.tx.Prepare(`INSERT INTO event_addresses (event_id, address_id) VALUES ($1, $2) ON CONFLICT (event_id, address_id) DO NOTHING`) + relevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id) VALUES ($1, $2) ON CONFLICT (event_id, address_id) DO NOTHING`) if err != nil { return fmt.Errorf("failed to prepare relevant address statement: %w", err) } @@ -570,7 +937,7 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { } var eventID int64 - err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Data.EventType(), buf.String(), encode(event.Index.ID), event.Index.Height).Scan(&eventID) + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Data.EventType(), buf.String(), indexID).Scan(&eventID) if errors.Is(err, sql.ErrNoRows) { continue // skip if the event already exists } else if err != nil { @@ -600,20 +967,45 @@ func (ut *updateTx) AddEvents(events []wallet.Event) error { return nil } -// RevertIndex reverts all siacoin_elements, siafund_elements or events that were added by the index -func (ut *updateTx) RevertIndex(index types.ChainIndex) error { - if _, err := ut.tx.Exec(`DELETE FROM siacoin_elements WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { - return fmt.Errorf("failed to delete siacoin elements: %w", err) - } else if _, err := ut.tx.Exec(`DELETE FROM siafund_elements WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { - return fmt.Errorf("failed to delete siafund elements: %w", err) - } else if _, err := ut.tx.Exec(`DELETE FROM events WHERE block_id=$1 AND height=$2`, encode(index.ID), index.Height); err != nil { - return fmt.Errorf("failed to delete events: %w", err) +// RevertEvents reverts any events that were added by the index +func revertEvents(tx *txn, index types.ChainIndex) error { + const query = `DELETE FROM events WHERE chain_index_id IN (SELECT id FROM chain_indices WHERE block_id=$1 AND height=$2)` + _, err := tx.Exec(query, encode(index.ID), index.Height) + return err +} + +func revertSpentOrphanedSiacoinElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]wallet.Balance, error) { + rows, err := tx.Query(`UPDATE siacoin_elements SET spent_index_id=NULL WHERE id IN (SELECT se.id FROM siacoin_elements se +INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING address_id, siacoin_value`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } - return nil + defer rows.Close() + + balances := make(map[int64]wallet.Balance) + for rows.Next() { + var addrID int64 + var value types.Currency + + if err := rows.Scan(&addrID, decode(&value)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + + balance := balances[addrID] + balance.Siacoins = balance.Siacoins.Add(value) + balances[addrID] = balance + log.Debug("reverting spent orphaned siacoin element", zap.Stringer("value", value)) + } + return balances, rows.Err() } -func (ut *updateTx) orphanedSiacoinBalances(index types.ChainIndex) (map[int64]wallet.Balance, error) { - rows, err := ut.tx.Query(`SELECT address_id, siacoin_value, matured FROM siacoin_elements WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) +func deleteOrphanedSiacoinElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]wallet.Balance, error) { + rows, err := tx.Query(`DELETE FROM siacoin_elements WHERE id IN (SELECT se.id FROM siacoin_elements se +INNER JOIN chain_indices ci ON (ci.id=se.chain_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siacoin_value, matured, spent_index_id IS NOT NULL`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } @@ -621,27 +1013,33 @@ func (ut *updateTx) orphanedSiacoinBalances(index types.ChainIndex) (map[int64]w balances := make(map[int64]wallet.Balance) for rows.Next() { + var outputID types.SiacoinOutputID var addrID int64 var value types.Currency var matured bool + var spent bool - if err := rows.Scan(&addrID, decode(&value), &matured); err != nil { + if err := rows.Scan(decode(&outputID), &addrID, decode(&value), &matured, &spent); err != nil { return nil, fmt.Errorf("failed to scan siacoin element: %w", err) } balance := balances[addrID] - if matured { - balance.Siacoins = balance.Siacoins.Add(value) - } else { + if !matured { balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(value) + } else if !spent { + balance.Siacoins = balance.Siacoins.Add(value) } balances[addrID] = balance + log.Debug("deleting orphaned siacoin element", zap.Stringer("id", outputID), zap.Stringer("value", value), zap.Bool("matured", matured), zap.Bool("spent", spent)) } return balances, rows.Err() } -func (ut *updateTx) orphanedSiafundBalances(index types.ChainIndex) (map[int64]uint64, error) { - rows, err := ut.tx.Query(`SELECT address_id, siafund_value FROM siafund_elements WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) +func revertSpentOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { + rows, err := tx.Query(`UPDATE siafund_elements SET spent_index_id=NULL WHERE id IN (SELECT se.id FROM siafund_elements se +INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siafund_value`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query siafund elements: %w", err) } @@ -649,168 +1047,175 @@ func (ut *updateTx) orphanedSiafundBalances(index types.ChainIndex) (map[int64]u balances := make(map[int64]uint64) for rows.Next() { + var outputID types.SiafundOutputID var addrID int64 var value uint64 - if err := rows.Scan(&addrID, &value); err != nil { + if err := rows.Scan(decode(&outputID), &addrID, value); err != nil { return nil, fmt.Errorf("failed to scan siafund element: %w", err) } - balances[addrID] += value + + balance := balances[addrID] + balance += value + balances[addrID] = balance + log.Debug("reverting spent orphaned siafund element", zap.Stringer("id", outputID), zap.Uint64("value", value)) } return balances, rows.Err() } -func (ut *updateTx) deleteOrphanedSiacoinElements(index types.ChainIndex) (reverted []types.BlockID, _ error) { - rows, err := ut.tx.Query(`DELETE FROM siacoin_elements WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) +func deleteOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { + rows, err := tx.Query(`DELETE FROM siafund_elements WHERE id IN (SELECT se.id FROM siafund_elements se +INNER JOIN chain_indices ci ON (ci.id=se.chain_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siafund_value, spent_index_id IS NOT NULL`, index.Height, encode(index.ID)) if err != nil { - return nil, fmt.Errorf("failed to query orphans: %w", err) + return nil, fmt.Errorf("failed to query siafund elements: %w", err) } defer rows.Close() + balances := make(map[int64]uint64) for rows.Next() { - var orphan types.BlockID - if err := rows.Scan(decode(&orphan)); err != nil { - return nil, fmt.Errorf("failed to scan orphan: %w", err) + var outputID types.SiafundOutputID + var addrID int64 + var value uint64 + var spent bool + + if err := rows.Scan(decode(&outputID), &addrID, &value, &spent); err != nil { + return nil, fmt.Errorf("failed to scan siafund element: %w", err) } - reverted = append(reverted, orphan) + balances[addrID] += value + log.Debug("deleting orphaned siafund element", zap.Stringer("id", outputID), zap.Uint64("value", value), zap.Bool("spent", spent)) } - return reverted, rows.Err() + return balances, rows.Err() } -func (ut *updateTx) deleteOrphanedSiafundElements(index types.ChainIndex) (reverted []types.BlockID, _ error) { - rows, err := ut.tx.Query(`DELETE FROM siafund_elements WHERE height=$1 AND block_id<>$2 RETURNING block_id`, index.Height, encode(index.ID)) +func deleteOrphanedEvents(tx *txn, index types.ChainIndex) error { + _, err := tx.Exec(`DELETE FROM events WHERE id IN (SELECT ev.id FROM events ev +INNER JOIN chain_indices ci ON (ev.chain_index_id=ci.id) +WHERE ci.height=$1 AND ci.block_id<>$2);`, index.Height, encode(index.ID)) + return err +} + +// revertOrphans reverts any chain indices that were orphaned by the given index +func revertOrphans(tx *txn, index types.ChainIndex, log *zap.Logger) error { + // fetch orphaned siacoin balances + deletedSiacoins, err := deleteOrphanedSiacoinElements(tx, index, log.Named("deleteOrphanedSiacoinElements")) if err != nil { - return nil, fmt.Errorf("failed to query orphans: %w", err) + return fmt.Errorf("failed to get orphaned siacoin elements: %w", err) } - defer rows.Close() - for rows.Next() { - var orphan types.BlockID - if err := rows.Scan(decode(&orphan)); err != nil { - return nil, fmt.Errorf("failed to scan orphan: %w", err) - } - reverted = append(reverted, orphan) + // fetch orphaned siafund balances + deletedSiafunds, err := deleteOrphanedSiafundElements(tx, index, log.Named("deleteOrphanedSiafundElements")) + if err != nil { + return fmt.Errorf("failed to get orphaned siafund elements: %w", err) } - return reverted, rows.Err() -} - -// RevertOrphans reverts any chain indices that were orphaned by the given index -func (ut *updateTx) RevertOrphans(index types.ChainIndex) (reverted []types.BlockID, err error) { - log := ut.tx.log.Named("RevertOrphans").With(zap.Uint64("height", index.Height), zap.Stringer("applied", index.ID)) - // fetch orphaned siacoin balances - siacoins, err := ut.orphanedSiacoinBalances(index) + unspentSiacoins, err := revertSpentOrphanedSiacoinElements(tx, index, log.Named("revertSpentOrphanedSiacoinElements")) if err != nil { - return nil, fmt.Errorf("failed to get orphaned siacoin elements: %w", err) + return fmt.Errorf("failed to revert spent orphaned siacoin elements: %w", err) } - // fetch orphaned siafund balances - siafunds, err := ut.orphanedSiafundBalances(index) + unspentSiafunds, err := revertSpentOrphanedSiafundElements(tx, index, log.Named("revertSpentOrphanedSiafundElements")) if err != nil { - return nil, fmt.Errorf("failed to get orphaned siafund elements: %w", err) + return fmt.Errorf("failed to revert spent orphaned siafund elements: %w", err) } - // merge the balances - revertedBalance := siacoins - for addr, balance := range siafunds { - b := revertedBalance[addr] - b.Siafunds = balance - revertedBalance[addr] = b + // get the addrIDs of all affected addresses + addrIDs := make(map[int64]bool) + for id := range deletedSiacoins { + addrIDs[id] = true + } + for id := range deletedSiafunds { + addrIDs[id] = true + } + for id := range unspentSiacoins { + addrIDs[id] = true + } + for id := range unspentSiafunds { + addrIDs[id] = true } - getBalanceStmt, err := ut.tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE id=$1`) + getBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE id=$1`) if err != nil { - return nil, fmt.Errorf("failed to prepare balance statement: %w", err) + return fmt.Errorf("failed to prepare balance statement: %w", err) } defer getBalanceStmt.Close() - updateBalanceStmt, err := ut.tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE id=$4`) + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE id=$4`) if err != nil { - return nil, fmt.Errorf("failed to prepare update statement: %w", err) + return fmt.Errorf("failed to prepare update statement: %w", err) } defer updateBalanceStmt.Close() - for addrID, balance := range revertedBalance { + for addrID := range addrIDs { var existing wallet.Balance err := getBalanceStmt.QueryRow(addrID).Scan(decode(&existing.Siacoins), decode(&existing.ImmatureSiacoins), &existing.Siafunds) if err != nil { - return nil, fmt.Errorf("failed to get balance: %w", err) + return fmt.Errorf("failed to get balance: %w", err) } - existing.Siacoins = existing.Siacoins.Sub(balance.Siacoins) - existing.ImmatureSiacoins = existing.ImmatureSiacoins.Sub(balance.ImmatureSiacoins) - if existing.Siafunds < balance.Siafunds { + existing.Siacoins = existing.Siacoins.Sub(deletedSiacoins[addrID].Siacoins) + existing.ImmatureSiacoins = existing.ImmatureSiacoins.Sub(deletedSiacoins[addrID].ImmatureSiacoins) + if existing.Siafunds < deletedSiafunds[addrID] { panic("siafund balance cannot be negative") } - existing.Siafunds -= balance.Siafunds + existing.Siafunds -= deletedSiafunds[addrID] + + existing.Siacoins = existing.Siacoins.Add(unspentSiacoins[addrID].Siacoins) + existing.Siafunds += unspentSiafunds[addrID] res, err := updateBalanceStmt.Exec(encode(existing.Siacoins), encode(existing.ImmatureSiacoins), existing.Siafunds, addrID) if err != nil { - return nil, fmt.Errorf("failed to update balance: %w", err) + return fmt.Errorf("failed to update balance: %w", err) } else if n, err := res.RowsAffected(); err != nil { - return nil, fmt.Errorf("failed to get rows affected: %w", err) + return fmt.Errorf("failed to get rows affected: %w", err) } else if n != 1 { - return nil, fmt.Errorf("expected 1 row affected, got %v", n) + return fmt.Errorf("expected 1 row affected, got %v", n) } } - // delete events - if _, err := ut.tx.Exec(`DELETE FROM events WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)); err != nil { - return nil, err + if err := deleteOrphanedEvents(tx, index); err != nil { + return fmt.Errorf("failed to delete orphaned events: %w", err) } - // delete orphaned siacoin elements - orphanedSCEs, err := ut.deleteOrphanedSiacoinElements(index) - if err != nil { - return nil, err - } + _, err = tx.Exec(`DELETE FROM chain_indices WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) + return err +} - // delete orphaned siafund elements - orphanedSFEs, err := ut.deleteOrphanedSiafundElements(index) +func pruneSpentSiacoinElements(tx *txn, height uint64) (removed []types.SiacoinOutputID, err error) { + const query = `DELETE FROM siacoin_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1) RETURNING id` + rows, err := tx.Query(query, height) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) } + defer rows.Close() - // dedupe orphans - seen := make(map[types.BlockID]struct{}) - for _, orphan := range append(orphanedSCEs, orphanedSFEs...) { - if _, ok := seen[orphan]; !ok { - seen[orphan] = struct{}{} - reverted = append(reverted, orphan) - log.Debug("reverted orphan", zap.Stringer("orphan", orphan)) + for rows.Next() { + var id types.SiacoinOutputID + if err := rows.Scan(decode(&id)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) } + removed = append(removed, id) } - - return reverted, nil + return removed, rows.Err() } -// ProcessChainApplyUpdate implements chain.Subscriber -func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { - return s.transaction(func(tx *txn) error { - utx := &updateTx{ - tx: tx, - relevantAddresses: make(map[types.Address]bool), - } +func pruneSpentSiafundElements(tx *txn, height uint64) (removed []types.SiafundOutputID, err error) { + const query = `DELETE FROM siafund_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1) RETURNING id` + rows, err := tx.Query(query, height) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() - if err := wallet.UpdateChainState(utx, reverted, applied); err != nil { - return fmt.Errorf("failed to update chain state: %w", err) - } else if err := setLastCommittedIndex(tx, applied[len(applied)-1].State.Index); err != nil { - return fmt.Errorf("failed to set last committed index: %w", err) + for rows.Next() { + var id types.SiafundOutputID + if err := rows.Scan(decode(&id)); err != nil { + return nil, fmt.Errorf("failed to scan siafund element: %w", err) } - return nil - }) -} - -// LastCommittedIndex returns the last chain index that was committed. -func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { - err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index)) - return -} - -// ResetLastIndex resets the last indexed tip to trigger a full rescan. -func (s *Store) ResetLastIndex() error { - _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(types.ChainIndex{})) - return err + removed = append(removed, id) + } + return removed, rows.Err() } func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go new file mode 100644 index 0000000..af40db9 --- /dev/null +++ b/persist/sqlite/consensus_test.go @@ -0,0 +1,309 @@ +package sqlite + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" +) + +func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + } + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { + index, err := store.LastCommittedIndex() + if err != nil { + tb.Fatalf("failed to get last committed index: %v", err) + } + for index != cm.Tip() { + crus, caus, err := cm.UpdatesSince(index, 1000) + if err != nil { + tb.Fatalf("failed to subscribe to chain manager: %v", err) + } else if err := store.UpdateChainState(crus, caus); err != nil { + tb.Fatalf("failed to update chain state: %v", err) + } + index = caus[len(caus)-1].State.Index + } +} + +func TestPruneSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + // create a wallet + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + // mine a block to the wallet + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + assertBalance := func(siacoin, immature types.Currency) { + t.Helper() + + b, err := db.WalletBalance(w.ID) + if err != nil { + t.Fatalf("failed to get wallet balance: %v", err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } + } + + assertUTXOs := func(spent int, unspent int) { + t.Helper() + + var n int + err := db.db.QueryRow(`SELECT COUNT(*) FROM siacoin_elements WHERE spent_index_id IS NOT NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count spent siacoin elements: %v", err) + } else if n != spent { + t.Fatalf("expected %v spent siacoin elements, got %v", spent, n) + } + + err = db.db.QueryRow(`SELECT COUNT(*) FROM siacoin_elements WHERE spent_index_id IS NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count unspent siacoin elements: %v", err) + } else if n != unspent { + t.Fatalf("expected %v unspent siacoin elements, got %v", unspent, n) + } + } + + assertBalance(types.ZeroCurrency, expectedPayout) + assertUTXOs(0, 1) + + // mine until the payout matures + for i := 0; i < int(maturityHeight); i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + syncDB(t, db, cm) + assertBalance(expectedPayout, types.ZeroCurrency) + assertUTXOs(0, 1) + + // spend the utxo + utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatalf("failed to get wallet siacoin outputs: %v", err) + } + + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Value: utxos[0].SiacoinOutput.Value, Address: types.VoidAddress}, + }, + } + + sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: utxos[0].ID, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + PublicKeyIndex: 0, + Timelock: 0, + Signature: sig[:], + }) + + // mine a block with the transaction + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + // the utxo should now have 0 balance and 1 spent element + assertBalance(types.ZeroCurrency, types.ZeroCurrency) + assertUTXOs(1, 0) + + // mine until the element is pruned + for i := 0; i < spentElementRetentionBlocks-1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(1, 0) // check that the element is not pruned early + } + + // trigger the pruning + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(0, 0) +} + +func TestPruneSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafund airdrop to the wallet + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + // create a wallet + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + syncDB(t, db, cm) + + assertBalance := func(siafunds uint64) { + t.Helper() + + b, err := db.WalletBalance(w.ID) + if err != nil { + t.Fatalf("failed to get wallet balance: %v", err) + } else if b.Siafunds != siafunds { + t.Fatalf("expected siafund balance %v, got %v", siafunds, b.ImmatureSiacoins) + } + } + + assertUTXOs := func(spent int, unspent int) { + t.Helper() + + var n int + err := db.db.QueryRow(`SELECT COUNT(*) FROM siafund_elements WHERE spent_index_id IS NOT NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count spent siacoin elements: %v", err) + } else if n != spent { + t.Fatalf("expected %v spent siacoin elements, got %v", spent, n) + } + + err = db.db.QueryRow(`SELECT COUNT(*) FROM siafund_elements WHERE spent_index_id IS NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count unspent siacoin elements: %v", err) + } else if n != unspent { + t.Fatalf("expected %v unspent siacoin elements, got %v", unspent, n) + } + } + + assertBalance(cm.TipState().SiafundCount()) + assertUTXOs(0, 1) + + // spend the utxo + utxos, err := db.WalletSiafundOutputs(w.ID, 0, 100) + if err != nil { + t.Fatalf("failed to get wallet siacoin outputs: %v", err) + } + + txn := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: types.SiafundOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + SiafundOutputs: []types.SiafundOutput{ + {Value: utxos[0].SiafundOutput.Value, Address: types.VoidAddress}, + }, + } + + sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: utxos[0].ID, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + PublicKeyIndex: 0, + Timelock: 0, + Signature: sig[:], + }) + + // mine a block with the transaction + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + // the utxo should now have 0 balance and 1 spent element + assertBalance(0) + assertUTXOs(1, 0) + + // mine until the element is pruned + for i := 0; i < spentElementRetentionBlocks-1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) // check that the element is not pruned early + assertUTXOs(1, 0) + } + + // the spent element should now be pruned + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(0, 0) +} diff --git a/persist/sqlite/consts_default.go b/persist/sqlite/consts_default.go index 50b7330..19f84e5 100644 --- a/persist/sqlite/consts_default.go +++ b/persist/sqlite/consts_default.go @@ -9,4 +9,6 @@ const ( maxRetryAttempts = 30 // 30 attempts factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds maxBackoff = 15 * time.Second + + spentElementRetentionBlocks = 144 // 1 day ) diff --git a/persist/sqlite/consts_testing.go b/persist/sqlite/consts_testing.go index f4911e3..e4ade1e 100644 --- a/persist/sqlite/consts_testing.go +++ b/persist/sqlite/consts_testing.go @@ -9,4 +9,6 @@ const ( maxRetryAttempts = 10 // 10 attempts factor = 2.0 // factor ^ retryAttempts = backoff time in milliseconds maxBackoff = 15 * time.Second + + spentElementRetentionBlocks = 36 ) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index e316dfa..c4c3537 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -1,3 +1,10 @@ +CREATE TABLE chain_indices ( + id INTEGER PRIMARY KEY, + block_id BLOB UNIQUE NOT NULL, + height INTEGER UNIQUE NOT NULL +); +CREATE INDEX chain_indices_height ON chain_indices (block_id, height); + CREATE TABLE sia_addresses ( id INTEGER PRIMARY KEY, sia_address BLOB UNIQUE NOT NULL, @@ -14,12 +21,13 @@ CREATE TABLE siacoin_elements ( maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ address_id INTEGER NOT NULL REFERENCES sia_addresses (id), matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ - block_id BLOB NOT NULL, - height INTEGER NOT NULL + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ ); CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); -CREATE INDEX siacoin_elements_block_id_height ON siacoin_elements (block_id, height); +CREATE INDEX siacoin_elements_chain_index_id ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id ON siacoin_elements (spent_index_id); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -28,23 +36,23 @@ CREATE TABLE siafund_elements ( leaf_index INTEGER NOT NULL, siafund_value INTEGER NOT NULL, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - block_id BLOB NOT NULL, - height INTEGER NOT NULL + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ ); CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); -CREATE INDEX siafund_elements_block_id_height ON siafund_elements (block_id, height); +CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); CREATE TABLE events ( id INTEGER PRIMARY KEY, + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), event_id BLOB UNIQUE NOT NULL, maturity_height INTEGER NOT NULL, date_created INTEGER NOT NULL, event_type TEXT NOT NULL, - event_data BLOB NOT NULL, - block_id BLOB NOT NULL, - height INTEGER NOT NULL + event_data BLOB NOT NULL ); -CREATE INDEX events_block_id_height ON events (block_id, height); +CREATE INDEX events_chain_index_id ON events (chain_index_id); CREATE TABLE event_addresses ( event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 7b0e8eb..2608687 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -71,8 +71,9 @@ func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { } func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ev.height, ev.block_id, ev.event_type, ev.event_data + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev + INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` @@ -299,7 +300,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) + WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) LIMIT $2 OFFSET $3` rows, err := tx.Query(query, id, limit, offset) @@ -331,7 +332,7 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) + WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) LIMIT $2 OFFSET $3` rows, err := tx.Query(query, id, limit, offset) diff --git a/wallet/manager.go b/wallet/manager.go index a57ca6f..c75aefd 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -195,7 +195,7 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch } // NewManager creates a new wallet manager. -func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) { +func NewManager(cm ChainManager, store Store, log *zap.Logger) *Manager { m := &Manager{ chain: cm, store: store, @@ -203,31 +203,24 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) tg: threadgroup.New(), } - lastTip, err := store.LastCommittedIndex() - if err != nil { - return nil, fmt.Errorf("failed to get last committed index: %w", err) - } + reorgChan := make(chan struct{}, 1) + reorgChan <- struct{}{} + unsubscribe := cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgChan <- struct{}{}: + default: + } + }) go func() { + defer unsubscribe() + ctx, cancel, err := m.tg.AddWithContext(context.Background()) if err != nil { log.Panic("failed to add to threadgroup", zap.Error(err)) } defer cancel() - if err := syncStore(ctx, store, cm, lastTip); err != nil { - log.Fatal("failed to subscribe to chain manager", zap.Error(err)) - } - - reorgChan := make(chan types.ChainIndex, 1) - unsubscribe := cm.OnReorg(func(index types.ChainIndex) { - select { - case reorgChan <- index: - default: - } - }) - defer unsubscribe() - for { select { case <-ctx.Done(): @@ -246,5 +239,5 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) (*Manager, error) m.mu.Unlock() } }() - return m, nil + return m } diff --git a/wallet/update.go b/wallet/update.go index f0f1c12..81990e4 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -5,6 +5,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.uber.org/zap" ) type ( @@ -14,6 +15,21 @@ type ( Balance } + AppliedState struct { + Events []Event + CreatedSiacoinElements []types.SiacoinElement + SpentSiacoinElements []types.SiacoinElement + CreatedSiafundElements []types.SiafundElement + SpentSiafundElements []types.SiafundElement + } + + RevertedState struct { + UnspentSiacoinElements []types.SiacoinElement + DeletedSiacoinElements []types.SiacoinElement + UnspentSiafundElements []types.SiafundElement + DeletedSiafundElements []types.SiafundElement + } + // An UpdateTx atomically updates the state of a store. UpdateTx interface { SiacoinStateElements() ([]types.StateElement, error) @@ -22,34 +38,16 @@ type ( SiafundStateElements() ([]types.StateElement, error) UpdateSiafundStateElements([]types.StateElement) error - AddSiacoinElements([]types.SiacoinElement, types.ChainIndex) error - RemoveSiacoinElements([]types.SiacoinElement, types.ChainIndex) error - - AddSiafundElements([]types.SiafundElement, types.ChainIndex) error - RemoveSiafundElements([]types.SiafundElement, types.ChainIndex) error - AddressRelevant(types.Address) (bool, error) - ApplyMatureSiacoinBalance(types.ChainIndex) error - AddEvents([]Event) error - - RevertIndex(index types.ChainIndex) error - RevertMatureSiacoinBalance(types.ChainIndex) error - RevertOrphans(types.ChainIndex) (reverted []types.BlockID, err error) + ApplyIndex(types.ChainIndex, AppliedState) error + RevertIndex(types.ChainIndex, RevertedState) error } ) // applyChainUpdate atomically applies a chain update to a store func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { - // revert any orphaned chain indices - if _, err := tx.RevertOrphans(cau.State.Index); err != nil { - return fmt.Errorf("failed to revert orphans: %w", err) - } - - // update the immature balance of each relevant address - if err := tx.ApplyMatureSiacoinBalance(cau.State.Index); err != nil { - return fmt.Errorf("failed to get matured siacoin elements: %w", err) - } + var applied AppliedState // determine which siacoin and siafund elements are ephemeral // @@ -73,7 +71,6 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } // add new siacoin elements to the store - var newSiacoinElements, spentSiacoinElements []types.SiacoinElement cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if ephemeral[se.ID] { return @@ -87,19 +84,12 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } if spent { - spentSiacoinElements = append(spentSiacoinElements, se) + applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, se) } else { - newSiacoinElements = append(newSiacoinElements, se) + applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, se) } }) - if err := tx.AddSiacoinElements(newSiacoinElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to add siacoin elements: %w", err) - } else if err := tx.RemoveSiacoinElements(spentSiacoinElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to remove siacoin elements: %w", err) - } - - var newSiafundElements, spentSiafundElements []types.SiafundElement cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { if ephemeral[se.ID] { return @@ -113,18 +103,12 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } if spent { - spentSiafundElements = append(spentSiafundElements, se) + applied.SpentSiafundElements = append(applied.SpentSiafundElements, se) } else { - newSiafundElements = append(newSiafundElements, se) + applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, se) } }) - if err := tx.AddSiafundElements(newSiafundElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to add siafund elements: %w", err) - } else if err := tx.RemoveSiafundElements(spentSiafundElements, cau.State.Index); err != nil { - return fmt.Errorf("failed to remove siafund elements: %w", err) - } - // add events relevant := func(addr types.Address) bool { relevant, err := tx.AddressRelevant(addr) @@ -133,9 +117,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } return relevant } - if err := tx.AddEvents(AppliedEvents(cau.State, cau.Block, cau, relevant)); err != nil { - return fmt.Errorf("failed to add events: %w", err) - } + applied.Events = AppliedEvents(cau.State, cau.Block, cau, relevant) // fetch all siacoin and siafund state elements siacoinStateElements, err := tx.SiacoinStateElements() @@ -165,11 +147,17 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { if err := tx.UpdateSiafundStateElements(siafundStateElements); err != nil { return fmt.Errorf("failed to update siacoin state elements: %w", err) } + + if err := tx.ApplyIndex(cau.State.Index, applied); err != nil { + return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) + } return nil } // revertChainUpdate atomically reverts a chain update from a store func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { + var reverted RevertedState + // determine which siacoin and siafund elements are ephemeral // // note: I thought we could use LeafIndex == EphemeralLeafIndex, but @@ -191,7 +179,6 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } } - var removedSiacoinElements, addedSiacoinElements []types.SiacoinElement cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { if ephemeral[se.ID] { return @@ -206,20 +193,13 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. if spent { // re-add any spent siacoin elements - addedSiacoinElements = append(addedSiacoinElements, se) + reverted.UnspentSiacoinElements = append(reverted.UnspentSiacoinElements, se) } else { // delete any created siacoin elements - removedSiacoinElements = append(removedSiacoinElements, se) + reverted.DeletedSiacoinElements = append(reverted.DeletedSiacoinElements, se) } }) - if err := tx.AddSiacoinElements(addedSiacoinElements, revertedIndex); err != nil { - return fmt.Errorf("failed to add siacoin elements: %w", err) - } else if err := tx.RemoveSiacoinElements(removedSiacoinElements, revertedIndex); err != nil { - return fmt.Errorf("failed to remove siacoin elements: %w", err) - } - - var removedSiafundElements, addedSiafundElements []types.SiafundElement cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { if ephemeral[se.ID] { return @@ -234,23 +214,15 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. if spent { // re-add any spent siafund elements - addedSiafundElements = append(addedSiafundElements, se) + reverted.UnspentSiafundElements = append(reverted.UnspentSiafundElements, se) } else { // delete any created siafund elements - removedSiafundElements = append(removedSiafundElements, se) + reverted.DeletedSiafundElements = append(reverted.DeletedSiafundElements, se) } }) - // revert siafund element changes - if err := tx.AddSiafundElements(addedSiafundElements, revertedIndex); err != nil { - return fmt.Errorf("failed to add siafund elements: %w", err) - } else if err := tx.RemoveSiafundElements(removedSiafundElements, revertedIndex); err != nil { - return fmt.Errorf("failed to remove siafund elements: %w", err) - } - - // revert mature siacoin balance for each relevant address - if err := tx.RevertMatureSiacoinBalance(revertedIndex); err != nil { - return fmt.Errorf("failed to get matured siacoin elements: %w", err) + if err := tx.RevertIndex(revertedIndex, reverted); err != nil { + return fmt.Errorf("failed to revert index %q: %w", revertedIndex, err) } siacoinElements, err := tx.SiacoinStateElements() @@ -275,12 +247,10 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. if err := tx.UpdateSiafundStateElements(siafundElements); err != nil { return fmt.Errorf("failed to update siafund state elements: %w", err) } - - // revert index - return tx.RevertIndex(revertedIndex) + return nil } -func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { +func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate, log *zap.Logger) error { for _, cru := range reverted { revertedIndex := types.ChainIndex{ ID: cru.Block.ID(), @@ -289,12 +259,15 @@ func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chai if err := revertChainUpdate(tx, cru, revertedIndex); err != nil { return fmt.Errorf("failed to revert chain update %q: %w", revertedIndex, err) } + log.Debug("reverted chain update", zap.Stringer("blockID", revertedIndex.ID), zap.Uint64("height", revertedIndex.Height)) } for _, cau := range applied { + // apply the chain update if err := applyChainUpdate(tx, cau); err != nil { return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) } + log.Debug("applied chain update", zap.Stringer("blockID", cau.State.Index.ID), zap.Uint64("height", cau.State.Index.Height)) } return nil } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index df505c6..0283abb 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -20,6 +20,7 @@ import ( ) func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { + tb.Helper() for i := 0; i < 1000; i++ { time.Sleep(10 * time.Millisecond) tip, _ := ws.LastCommittedIndex() @@ -119,10 +120,7 @@ func TestReorg(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -181,9 +179,10 @@ func TestReorg(t *testing.T) { // mine to trigger a reorg var blocks []types.Block state := genesisState - for i := 0; i < 5; i++ { - blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) - state.Index.ID = blocks[len(blocks)-1].ID() + for i := 0; i < 10; i++ { + block := mineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + state.Index.ID = block.ID() state.Index.Height++ } if err := cm.AddBlocks(blocks); err != nil { @@ -321,10 +320,7 @@ func TestEphemeralBalance(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -517,10 +513,7 @@ func TestWalletAddresses(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() // Add a wallet @@ -647,10 +640,7 @@ func TestV2(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -742,7 +732,7 @@ func TestV2(t *testing.T) { } } -func TestResubscribe(t *testing.T) { +func TestScan(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) @@ -772,10 +762,7 @@ func TestResubscribe(t *testing.T) { cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() pk2 := types.GeneratePrivateKey() @@ -926,10 +913,7 @@ func TestSiafunds(t *testing.T) { cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() pk2 := types.GeneratePrivateKey() @@ -1056,6 +1040,8 @@ func TestOrphans(t *testing.T) { defer bdb.Close() network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + network.HardforkV2.AllowHeight = 200 + network.HardforkV2.RequireHeight = 201 store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { @@ -1063,10 +1049,7 @@ func TestOrphans(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -1082,21 +1065,28 @@ func TestOrphans(t *testing.T) { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } + + // mine until the maturity height + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } waitForBlock(t, cm, db) assertBalance := func(siacoin, immature types.Currency) error { b, err := wm.WalletBalance(w.ID) if err != nil { return fmt.Errorf("failed to check balance: %w", err) - } else if !b.Siacoins.Equals(siacoin) { - return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) } else if !b.ImmatureSiacoins.Equals(immature) { return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) } return nil } - if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { t.Fatal(err) } @@ -1122,8 +1112,54 @@ func TestOrphans(t *testing.T) { t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) } + resetState := cm.TipState() + + // send a transaction that will be orphaned + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: expectedPayout.Div64(2)}, // send the other half to the void + {Address: addr, Value: expectedPayout.Div64(2)}, // send half the payout back to the wallet + }, + Signatures: []types.TransactionSignature{ + { + ParentID: utxos[0].ID, + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } else if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + if err := assertBalance(expectedPayout.Div64(2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } + // simulate an interrupted rescan by closing the wallet manager, resetting the - // last rescan index, and initializing a new wallet manager. + // last scan index, and initializing a new wallet manager. if err := wm.Close(); err != nil { t.Fatal(err) } else if err := db.ResetLastIndex(); err != nil { @@ -1134,7 +1170,7 @@ func TestOrphans(t *testing.T) { // orphaned blocks that will not be cleanly reverted since the rescan was // interrupted. var blocks []types.Block - state := genesisState + state := resetState for i := 0; i < 5; i++ { blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) state.Index.ID = blocks[len(blocks)-1].ID() @@ -1144,32 +1180,31 @@ func TestOrphans(t *testing.T) { t.Fatal(err) } - wm, err = wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm = wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() waitForBlock(t, cm, db) - // check that the balance was reverted - if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + // check that the transaction was reverted + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { t.Fatal(err) } - // check that the payout event was reverted + // check that the transaction event was reverted events, err = wm.Events(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(events) != 0 { - t.Fatalf("expected 0 events, got %v", len(events)) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) } // check that the utxo was reverted utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(utxos) != 0 { - t.Fatalf("expected 0 output, got %v", len(utxos)) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if !utxos[0].SiacoinOutput.Value.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) } } From 9b0446d650ffe63d4194b142ada69f5350e9b9ab Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 30 Apr 2024 15:36:24 -0700 Subject: [PATCH 166/630] ci: fix lint errors --- api/api.go | 1 + persist/sqlite/consensus.go | 2 +- wallet/update.go | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api/api.go b/api/api.go index 9972477..b059350 100644 --- a/api/api.go +++ b/api/api.go @@ -94,6 +94,7 @@ type SeedSignRequest struct { Keys []uint64 `json:"keys"` } +// RescanResponse contains information about the state of a chain rescan. type RescanResponse struct { StartIndex types.ChainIndex `json:"startIndex"` Index types.ChainIndex `json:"index"` diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 362c1bc..c98bd56 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -181,7 +181,7 @@ func (ut *updateTx) RevertIndex(index types.ChainIndex, state wallet.RevertedSta return nil } -// ProcessChainApplyUpdate implements chain.Subscriber +// UpdateChainState implements chain.Subscriber func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { log := s.log.Named("UpdateChainState").With(zap.Int("reverted", len(reverted)), zap.Int("applied", len(applied))) return s.transaction(func(tx *txn) error { diff --git a/wallet/update.go b/wallet/update.go index 81990e4..8a42c6a 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -15,6 +15,8 @@ type ( Balance } + // AppliedState contains all state changes made to a store after applying a chain + // update. AppliedState struct { Events []Event CreatedSiacoinElements []types.SiacoinElement @@ -23,6 +25,8 @@ type ( SpentSiafundElements []types.SiafundElement } + // RevertedState contains all state changes made to a store after reverting + // a chain update. RevertedState struct { UnspentSiacoinElements []types.SiacoinElement DeletedSiacoinElements []types.SiacoinElement @@ -250,6 +254,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. return nil } +// UpdateChainState atomically updates the state of a store with a set of +// updates from the chain manager. func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate, log *zap.Logger) error { for _, cru := range reverted { revertedIndex := types.ChainIndex{ From 969647bcb3d685a8838038b923822d8d885f64f1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 1 May 2024 10:51:14 -0700 Subject: [PATCH 167/630] sqlite: fix foreign key failure when deleting a wallet --- persist/sqlite/wallet.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 7b0e8eb..5cb8c11 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -175,8 +175,13 @@ func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { // addresses that were previously associated with the wallet. func (s *Store) DeleteWallet(id wallet.ID) error { return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`DELETE FROM wallet_addresses WHERE wallet_id=$1`, id) + if err != nil { + return fmt.Errorf("failed to delete wallet addresses: %w", err) + } + var dummyID int64 - err := tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, id).Scan(&dummyID) + err = tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, id).Scan(&dummyID) if errors.Is(err, sql.ErrNoRows) { return wallet.ErrNotFound } From bc72ac3560b521f17b8be152c5efa5001f9b986a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 1 May 2024 10:52:36 -0700 Subject: [PATCH 168/630] wallet: add regression test --- wallet/wallet_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index df505c6..17f938c 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1173,3 +1173,47 @@ func TestOrphans(t *testing.T) { t.Fatalf("expected 0 output, got %v", len(utxos)) } } + +func TestDeleteWallet(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, log.Named("wallet")) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + if err := wm.DeleteWallet(w.ID); err != nil { + t.Fatal(err) + } +} From b5444ab0ebe2b22d84d10af7d84ba82ca7049efc Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 1 May 2024 13:31:49 -0700 Subject: [PATCH 169/630] sqlite: overwrite element proof during rescan --- persist/sqlite/consensus.go | 56 ++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index c98bd56..547a675 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -148,7 +148,7 @@ func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { return fmt.Errorf("failed to spend siafund elements: %w", err) - } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID); err != nil { + } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, log.Named("addSiafundElements")); err != nil { return fmt.Errorf("failed to add siafund elements: %w", err) } @@ -412,8 +412,14 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, } defer addrStmt.Close() + existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siacoin_elements WHERE id=$1)`) + if err != nil { + return fmt.Errorf("failed to prepare exists statement: %w", err) + } + defer existsStmt.Close() + // ignore elements already in the database. - insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO NOTHING RETURNING id`) + insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -428,16 +434,22 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, balanceChanges[addrRef.ID] = addrRef.Balance } - var dummyID types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID).Scan(decode(&dummyID)) - if errors.Is(err, sql.ErrNoRows) { - log.Debug("siacoin element already exists", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address)) - continue // skip if the element already exists - } else if err != nil { + var exists bool + err = existsStmt.QueryRow(encode(se.ID)).Scan(&exists) + if err != nil { + return fmt.Errorf("failed to check if siacoin element exists: %w", err) + } + + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) + if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } + // skip balance update if the element already exists + if exists { + log.Debug("updated siacoin element", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value)) + continue + } - // update the balance if the element does not exist balance := balanceChanges[addrRef.ID] if se.MaturityHeight == 0 { balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) @@ -659,7 +671,7 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 return nil } -func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) error { +func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, log *zap.Logger) error { if len(elements) == 0 { return nil } @@ -670,7 +682,13 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) } defer addrStmt.Close() - insertStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO NOTHING RETURNING id`) + existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siafund_elements WHERE id=$1)`) + if err != nil { + return fmt.Errorf("failed to prepare exists statement: %w", err) + } + defer existsStmt.Close() + + insertStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -685,12 +703,18 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) balanceChanges[addrRef.ID] = addrRef.Balance.Siafunds } - var dummy types.Hash256 - err = insertStmt.QueryRow(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID).Scan(decode(&dummy)) - if errors.Is(err, sql.ErrNoRows) { - continue // skip if the element already exists - } else if err != nil { + var exists bool + if err := existsStmt.QueryRow(encode(se.ID)).Scan(&exists); err != nil { + return fmt.Errorf("failed to check if siafund element exists: %w", err) + } + + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) + if err != nil { return fmt.Errorf("failed to execute statement: %w", err) + } else if exists { + // skip balance update if the element already exists + log.Debug("updated siafund element", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("value", se.SiafundOutput.Value)) + continue } balanceChanges[addrRef.ID] += se.SiafundOutput.Value } From 40374b2827d1771c56545b89e260138e8057d32a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 1 May 2024 13:34:01 -0700 Subject: [PATCH 170/630] sqlite: update all existing state elements --- persist/sqlite/consensus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 547a675..c529dd7 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -24,7 +24,7 @@ type addressRef struct { } func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { - const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements WHERE spent_index_id IS NULL` + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` rows, err := ut.tx.Query(query) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) @@ -65,7 +65,7 @@ func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) er } func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { - const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements WHERE spent_index_id IS NULL` + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` rows, err := ut.tx.Query(query) if err != nil { return nil, fmt.Errorf("failed to query siacoin elements: %w", err) From ece0273fcf3034345b14fb20b83e2cd2bd42f2ae Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 1 May 2024 13:53:48 -0700 Subject: [PATCH 171/630] wallet: additional v2 tests --- wallet/wallet_test.go | 857 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 739 insertions(+), 118 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 0283abb..ceee944 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -614,124 +614,6 @@ func TestWalletAddresses(t *testing.T) { } } -func TestV2(t *testing.T) { - pk := types.GeneratePrivateKey() - addr := types.StandardUnlockHash(pk.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) - defer wm.Close() - - w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } - - expectedPayout := cm.TipState().BlockReward() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) - - // check that the payout was received - balance, err := db.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } - - // mine until the payout matures - maturityHeight := cm.TipState().MaturityHeight() + 1 - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm, db) - - // create a v2 transaction that spends the matured payout - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } - - sce := utxos[0] - policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: policy}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{ - {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, - {Address: addr, Value: types.Siacoins(100)}, - }, - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} - - if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) - - // check that the change was received - balance, err = wm.AddressBalance(addr) - if err != nil { - t.Fatal(err) - } else if !balance.Siacoins.Equals(types.Siacoins(100)) { - t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) - } - - // check that a transaction event was recorded - events, err = wm.Events(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 2 { - t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) - } else if events[0].Relevant[0] != addr { - t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) - } -} - func TestScan(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() @@ -1208,3 +1090,742 @@ func TestOrphans(t *testing.T) { t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) } } + +func TestV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm := wallet.NewManager(cm, db, log.Named("wallet")) + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // mine until the payout matures + maturityHeight := cm.TipState().MaturityHeight() + 1 + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // create a v2 transaction that spends the matured payout + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, + {Address: addr, Value: types.Siacoins(100)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the change was received + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(100)) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a transaction event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Relevant[0] != addr { + t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) + } +} + +func TestScanV2(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testV2Network(addr) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm := wallet.NewManager(cm, db, log.Named("wallet")) + defer wm.Close() + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + // rescan to get the genesis Siafund state + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + checkBalance := func(siacoin, immature types.Currency) error { + waitForBlock(t, cm, db) + + // note: the siafund balance is currently hardcoded to the number of + // siafunds in genesis. If we ever modify this test to also spend + // siafunds, this will need to be updated. + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if b.Siafunds != network.GenesisState().SiafundCount() { + return fmt.Errorf("expected siafund balance %v, got %v", network.GenesisState().SiafundCount(), b.Siafunds) + } + return nil + } + + // check that the wallet has no balance + if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + expectedBalance1 := cm.TipState().BlockReward() + // mine a block to fund the first address + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr)}); err != nil { + t.Fatal(err) + } + + // mine a block to fund the second address + expectedBalance2 := cm.TipState().BlockReward() + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr2)}); err != nil { + t.Fatal(err) + } + + // check that the wallet has one immature payout + if err := checkBalance(types.ZeroCurrency, expectedBalance1); err != nil { + t.Fatal(err) + } + + // mine until the first payout matures + for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance did not change + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // add the second address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + if err := checkBalance(expectedBalance1, expectedBalance2); err != nil { + t.Fatal(err) + } + + // mine a block to mature the second payout + if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // sanity check + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + // spend the payout + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the first address has a balance of zero + if err := checkBalance(expectedBalance2, types.ZeroCurrency); err != nil { + t.Fatal(err) + } +} + +func TestReorgV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm := wallet.NewManager(cm, db, log.Named("wallet")) + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } + + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 10; i++ { + block := mineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + state.Index.ID = block.ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the payout event was reverted + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // check that the utxo was removed + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } + + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight = cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + waitForBlock(t, cm, db) + + // check that the balance was updated + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that only the single utxo still exists + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // spend the payout + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that all UTXOs have been spent + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 output, got %v", len(utxos)) + } +} + +func TestOrphansV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm := wallet.NewManager(cm, db, log.Named("wallet")) + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // mine until the maturity height + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } + return nil + } + + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } + + // check that the utxo was created + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + resetState := cm.TipState() + + // send a transaction that will be orphaned + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: expectedPayout.Div64(2)}, // send the other half to the void + {Address: addr, Value: expectedPayout.Div64(2)}, // send half the payout back to the wallet + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + // broadcast the transaction + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + if err := assertBalance(expectedPayout.Div64(2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // simulate an interrupted rescan by closing the wallet manager, resetting the + // last scan index, and initializing a new wallet manager. + if err := wm.Close(); err != nil { + t.Fatal(err) + } else if err := db.ResetLastIndex(); err != nil { + t.Fatal(err) + } + + // mine to trigger a reorg. The underlying store must properly revert the + // orphaned blocks that will not be cleanly reverted since the rescan was + // interrupted. + var blocks []types.Block + state := resetState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + wm = wallet.NewManager(cm, db, log.Named("wallet")) + defer wm.Close() + + waitForBlock(t, cm, db) + + // check that the transaction was reverted + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was reverted + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } + + // check that the utxo was reverted + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if !utxos[0].SiacoinOutput.Value.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } + + // spend the payout + txn = types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that all UTXOs have been spent + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 output, got %v", len(utxos)) + } +} From be7ac0e72e39386113ea2010961d57797d2aca27 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 2 May 2024 14:27:35 -0700 Subject: [PATCH 172/630] wallet: fix wallet Event JSON serialization --- wallet/wallet.go | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 3255ebb..a80958b 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -227,31 +227,34 @@ func (*EventContractPayout) EventType() string { return EventTypeContractPayout func (e Event) MarshalJSON() ([]byte, error) { val, _ := json.Marshal(e.Data) return json.Marshal(struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Val json.RawMessage `json:"val"` + ID types.Hash256 `json:"id"` + Timestamp time.Time `json:"timestamp"` + Index types.ChainIndex `json:"index"` + MaturityHeight uint64 `json:"maturityHeight"` + Relevant []types.Address `json:"relevant"` + Type string `json:"type"` + Data json.RawMessage `json:"data"` }{ - ID: e.ID, - Timestamp: e.Timestamp, - Index: e.Index, - Relevant: e.Relevant, - Type: e.Data.EventType(), - Val: val, + ID: e.ID, + Timestamp: e.Timestamp, + Index: e.Index, + MaturityHeight: e.MaturityHeight, + Relevant: e.Relevant, + Type: e.Data.EventType(), + Data: val, }) } // UnmarshalJSON implements json.Unarshaler. func (e *Event) UnmarshalJSON(data []byte) error { var s struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Val json.RawMessage `json:"val"` + ID types.Hash256 `json:"id"` + Timestamp time.Time `json:"timestamp"` + Index types.ChainIndex `json:"index"` + MaturityHeight uint64 `json:"maturityHeight"` + Relevant []types.Address `json:"relevant"` + Type string `json:"type"` + Data json.RawMessage `json:"data"` } if err := json.Unmarshal(data, &s); err != nil { return err @@ -259,6 +262,7 @@ func (e *Event) UnmarshalJSON(data []byte) error { e.ID = s.ID e.Timestamp = s.Timestamp e.Index = s.Index + e.MaturityHeight = s.MaturityHeight e.Relevant = s.Relevant switch s.Type { case (*EventTransaction)(nil).EventType(): @@ -271,7 +275,7 @@ func (e *Event) UnmarshalJSON(data []byte) error { if e.Data == nil { return fmt.Errorf("unknown event type %q", s.Type) } - return json.Unmarshal(s.Val, e.Data) + return json.Unmarshal(s.Data, e.Data) } // A HostAnnouncement represents a host announcement within an EventTransaction. From cbe75bba7d1862e51ae9634aeef28e739cc5ac6e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 3 May 2024 08:18:02 -0700 Subject: [PATCH 173/630] wallet: fix broken test --- wallet/wallet_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index f6cf8d0..3fbe6f4 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1856,10 +1856,7 @@ func TestDeleteWallet(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, log.Named("wallet")) - if err != nil { - t.Fatal(err) - } + wm := wallet.NewManager(cm, db, log.Named("wallet")) defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) From 9651347a902a2b0e06163c7acc72f7189c76a006 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 16:57:00 +0000 Subject: [PATCH 174/630] build(deps): bump golang.org/x/term from 0.19.0 to 0.20.0 Bumps [golang.org/x/term](https://github.com/golang/term) from 0.19.0 to 0.20.0. - [Commits](https://github.com/golang/term/compare/v0.19.0...v0.20.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bf6cff4..c5f4a1e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.19.1 go.uber.org/zap v1.27.0 - golang.org/x/term v0.19.0 + golang.org/x/term v0.20.0 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 @@ -25,6 +25,6 @@ require ( go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index cb68adc..6804257 100644 --- a/go.sum +++ b/go.sum @@ -37,10 +37,10 @@ golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From ce08271805f71bdcf50ce79177bb12f5ebbc4df6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 07:18:07 +0000 Subject: [PATCH 175/630] build(deps): bump go.sia.tech/core from 0.2.2 to 0.2.3 Bumps [go.sia.tech/core](https://github.com/SiaFoundation/core) from 0.2.2 to 0.2.3. - [Commits](https://github.com/SiaFoundation/core/compare/v0.2.2...v0.2.3) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c5f4a1e..386112e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.21.8 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.2 + go.sia.tech/core v0.2.3 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.19.1 @@ -24,7 +24,7 @@ require ( go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 6804257..aca616f 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.sia.tech/core v0.2.2 h1:33RJrt08o7KyUOY4tITH6ECmRq1lhtapqc/SncIF/2A= -go.sia.tech/core v0.2.2/go.mod h1:Zk7HaybEPgkPC1p6e6tTQr8PIeZClTgNcLNGYDLQJeE= +go.sia.tech/core v0.2.3 h1:k+10zeV1V4bYFCGFaUiubbwRxlyV96WXForVlKnc8Rc= +go.sia.tech/core v0.2.3/go.mod h1:24liZWimivGQF+h3d14ly9oEpMIYxHPSgEMKmunxxi0= go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 h1:5AuiglkLdoBenrg41cJXJ4wTxkVTo85Asj9SPljnmiE= go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= @@ -30,8 +30,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= From c5c90cfe1593fd1b33631c6b140d408e26281c93 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 2 May 2024 13:55:56 -0700 Subject: [PATCH 176/630] api,cmd,sqlite,wallet: add "full" and "none" indexing modes --- api/api.go | 11 +- api/api_test.go | 30 +- api/server.go | 2 + cmd/walletd/main.go | 16 +- cmd/walletd/node.go | 8 +- persist/sqlite/addresses.go | 49 ++- persist/sqlite/consensus.go | 220 +++++++++---- persist/sqlite/init.go | 2 +- persist/sqlite/init.sql | 11 +- persist/sqlite/peers_test.go | 4 +- persist/sqlite/store.go | 5 +- persist/sqlite/wallet.go | 268 +++++++++------ wallet/manager.go | 87 ++++- wallet/options.go | 20 ++ wallet/update.go | 137 ++++---- wallet/wallet_test.go | 621 +++++++++++++++++++++++++---------- 16 files changed, 1054 insertions(+), 437 deletions(-) create mode 100644 wallet/options.go diff --git a/api/api.go b/api/api.go index b059350..45a6a02 100644 --- a/api/api.go +++ b/api/api.go @@ -11,11 +11,12 @@ import ( // A StateResponse returns information about the current state of the walletd // daemon. type StateResponse struct { - Version string `json:"version"` - Commit string `json:"commit"` - OS string `json:"os"` - BuildTime time.Time `json:"buildTime"` - StartTime time.Time `json:"startTime"` + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + BuildTime time.Time `json:"buildTime"` + StartTime time.Time `json:"startTime"` + IndexMode wallet.IndexMode `json:"indexMode"` } // A GatewayPeer is a currently-connected peer. diff --git a/api/api_test.go b/api/api_test.go index d6e1de0..a7222b8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -88,7 +88,10 @@ func TestWalletAdd(t *testing.T) { } defer ws.Close() - wm := wallet.NewManager(cm, ws, log.Named("wallet")) + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() c, shutdown := runServer(cm, nil, wm) @@ -273,7 +276,10 @@ func TestWallet(t *testing.T) { }) // create the wallet manager - wm := wallet.NewManager(cm, ws, log.Named("wallet")) + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() // create seed address vault @@ -492,7 +498,10 @@ func TestAddresses(t *testing.T) { } defer ws.Close() - wm := wallet.NewManager(cm, ws, log.Named("wallet")) + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) @@ -686,7 +695,10 @@ func TestV2(t *testing.T) { t.Fatal(err) } defer ws.Close() - wm := wallet.NewManager(cm, ws, log.Named("wallet")) + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() c, shutdown := runServer(cm, nil, wm) @@ -909,7 +921,10 @@ func TestP2P(t *testing.T) { t.Fatal(err) } - wm1 := wallet.NewManager(cm1, store1, log1.Named("wallet")) + wm1, err := wallet.NewManager(cm1, store1, wallet.WithLogger(log1.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm1.Close() l1, err := net.Listen("tcp", ":0") @@ -949,7 +964,10 @@ func TestP2P(t *testing.T) { t.Fatal(err) } defer store2.Close() - wm2 := wallet.NewManager(cm2, store2, log2.Named("wallet")) + wm2, err := wallet.NewManager(cm2, store2, wallet.WithLogger(log2.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm2.Close() l2, err := net.Listen("tcp", ":0") diff --git a/api/server.go b/api/server.go index 3778993..23b5603 100644 --- a/api/server.go +++ b/api/server.go @@ -48,6 +48,7 @@ type ( // A WalletManager manages wallets, keyed by name. WalletManager interface { + IndexMode() wallet.IndexMode Tip() (types.ChainIndex, error) Scan(_ context.Context, index types.ChainIndex) error @@ -97,6 +98,7 @@ func (s *server) stateHandler(jc jape.Context) { OS: runtime.GOOS, BuildTime: build.Time(), StartTime: s.startTime, + IndexMode: s.wm.IndexMode(), }) } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index cab41ed..ac75bb4 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -11,6 +11,7 @@ import ( cwallet "go.sia.tech/coreutils/wallet" "go.sia.tech/walletd/api" "go.sia.tech/walletd/build" + "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/term" @@ -68,7 +69,7 @@ Runs a CPU miner. Not intended for production use. func main() { log.SetFlags(0) - var gatewayAddr, apiAddr, dir, network, seed string + var gatewayAddr, apiAddr, dir, network, seed, indexModeStr string var upnp, bootstrap bool var minerAddrStr string @@ -83,6 +84,7 @@ func main() { rootCmd.BoolVar(&upnp, "upnp", true, "attempt to forward ports and discover IP with UPnP") rootCmd.BoolVar(&bootstrap, "bootstrap", true, "attempt to bootstrap the network") rootCmd.StringVar(&seed, "seed", "", "testnet seed") + rootCmd.StringVar(&indexModeStr, "index", "full", "address index mode (full, partial, off)") versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) mineCmd := flagg.New("mine", mineUsage) @@ -135,7 +137,17 @@ func main() { // redirect stdlib log to zap zap.RedirectStdLog(logger.Named("stdlib")) - n, err := newNode(gatewayAddr, dir, network, upnp, bootstrap, logger) + var indexMode wallet.IndexMode + switch indexModeStr { + case "full": + indexMode = wallet.IndexModeFull + case "partial": + indexMode = wallet.IndexModePartial + case "off": + indexMode = wallet.IndexModeNone + } + + n, err := newNode(gatewayAddr, dir, network, upnp, bootstrap, indexMode, logger) if err != nil { log.Fatal(err) } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 9eb9c7f..cc55770 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -100,7 +100,7 @@ func (n *node) Close() error { return n.store.Close() } -func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, log *zap.Logger) (*node, error) { +func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, indexMode wallet.IndexMode, log *zap.Logger) (*node, error) { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -187,8 +187,10 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, } s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) - wm := wallet.NewManager(cm, store, log.Named("wallet")) - + wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(indexMode)) + if err != nil { + return nil, fmt.Errorf("failed to create wallet manager: %w", err) + } return &node{ chainStore: bdb, cm: cm, diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index b3fadd3..e2da36b 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -1,6 +1,8 @@ package sqlite import ( + "database/sql" + "errors" "fmt" "go.sia.tech/core/types" @@ -11,7 +13,12 @@ import ( func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - return tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) + err := tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) + if errors.Is(err, sql.ErrNoRows) { + balance = wallet.Balance{} + return nil + } + return err }) return } @@ -70,7 +77,25 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) siacoins = append(siacoins, siacoin) } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siacoins)) + for i, se := range siacoins { + indices[i] = se.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siacoins[i].MerkleProof = proof + } + } + return nil }) return } @@ -97,7 +122,25 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) } siafunds = append(siafunds, siafund) } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siafund elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siafunds)) + for i, se := range siafunds { + indices[i] = se.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siafunds[i].MerkleProof = proof + } + } + return nil }) return } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index c529dd7..92761a6 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -14,6 +14,8 @@ import ( ) type updateTx struct { + indexMode wallet.IndexMode + tx *txn relevantAddresses map[types.Address]bool } @@ -24,6 +26,10 @@ type addressRef struct { } func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { + if ut.indexMode == wallet.IndexModeFull { + panic("SiacoinStateElements called in full index mode") + } + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` rows, err := ut.tx.Query(query) if err != nil { @@ -43,6 +49,10 @@ func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { } func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) error { + if ut.indexMode == wallet.IndexModeFull { + panic("UpdateSiacoinStateElements called in full index mode") + } + log := ut.tx.log.Named("UpdateSiacoinStateElements") log.Debug("updating siacoin state elements", zap.Int("count", len(elements))) @@ -65,6 +75,10 @@ func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) er } func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { + if ut.indexMode == wallet.IndexModeFull { + panic("SiafundStateElements called in full index mode") + } + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` rows, err := ut.tx.Query(query) if err != nil { @@ -84,6 +98,10 @@ func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { } func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) error { + if ut.indexMode == wallet.IndexModeFull { + panic("UpdateSiafundStateElements called in full index mode") + } + const query = `UPDATE siafund_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` stmt, err := ut.tx.Prepare(query) if err != nil { @@ -101,7 +119,31 @@ func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) er return nil } +func (ut *updateTx) UpdateStateTree(changes []wallet.TreeNodeUpdate) error { + if ut.indexMode != wallet.IndexModeFull { + panic("UpdateStateTree called in partial index mode") + } + + stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES ($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, change := range changes { + _, err := stmt.Exec(change.Row, change.Column, encode(change.Hash)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + } + return nil +} + func (ut *updateTx) AddressRelevant(addr types.Address) (bool, error) { + if ut.indexMode == wallet.IndexModeFull { + return true, nil + } + if relevant, ok := ut.relevantAddresses[addr]; ok { return relevant, nil } @@ -142,13 +184,13 @@ func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { return fmt.Errorf("failed to spend siacoin elements: %w", err) - } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, log.Named("addSiacoinElements")); err != nil { + } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, ut.indexMode, log.Named("addSiacoinElements")); err != nil { return fmt.Errorf("failed to add siacoin elements: %w", err) } if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { return fmt.Errorf("failed to spend siafund elements: %w", err) - } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, log.Named("addSiafundElements")); err != nil { + } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, ut.indexMode, log.Named("addSiafundElements")); err != nil { return fmt.Errorf("failed to add siafund elements: %w", err) } @@ -186,20 +228,22 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. log := s.log.Named("UpdateChainState").With(zap.Int("reverted", len(reverted)), zap.Int("applied", len(applied))) return s.transaction(func(tx *txn) error { utx := &updateTx{ + indexMode: s.indexMode, + tx: tx, relevantAddresses: make(map[types.Address]bool), } - if err := wallet.UpdateChainState(utx, reverted, applied, log); err != nil { - return fmt.Errorf("failed to update chain state: %w", err) - } else if err := setLastCommittedIndex(tx, applied[len(applied)-1].State.Index); err != nil { + state := applied[len(applied)-1].State + + if err := wallet.UpdateChainState(utx, reverted, applied, s.indexMode, log); err != nil { + return err + } else if err := setGlobalState(tx, state.Index, state.Elements.NumLeaves); err != nil { return fmt.Errorf("failed to set last committed index: %w", err) } - height := applied[len(applied)-1].State.Index.Height - - if height > spentElementRetentionBlocks { - pruneHeight := height - spentElementRetentionBlocks + if state.Index.Height > spentElementRetentionBlocks { + pruneHeight := state.Index.Height - spentElementRetentionBlocks siacoins, err := pruneSpentSiacoinElements(tx, pruneHeight) if err != nil { @@ -210,10 +254,7 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. if err != nil { return fmt.Errorf("failed to cleanup siafund elements: %w", err) } - - if len(siacoins) > 0 || len(siafunds) > 0 { - log.Debug("pruned elements", zap.Stringers("siacoins", siacoins), zap.Stringers("siafunds", siafunds), zap.Uint64("pruneHeight", pruneHeight)) - } + log.Debug("pruned elements", zap.Int64("siacoins", siacoins), zap.Int64("siafunds", siafunds), zap.Uint64("pruneHeight", pruneHeight)) } return nil }) @@ -231,6 +272,35 @@ func (s *Store) ResetLastIndex() error { return err } +// IndexMode returns the current index mode. +func (s *Store) IndexMode() (wallet.IndexMode, error) { + var mode wallet.IndexMode + err := s.db.QueryRow(`SELECT index_mode FROM global_settings`).Scan(&mode) + return mode, err +} + +// SetIndexMode sets the index mode. If the index mode is already set, this +// function will return an error. +func (s *Store) SetIndexMode(mode wallet.IndexMode) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`UPDATE global_settings SET index_mode=$1 WHERE index_mode IS NULL`, mode) + if err != nil { + return fmt.Errorf("failed to set index mode: %w", err) + } + + // check that the index mode was set + var existingMode wallet.IndexMode + err = tx.QueryRow(`SELECT index_mode FROM global_settings`).Scan(&existingMode) + if err != nil { + return fmt.Errorf("failed to query index mode: %w", err) + } else if existingMode != mode { + return fmt.Errorf("cannot change index mode from %v to %v", existingMode, mode) + } + s.indexMode = mode // this is a bit annoying + return nil + }) +} + func scanStateElement(s scanner) (se types.StateElement, err error) { err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) return @@ -401,16 +471,16 @@ func revertMatureSiacoinBalance(tx *txn, index types.ChainIndex) error { return nil } -func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, log *zap.Logger) error { +func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siacoin_elements WHERE id=$1)`) if err != nil { @@ -427,7 +497,7 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -440,6 +510,12 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, return fmt.Errorf("failed to check if siacoin element exists: %w", err) } + // in full index mode, Merkle proofs are stored in the state tree table + // rather than per element. + if indexMode == wallet.IndexModeFull { + se.MerkleProof = nil + } + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) @@ -489,11 +565,11 @@ func removeSiacoinElements(tx *txn, elements []types.SiacoinElement) error { return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id, matured`) if err != nil { @@ -503,7 +579,7 @@ func removeSiacoinElements(tx *txn, elements []types.SiacoinElement) error { balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -554,11 +630,11 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) if err != nil { @@ -568,7 +644,7 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -615,11 +691,11 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) if err != nil { @@ -629,7 +705,7 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiacoinOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -671,16 +747,16 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 return nil } -func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, log *zap.Logger) error { +func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { if len(elements) == 0 { return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siafund_elements WHERE id=$1)`) if err != nil { @@ -696,7 +772,7 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, balanceChanges := make(map[int64]uint64) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiafundOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -708,6 +784,12 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, return fmt.Errorf("failed to check if siafund element exists: %w", err) } + // in full index mode, Merkle proofs are stored in the state tree table + // rather than per element. + if indexMode == wallet.IndexModeFull { + se.MerkleProof = nil + } + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) @@ -747,11 +829,11 @@ func removeSiafundElements(tx *txn, elements []types.SiafundElement) error { return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) if err != nil { @@ -761,7 +843,7 @@ func removeSiafundElements(tx *txn, elements []types.SiafundElement) error { balanceChanges := make(map[int64]uint64) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiafundOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -808,11 +890,11 @@ func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int6 return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) if err != nil { @@ -822,7 +904,7 @@ func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int6 balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiafundOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -873,11 +955,11 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error return nil } - addrStmt, err := insertAddressStatement(tx) + addressRefStmt, done, err := addressRefStmt(tx) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } - defer addrStmt.Close() + defer done() stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) if err != nil { @@ -887,7 +969,7 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error balanceChanges := make(map[int64]wallet.Balance) for _, se := range elements { - addrRef, err := scanAddress(addrStmt.QueryRow(encode(se.SiafundOutput.Address), encode(types.ZeroCurrency), 0)) + addrRef, err := addressRefStmt(se.SiafundOutput.Address) if err != nil { return fmt.Errorf("failed to query address: %w", err) } else if _, ok := balanceChanges[addrRef.ID]; !ok { @@ -940,7 +1022,7 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { } defer insertEventStmt.Close() - addrStmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) + addrStmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare address statement: %w", err) } @@ -975,7 +1057,7 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { } var addressID int64 - err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency), 0).Scan(&addressID) + err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency)).Scan(&addressID) if err != nil { return fmt.Errorf("failed to get address: %w", err) } @@ -1206,48 +1288,40 @@ func revertOrphans(tx *txn, index types.ChainIndex, log *zap.Logger) error { return err } -func pruneSpentSiacoinElements(tx *txn, height uint64) (removed []types.SiacoinOutputID, err error) { - const query = `DELETE FROM siacoin_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1) RETURNING id` - rows, err := tx.Query(query, height) +func pruneSpentSiacoinElements(tx *txn, height uint64) (removed int64, err error) { + const query = `DELETE FROM siacoin_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` + res, err := tx.Exec(query, height) if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + return 0, fmt.Errorf("failed to query siacoin elements: %w", err) } - defer rows.Close() - - for rows.Next() { - var id types.SiacoinOutputID - if err := rows.Scan(decode(&id)); err != nil { - return nil, fmt.Errorf("failed to scan siacoin element: %w", err) - } - removed = append(removed, id) - } - return removed, rows.Err() + return res.RowsAffected() } -func pruneSpentSiafundElements(tx *txn, height uint64) (removed []types.SiafundOutputID, err error) { - const query = `DELETE FROM siafund_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1) RETURNING id` - rows, err := tx.Query(query, height) +func pruneSpentSiafundElements(tx *txn, height uint64) (removed int64, err error) { + const query = `DELETE FROM siafund_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` + res, err := tx.Exec(query, height) if err != nil { - return nil, fmt.Errorf("failed to query siafund elements: %w", err) - } - defer rows.Close() - - for rows.Next() { - var id types.SiafundOutputID - if err := rows.Scan(decode(&id)); err != nil { - return nil, fmt.Errorf("failed to scan siafund element: %w", err) - } - removed = append(removed, id) + return 0, fmt.Errorf("failed to query siacoin elements: %w", err) } - return removed, rows.Err() + return res.RowsAffected() } -func setLastCommittedIndex(tx *txn, index types.ChainIndex) error { - _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(index)) +func setGlobalState(tx *txn, index types.ChainIndex, numLeaves uint64) error { + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1, element_num_leaves=$2`, encode(index), numLeaves) return err } -func insertAddressStatement(tx *txn) (*stmt, error) { +func addressRefStmt(tx *txn) (func(types.Address) (addressRef, error), func() error, error) { + stmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, $4) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) + if err != nil { + return nil, nil, fmt.Errorf("failed to prepare address statement: %w", err) + } // the on conflict is effectively a no-op, but enables us to return the id of the existing address - return tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, $3) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) + return func(addr types.Address) (addressRef, error) { + ref, err := scanAddress(stmt.QueryRow(encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency), 0)) + if err != nil { + return addressRef{}, fmt.Errorf("failed to get address %q: %w", addr, err) + } + return ref, nil + }, stmt.Close, nil } diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index a29a00c..329e9a8 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -18,7 +18,7 @@ import ( var initDatabase string func initializeSettings(tx *txn, target int64) error { - _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_tip) VALUES (0, ?, ?)`, target, encode(types.ChainIndex{})) + _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_tip, element_num_leaves) VALUES (0, ?, ?, ?)`, target, encode(types.ChainIndex{}), 0) return err } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index c4c3537..133cad1 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -43,6 +43,13 @@ CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); +CREATE TABLE state_tree ( + row INTEGER, + column INTEGER, + value BLOB NOT NULL, + PRIMARY KEY (row, column) +); + CREATE TABLE events ( id INTEGER PRIMARY KEY, chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), @@ -97,5 +104,7 @@ CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row db_version INTEGER NOT NULL, -- used for migrations - last_indexed_tip BLOB NOT NULL -- the last chain index that was processed + index_mode INTEGER, -- the mode of the data store + last_indexed_tip BLOB NOT NULL, -- the last chain index that was processed + element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree ); diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 135ad26..eca3a77 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -12,7 +12,7 @@ import ( func TestAddPeer(t *testing.T) { log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestAddPeer(t *testing.T) { func TestBanPeer(t *testing.T) { log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 417ee19..1237413 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "go.sia.tech/walletd/wallet" "go.uber.org/zap" "lukechampine.com/frand" ) @@ -16,6 +17,8 @@ import ( type ( // A Store is a persistent store that uses a SQL database as its backend. Store struct { + indexMode wallet.IndexMode + db *sql.DB log *zap.Logger } @@ -75,11 +78,11 @@ func sqliteFilepath(fp string) string { // an error, the transaction is rolled back. Otherwise, the transaction is // committed. func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { - start := time.Now() dbtx, err := db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } + start := time.Now() defer func() { if err := dbtx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { log.Error("failed to rollback transaction", zap.Error(err)) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index b812e7a..8e01e99 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -5,100 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "math/bits" "time" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" ) -func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { - err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) - return -} - -func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) - return -} - -func insertAddress(tx *txn, addr types.Address) (id int64, err error) { - const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) -VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address -RETURNING id` - - err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency)).Scan(&id) - return -} - -func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { - var eventType string - var eventBuf []byte - - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &eventType, &eventBuf) - if err != nil { - return - } - - switch eventType { - case wallet.EventTypeTransaction: - var tx wallet.EventTransaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - ev.Data = &tx - case wallet.EventTypeContractPayout: - var m wallet.EventContractPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - ev.Data = &m - case wallet.EventTypeMinerPayout: - var m wallet.EventMinerPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal payout event: %w", err) - } - ev.Data = &m - case wallet.EventTypeFoundationSubsidy: - var m wallet.EventFoundationSubsidy - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) - } - ev.Data = &m - default: - return wallet.Event{}, 0, fmt.Errorf("unknown event type: %s", eventType) - } - return -} - -func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev - INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) - WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) - ORDER BY ev.maturity_height DESC, ev.id DESC - LIMIT $2 OFFSET $3` - - rows, err := tx.Query(query, id, limit, offset) - if err != nil { - return nil, nil, err - } - defer rows.Close() - - for rows.Next() { - event, eventID, err := scanEvent(rows) - if err != nil { - return nil, nil, fmt.Errorf("failed to scan event: %w", err) - } - - events = append(events, event) - eventIDs = append(eventIDs, eventID) - } - if err := rows.Err(); err != nil { - return nil, nil, err - } - return -} - func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.ID, eventIDs []int64) (map[int64][]types.Address, error) { query := `SELECT ea.event_id, sa.sia_address FROM event_addresses ea @@ -322,7 +235,26 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins siacoins = append(siacoins, siacoin) } - return rows.Err() + + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siacoins)) + for i, se := range siacoins { + indices[i] = se.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siacoins[i].MerkleProof = proof + } + } + return nil }) return } @@ -353,7 +285,25 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds } siafunds = append(siafunds, siafund) } - return rows.Err() + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siafunds)) + for i, se := range siafunds { + indices[i] = se.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siafunds[i].MerkleProof = proof + } + } + return nil }) return } @@ -434,10 +384,142 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` return } +func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + return +} + +func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { + err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + return +} + +func insertAddress(tx *txn, addr types.Address) (id int64, err error) { + const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) +VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address +RETURNING id` + + err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency)).Scan(&id) + return +} + +func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ error) { + if len(indices) == 0 { + return nil, nil + } + + var numLeaves uint64 + if err := tx.QueryRow(`SELECT element_num_leaves FROM global_settings LIMIT 1`).Scan(&numLeaves); err != nil { + return nil, fmt.Errorf("failed to query state tree leaves: %w", err) + } + + stmt, err := tx.Prepare(`SELECT value FROM state_tree WHERE row=? AND column=?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + data := make(map[uint64]map[uint64]types.Hash256) + for _, leafIndex := range indices { + proof := make([]types.Hash256, bits.Len64(leafIndex^numLeaves)-1) + for j := range proof { + row, col := uint64(j), (leafIndex>>j)^1 + + // check if the hash is already in the cache + if h, ok := data[row][col]; ok { + proof[j] = h + continue + } + + // query the hash from the database + if err := stmt.QueryRow(row, col).Scan(decode(&proof[j])); err != nil { + return nil, fmt.Errorf("failed to query state element (%d,%d): %w", row, col, err) + } + + // cache the hash + if _, ok := data[row]; !ok { + data[row] = make(map[uint64]types.Hash256) + } + data[row][col] = proof[j] + } + proofs = append(proofs, proof) + } + return +} + +func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { + var eventType string + var eventBuf []byte + + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &eventType, &eventBuf) + if err != nil { + return + } + + switch eventType { + case wallet.EventTypeTransaction: + var tx wallet.EventTransaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + ev.Data = &tx + case wallet.EventTypeContractPayout: + var m wallet.EventContractPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) + } + ev.Data = &m + case wallet.EventTypeMinerPayout: + var m wallet.EventMinerPayout + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal payout event: %w", err) + } + ev.Data = &m + case wallet.EventTypeFoundationSubsidy: + var m wallet.EventFoundationSubsidy + if err = json.Unmarshal(eventBuf, &m); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) + } + ev.Data = &m + default: + return wallet.Event{}, 0, fmt.Errorf("unknown event type: %s", eventType) + } + return +} + +func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + FROM events ev + INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) + WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) + ORDER BY ev.maturity_height DESC, ev.id DESC + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, id, limit, offset) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + for rows.Next() { + event, eventID, err := scanEvent(rows) + if err != nil { + return nil, nil, fmt.Errorf("failed to scan event: %w", err) + } + + events = append(events, event) + eventIDs = append(eventIDs, eventID) + } + if err := rows.Err(); err != nil { + return nil, nil, err + } + return +} + func walletExists(tx *txn, id wallet.ID) error { - const query = `SELECT id FROM wallets WHERE id=$1` - var dummyID int64 - err := tx.QueryRow(query, id).Scan(&dummyID) + const query = `SELECT 1 FROM wallets WHERE id=$1` + var dummy int + err := tx.QueryRow(query, id).Scan(&dummy) if errors.Is(err, sql.ErrNoRows) { return wallet.ErrNotFound } diff --git a/wallet/manager.go b/wallet/manager.go index c75aefd..94f342d 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -2,6 +2,7 @@ package wallet import ( "context" + "errors" "fmt" "sync" "time" @@ -12,7 +13,32 @@ import ( "go.uber.org/zap" ) +// IndexMode represents the index mode of the wallet manager. The index mode +// determines how the wallet manager stores the consensus state. +// +// IndexModePartial - The wallet manager scans the blockchain starting at +// genesis. Only state from addresses that are registered with a +// wallet will be stored. If an address is added to a wallet after the +// scan completes, the manager will need to rescan. +// +// IndexModeFull - The wallet manager scans the blockchain starting at genesis +// and stores the state of all addresses. +// +// IndexModeNone - The wallet manager does not scan the blockchain. This is +// useful for multiple nodes sharing the same database. None should only be used +// when connecting to a database that is in "Full" mode. +const ( + IndexModePartial IndexMode = iota + IndexModeFull + IndexModeNone +) + +const syncBatchSize = 250 + type ( + // An IndexMode determines the chain state that the wallet manager stores. + IndexMode uint8 + // A ChainManager manages the consensus state ChainManager interface { Tip() types.ChainIndex @@ -46,11 +72,14 @@ type ( AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + SetIndexMode(IndexMode) error LastCommittedIndex() (types.ChainIndex, error) } // A Manager manages wallets. Manager struct { + indexMode IndexMode + chain ChainManager store Store log *zap.Logger @@ -61,6 +90,25 @@ type ( } ) +// String returns the string representation of the index mode. +func (i IndexMode) String() string { + switch i { + case IndexModePartial: + return "partial" + case IndexModeFull: + return "full" + case IndexModeNone: + return "none" + default: + return "unknown" + } +} + +// MarshalText implements the encoding.TextMarshaler interface. +func (i IndexMode) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + // Tip returns the last scanned chain index of the manager. func (m *Manager) Tip() (types.ChainIndex, error) { return m.store.LastCommittedIndex() @@ -159,6 +207,10 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { // Scan rescans the chain starting from the given index. The scan will complete // when the chain manager reaches the current tip or the context is canceled. func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { + if m.indexMode != IndexModePartial { + return fmt.Errorf("scans are disabled in index mode %s", m.indexMode) + } + ctx, cancel, err := m.tg.AddWithContext(ctx) if err != nil { return err @@ -170,6 +222,11 @@ func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { return syncStore(ctx, m.store, m.chain, index) } +// IndexMode returns the index mode of the wallet manager. +func (m *Manager) IndexMode() IndexMode { + return m.indexMode +} + // Close closes the wallet manager. func (m *Manager) Close() error { m.tg.Stop() @@ -183,7 +240,7 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch return ctx.Err() default: } - crus, caus, err := cm.UpdatesSince(index, 1000) + crus, caus, err := cm.UpdatesSince(index, syncBatchSize) if err != nil { return fmt.Errorf("failed to subscribe to chain manager: %w", err) } else if err := store.UpdateChainState(crus, caus); err != nil { @@ -195,14 +252,29 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch } // NewManager creates a new wallet manager. -func NewManager(cm ChainManager, store Store, log *zap.Logger) *Manager { +func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { m := &Manager{ + indexMode: IndexModePartial, + chain: cm, store: store, - log: log, + log: zap.NewNop(), tg: threadgroup.New(), } + for _, opt := range opts { + opt(m) + } + + // if the index mode is none, skip setting the index mode in the store + // and return the manager + if m.indexMode == IndexModeNone { + return m, nil + } else if err := store.SetIndexMode(m.indexMode); err != nil { + return nil, err + } + + // start a goroutine to sync the store with the chain manager reorgChan := make(chan struct{}, 1) reorgChan <- struct{}{} unsubscribe := cm.OnReorg(func(index types.ChainIndex) { @@ -215,6 +287,7 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) *Manager { go func() { defer unsubscribe() + log := m.log.Named("sync") ctx, cancel, err := m.tg.AddWithContext(context.Background()) if err != nil { log.Panic("failed to add to threadgroup", zap.Error(err)) @@ -232,12 +305,12 @@ func NewManager(cm ChainManager, store Store, log *zap.Logger) *Manager { // update the store lastTip, err := store.LastCommittedIndex() if err != nil { - log.Error("failed to get last committed index", zap.Error(err)) - } else if err := syncStore(ctx, store, cm, lastTip); err != nil { - log.Error("failed to sync store", zap.Error(err)) + log.Panic("failed to get last committed index", zap.Error(err)) + } else if err := syncStore(ctx, store, cm, lastTip); err != nil && !errors.Is(err, context.Canceled) { + log.Panic("failed to sync store", zap.Error(err)) } m.mu.Unlock() } }() - return m + return m, nil } diff --git a/wallet/options.go b/wallet/options.go new file mode 100644 index 0000000..7033321 --- /dev/null +++ b/wallet/options.go @@ -0,0 +1,20 @@ +package wallet + +import "go.uber.org/zap" + +// An Option configures a wallet Manager. +type Option func(*Manager) + +// WithLogger sets the logger used by the manager. +func WithLogger(log *zap.Logger) Option { + return func(m *Manager) { + m.log = log + } +} + +// WithIndexMode sets the index mode used by the manager. +func WithIndexMode(mode IndexMode) Option { + return func(m *Manager) { + m.indexMode = mode + } +} diff --git a/wallet/update.go b/wallet/update.go index 8a42c6a..672dd0b 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -9,6 +9,13 @@ import ( ) type ( + // A stateTreeUpdater is an interface for applying and reverting + // Merkle tree updates. + stateTreeUpdater interface { + UpdateElementProof(e *types.StateElement) + ForEachTreeNode(fn func(row uint64, col uint64, h types.Hash256)) + } + // AddressBalance pairs an address with its balance. AddressBalance struct { Address types.Address `json:"address"` @@ -18,6 +25,7 @@ type ( // AppliedState contains all state changes made to a store after applying a chain // update. AppliedState struct { + NumLeaves uint64 Events []Event CreatedSiacoinElements []types.SiacoinElement SpentSiacoinElements []types.SiacoinElement @@ -28,12 +36,21 @@ type ( // RevertedState contains all state changes made to a store after reverting // a chain update. RevertedState struct { + NumLeaves uint64 UnspentSiacoinElements []types.SiacoinElement DeletedSiacoinElements []types.SiacoinElement UnspentSiafundElements []types.SiafundElement DeletedSiafundElements []types.SiafundElement } + // A TreeNodeUpdate contains the hash of a Merkle tree node and its row and + // column indices. + TreeNodeUpdate struct { + Hash types.Hash256 + Row int + Column int + } + // An UpdateTx atomically updates the state of a store. UpdateTx interface { SiacoinStateElements() ([]types.StateElement, error) @@ -42,6 +59,8 @@ type ( SiafundStateElements() ([]types.StateElement, error) UpdateSiafundStateElements([]types.StateElement) error + UpdateStateTree([]TreeNodeUpdate) error + AddressRelevant(types.Address) (bool, error) ApplyIndex(types.ChainIndex, AppliedState) error @@ -49,9 +68,49 @@ type ( } ) +// updateStateElements updates the state elements in a store according to the +// changes made by a chain update. +func updateStateElements(tx UpdateTx, update stateTreeUpdater, indexMode IndexMode) error { + if indexMode == IndexModeFull { + var updates []TreeNodeUpdate + update.ForEachTreeNode(func(row, col uint64, h types.Hash256) { + updates = append(updates, TreeNodeUpdate{h, int(row), int(col)}) + }) + return tx.UpdateStateTree(updates) + } else { + // fetch all siacoin and siafund state elements + siacoinStateElements, err := tx.SiacoinStateElements() + if err != nil { + return fmt.Errorf("failed to get siacoin state elements: %w", err) + } + + // update siacoin element proofs + for i := range siacoinStateElements { + update.UpdateElementProof(&siacoinStateElements[i]) + } + + if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) + } + + siafundStateElements, err := tx.SiafundStateElements() + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } + + // update siafund element proofs + for i := range siafundStateElements { + update.UpdateElementProof(&siafundStateElements[i]) + } + return tx.UpdateSiafundStateElements(siafundStateElements) + } +} + // applyChainUpdate atomically applies a chain update to a store -func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { - var applied AppliedState +func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) error { + applied := AppliedState{ + NumLeaves: cau.State.Elements.NumLeaves, + } // determine which siacoin and siafund elements are ephemeral // @@ -123,44 +182,19 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate) error { } applied.Events = AppliedEvents(cau.State, cau.Block, cau, relevant) - // fetch all siacoin and siafund state elements - siacoinStateElements, err := tx.SiacoinStateElements() - if err != nil { - return fmt.Errorf("failed to get siacoin state elements: %w", err) - } - - // update siacoin element proofs - for i := range siacoinStateElements { - cau.UpdateElementProof(&siacoinStateElements[i]) - } - - if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) - } - - siafundStateElements, err := tx.SiafundStateElements() - if err != nil { - return fmt.Errorf("failed to get siafund state elements: %w", err) - } - - // update siafund element proofs - for i := range siafundStateElements { - cau.UpdateElementProof(&siafundStateElements[i]) - } - - if err := tx.UpdateSiafundStateElements(siafundStateElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) - } - - if err := tx.ApplyIndex(cau.State.Index, applied); err != nil { - return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) + if err := updateStateElements(tx, cau, indexMode); err != nil { + return fmt.Errorf("failed to update state elements: %w", err) + } else if err := tx.ApplyIndex(cau.State.Index, applied); err != nil { + return fmt.Errorf("failed to apply index: %w", err) } return nil } // revertChainUpdate atomically reverts a chain update from a store -func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex) error { - var reverted RevertedState +func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex, indexMode IndexMode) error { + reverted := RevertedState{ + NumLeaves: cru.State.Elements.NumLeaves, + } // determine which siacoin and siafund elements are ephemeral // @@ -226,43 +260,20 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. }) if err := tx.RevertIndex(revertedIndex, reverted); err != nil { - return fmt.Errorf("failed to revert index %q: %w", revertedIndex, err) - } - - siacoinElements, err := tx.SiacoinStateElements() - if err != nil { - return fmt.Errorf("failed to get siacoin state elements: %w", err) - } - for i := range siacoinElements { - cru.UpdateElementProof(&siacoinElements[i]) - } - if err := tx.UpdateSiacoinStateElements(siacoinElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) + return fmt.Errorf("failed to revert index: %w", err) } - - // update siafund element proofs - siafundElements, err := tx.SiafundStateElements() - if err != nil { - return fmt.Errorf("failed to get siafund state elements: %w", err) - } - for i := range siafundElements { - cru.UpdateElementProof(&siafundElements[i]) - } - if err := tx.UpdateSiafundStateElements(siafundElements); err != nil { - return fmt.Errorf("failed to update siafund state elements: %w", err) - } - return nil + return updateStateElements(tx, cru, indexMode) } // UpdateChainState atomically updates the state of a store with a set of // updates from the chain manager. -func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate, log *zap.Logger) error { +func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate, indexMode IndexMode, log *zap.Logger) error { for _, cru := range reverted { revertedIndex := types.ChainIndex{ ID: cru.Block.ID(), Height: cru.State.Index.Height + 1, } - if err := revertChainUpdate(tx, cru, revertedIndex); err != nil { + if err := revertChainUpdate(tx, cru, revertedIndex, indexMode); err != nil { return fmt.Errorf("failed to revert chain update %q: %w", revertedIndex, err) } log.Debug("reverted chain update", zap.Stringer("blockID", revertedIndex.ID), zap.Uint64("height", revertedIndex.Height)) @@ -270,7 +281,7 @@ func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chai for _, cau := range applied { // apply the chain update - if err := applyChainUpdate(tx, cau); err != nil { + if err := applyChainUpdate(tx, cau, indexMode); err != nil { return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) } log.Debug("applied chain update", zap.Stringer("blockID", cau.State.Index.ID), zap.Uint64("height", cau.State.Index.Height)) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 3fbe6f4..76fbfb0 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -98,200 +98,220 @@ func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() + setupNode := func(t *testing.T, mode wallet.IndexMode) (consensus.State, *sqlite.Store, *chain.Manager, *wallet.Manager) { + t.Helper() - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { bdb.Close() }) - wm := wallet.NewManager(cm, db, log.Named("wallet")) - defer wm.Close() + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) - if err != nil { - t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { - t.Fatal(err) - } + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) - expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(mode)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wm.Close() }) + return genesisState, db, cm, wm } - waitForBlock(t, cm, db) - assertBalance := func(siacoin, immature types.Currency) error { - b, err := wm.WalletBalance(w.ID) + testReorg := func(t *testing.T, genesisState consensus.State, db *sqlite.Store, cm *chain.Manager, wm *wallet.Manager) { + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { - return fmt.Errorf("failed to check balance: %w", err) - } else if !b.Siacoins.Equals(siacoin) { - return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) - } else if !b.ImmatureSiacoins.Equals(immature) { - return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) } - return nil - } - if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { - t.Fatal(err) - } + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) - // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } - // check that the utxo was created - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } - // mine to trigger a reorg - var blocks []types.Block - state := genesisState - for i := 0; i < 10; i++ { - block := mineBlock(state, nil, types.VoidAddress) - blocks = append(blocks, block) - state.Index.ID = block.ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) + // check that a payout event was recorded + events, err := wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } - // check that the balance was reverted - if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { - t.Fatal(err) - } + // check that the utxo was created + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } - // check that the payout event was reverted - events, err = wm.Events(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 0 { - t.Fatalf("expected 0 events, got %v", len(events)) - } + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 10; i++ { + block := mineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + state.Index.ID = block.ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) - // check that the utxo was removed - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 0 { - t.Fatalf("expected 0 outputs, got %v", len(utxos)) - } + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } - // mine a new payout - expectedPayout = cm.TipState().BlockReward() - maturityHeight = cm.TipState().MaturityHeight() - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) + // check that the payout event was reverted + events, err = wm.Events(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } - // check that the payout was received - if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { - t.Fatal(err) - } + // check that the utxo was removed + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } - // check that a payout event was recorded - events, err = wm.Events(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 1 { - t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) - } + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight = cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) - // check that the utxo was created - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) - } + // check that the payout was received + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } - // mine until the payout matures - var prevState consensus.State - for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + // check that a payout event was recorded + events, err = wm.Events(w.ID, 0, 100) + if err != nil { t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) } - if i == maturityHeight-5 { - prevState = cm.TipState() + + // check that the utxo was created + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) } - } - waitForBlock(t, cm, db) - // check that the balance was updated - if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { - t.Fatal(err) - } + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + waitForBlock(t, cm, db) - // reorg the last few blocks to re-mature the payout - blocks = nil - state = prevState - for i := 0; i < 10; i++ { - blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) - state.Index.ID = blocks[len(blocks)-1].ID() - state.Index.Height++ - } - if err := cm.AddBlocks(blocks); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) + // check that the balance was updated + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } - // check that the balance is correct - if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { - t.Fatal(err) - } + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) - // check that only the single utxo still exists - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + // check that the balance is correct + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that only the single utxo still exists + utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } } + + t.Run("IndexModePartial", func(t *testing.T) { + state, db, cm, w := setupNode(t, wallet.IndexModePartial) + testReorg(t, state, db, cm, w) + }) + + t.Run("IndexModeFull", func(t *testing.T) { + state, db, cm, w := setupNode(t, wallet.IndexModeFull) + testReorg(t, state, db, cm, w) + }) } func TestEphemeralBalance(t *testing.T) { @@ -320,7 +340,10 @@ func TestEphemeralBalance(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -513,7 +536,10 @@ func TestWalletAddresses(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() // Add a wallet @@ -644,7 +670,10 @@ func TestScan(t *testing.T) { cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() pk2 := types.GeneratePrivateKey() @@ -795,7 +824,10 @@ func TestSiafunds(t *testing.T) { cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() pk2 := types.GeneratePrivateKey() @@ -931,7 +963,10 @@ func TestOrphans(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -1062,7 +1097,10 @@ func TestOrphans(t *testing.T) { t.Fatal(err) } - wm = wallet.NewManager(cm, db, log.Named("wallet")) + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() waitForBlock(t, cm, db) @@ -1091,6 +1129,217 @@ func TestOrphans(t *testing.T) { } } +func TestFullIndex(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(addr2) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + assertBalance := func(t *testing.T, address types.Address, siacoin, immature types.Currency, siafund uint64) { + t.Helper() + + b, err := wm.AddressBalance(address) + if err != nil { + t.Fatal(err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if b.Siafunds != siafund { + t.Fatalf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } + } + + // check the events are empty for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // assert that the airdropped siafunds are on the second address + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + // check the events for the air dropped siafunds + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } + + // mine a block and send the payout to the first address + expectedBalance1 := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check the payout was received + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Data.EventType()) + } + + assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // check that the events did not change + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Data.EventType()) + } + + assertBalance(t, addr, expectedBalance1, types.ZeroCurrency, 0) + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + + // send half siacoins to the second address + utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } + + // check the events for the second address + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } + + sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + + // send the siafunds to the first address + policy = types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk2.PublicKey())) + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sf[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + ClaimAddress: addr2, // claim address shouldn't create an event since the value is 0 + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sf[0].SiafundOutput.Value}, + }, + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk2.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } + + // check the events for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Data.EventType() != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } +} + func TestV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -1117,7 +1366,10 @@ func TestV2(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -1236,7 +1488,10 @@ func TestScanV2(t *testing.T) { cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() pk2 := types.GeneratePrivateKey() @@ -1414,7 +1669,10 @@ func TestReorgV2(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -1647,7 +1905,10 @@ func TestOrphansV2(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) @@ -1769,7 +2030,10 @@ func TestOrphansV2(t *testing.T) { t.Fatal(err) } - wm = wallet.NewManager(cm, db, log.Named("wallet")) + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() waitForBlock(t, cm, db) @@ -1856,7 +2120,10 @@ func TestDeleteWallet(t *testing.T) { } cm := chain.NewManager(store, genesisState) - wm := wallet.NewManager(cm, db, log.Named("wallet")) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } defer wm.Close() w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) From 7a945d4e422769a86973934d9ce6d69e6434f4e1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 6 May 2024 21:27:56 -0700 Subject: [PATCH 177/630] wallet: add updateStateElements sanity check --- wallet/update.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wallet/update.go b/wallet/update.go index 672dd0b..83852b0 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -71,6 +71,10 @@ type ( // updateStateElements updates the state elements in a store according to the // changes made by a chain update. func updateStateElements(tx UpdateTx, update stateTreeUpdater, indexMode IndexMode) error { + if indexMode == IndexModeNone { + panic("updateStateElements called with IndexModeNone") // developer error + } + if indexMode == IndexModeFull { var updates []TreeNodeUpdate update.ForEachTreeNode(func(row, col uint64, h types.Hash256) { From e535801be558671cf9d0c5ac2f181697a312bd2c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 8 May 2024 12:36:13 -0700 Subject: [PATCH 178/630] cmd: default to partial index mode --- cmd/walletd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index ac75bb4..c5faefa 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -84,7 +84,7 @@ func main() { rootCmd.BoolVar(&upnp, "upnp", true, "attempt to forward ports and discover IP with UPnP") rootCmd.BoolVar(&bootstrap, "bootstrap", true, "attempt to bootstrap the network") rootCmd.StringVar(&seed, "seed", "", "testnet seed") - rootCmd.StringVar(&indexModeStr, "index", "full", "address index mode (full, partial, off)") + rootCmd.StringVar(&indexModeStr, "index", "partial", "address index mode (full, partial, off)") versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) mineCmd := flagg.New("mine", mineUsage) From 2330a0d2d7d341598e037a1acbb5779bd076a35f Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Thu, 9 May 2024 13:32:18 +0000 Subject: [PATCH 179/630] ui: v0.20.0 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 386112e..bad7170 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.3 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.19.1 + go.sia.tech/web/walletd v0.20.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.20.0 lukechampine.com/flagg v1.1.1 @@ -22,7 +22,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect go.etcd.io/bbolt v1.3.9 // indirect go.sia.tech/mux v1.2.0 // indirect - go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b // indirect + go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/sys v0.20.0 // indirect diff --git a/go.sum b/go.sum index aca616f..337625e 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,10 @@ go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+ go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= -go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b h1:nwfLGAR0sjN/zb9QW0xWeNR8MjdtJl6KKZqPo2Amz3U= -go.sia.tech/web v0.0.0-20240403150056-0f3b9f006d5b/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.19.1 h1:RvNO/S9bMJTMDJGkzauCK96BZ75Iri2j3k9VfzJkx5Y= -go.sia.tech/web/walletd v0.19.1/go.mod h1:xlrUEt6cNA3vABwXpZaNDS3DM5Jh8IpqutdHqlVW+Os= +go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= +go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= +go.sia.tech/web/walletd v0.20.0 h1:XqauYxR8AKGaUROUIVwlzWP535NQjP8+2PdrFKaFv3Y= +go.sia.tech/web/walletd v0.20.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 965101d90a8c5cb7569515a0f375c05c26c6dbf4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 14 May 2024 07:53:58 -0700 Subject: [PATCH 180/630] cmd,config,wallet: add yaml support --- cmd/walletd/main.go | 366 +++++++++++++++++++++++++++++++++++--------- cmd/walletd/node.go | 24 +-- config/config.go | 62 ++++++++ go.mod | 1 + go.sum | 2 + wallet/manager.go | 33 +++- wallet/options.go | 9 ++ 7 files changed, 406 insertions(+), 91 deletions(-) create mode 100644 config/config.go diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index c5faefa..d499fc2 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -1,44 +1,34 @@ package main import ( + "context" "fmt" "log" "net" "os" "os/signal" + "path/filepath" + "runtime" + "runtime/pprof" + "syscall" + "time" "go.sia.tech/core/types" cwallet "go.sia.tech/coreutils/wallet" "go.sia.tech/walletd/api" "go.sia.tech/walletd/build" + "go.sia.tech/walletd/config" "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/term" + "gopkg.in/yaml.v3" "lukechampine.com/flagg" -) - -func check(context string, err error) { - if err != nil { - log.Fatalf("%v: %v", context, err) - } -} -func getAPIPassword() string { - apiPassword := os.Getenv("WALLETD_API_PASSWORD") - if apiPassword != "" { - fmt.Println("env: Using WALLETD_API_PASSWORD environment variable") - } else { - fmt.Print("Enter API password: ") - pw, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - check("Could not read API password:", err) - apiPassword = string(pw) - } - return apiPassword -} + _ "net/http/pprof" +) -var ( +const ( rootUsage = `Usage: walletd [flags] [action] @@ -66,27 +56,180 @@ Runs a CPU miner. Not intended for production use. ` ) +var cfg = config.Config{ + Name: "walletd", + Directory: ".", + AutoOpenWebUI: true, + HTTP: config.HTTP{ + Address: "localhost:9980", + Password: os.Getenv("WALLETD_API_PASSWORD"), + }, + Consensus: config.Consensus{ + Network: "mainnet", + GatewayAddress: ":9981", + Bootstrap: true, + }, + Index: config.Index{ + Mode: wallet.IndexModePartial, + BatchSize: 64, + }, + Log: config.Log{ + Level: "info", + File: config.LogFile{ + Enabled: true, + Format: "json", + Path: os.Getenv("WALLETD_LOG_FILE"), + }, + StdOut: config.StdOut{ + Enabled: true, + Format: "human", + EnableANSI: runtime.GOOS != "windows", + }, + }, +} + +func check(context string, err error) { + if err != nil { + log.Fatalf("%v: %v", context, err) + } +} + +func getAPIPassword() string { + apiPassword := cfg.HTTP.Password + if apiPassword == "" { + fmt.Print("Enter API password: ") + pw, err := term.ReadPassword(int(os.Stdin.Fd())) + fmt.Println() + check("Could not read API password:", err) + apiPassword = string(pw) + } + return apiPassword +} + +// stdoutFatalError prints an error message to stdout and exits with a 1 exit code. +func stdoutFatalError(msg string) { + stdoutError(msg) + os.Exit(1) +} + +// wrapANSI wraps the output in ANSI escape codes if enabled. +func wrapANSI(prefix, output, suffix string) string { + if cfg.Log.StdOut.EnableANSI { + return prefix + output + suffix + } + return output +} + +// stdoutError prints an error message to stdout +func stdoutError(msg string) { + if cfg.Log.StdOut.EnableANSI { + fmt.Println(wrapANSI("\033[31m", msg, "\033[0m")) + } else { + fmt.Println(msg) + } +} + +// tryLoadConfig loads the config file specified by the WALLETD_CONFIG_PATH. If +// the config file does not exist, it will not be loaded. +func tryLoadConfig() { + configPath := "walletd.yml" + if str := os.Getenv("WALLETD_CONFIG_FILE"); str != "" { + configPath = str + } + fmt.Println("loading config from", configPath) + + // If the config file doesn't exist, don't try to load it. + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return + } + + f, err := os.Open(configPath) + if err != nil { + stdoutFatalError("failed to open config file: " + err.Error()) + return + } + defer f.Close() + + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + + if err := dec.Decode(&cfg); err != nil { + fmt.Println("failed to decode config file:", err) + os.Exit(1) + } + fmt.Println("config loaded") +} + +// jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for +// parsing. +func jsonEncoder() zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.TimeKey = "timestamp" + return zapcore.NewJSONEncoder(cfg) +} + +// humanEncoder returns a zapcore.Encoder that encodes logs as human-readable +// text. +func humanEncoder(showColors bool) zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.EncodeDuration = zapcore.StringDurationEncoder + + if showColors { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + } else { + cfg.EncodeLevel = zapcore.CapitalLevelEncoder + } + + cfg.StacktraceKey = "" + cfg.CallerKey = "" + return zapcore.NewConsoleEncoder(cfg) +} + +func parseLogLevel(level string) zap.AtomicLevel { + switch level { + case "debug": + return zap.NewAtomicLevelAt(zap.DebugLevel) + case "info": + return zap.NewAtomicLevelAt(zap.InfoLevel) + case "warn": + return zap.NewAtomicLevelAt(zap.WarnLevel) + case "error": + return zap.NewAtomicLevelAt(zap.ErrorLevel) + default: + fmt.Printf("invalid log level %q", level) + os.Exit(1) + } + panic("unreachable") +} + func main() { - log.SetFlags(0) + // attempt to load the config file first, command line flags will override + // any values set in the config file + tryLoadConfig() - var gatewayAddr, apiAddr, dir, network, seed, indexModeStr string - var upnp, bootstrap bool + indexModeStr := cfg.Index.Mode.String() var minerAddrStr string var minerBlocks int rootCmd := flagg.Root rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) - rootCmd.StringVar(&gatewayAddr, "addr", ":9981", "p2p address to listen on") - rootCmd.StringVar(&apiAddr, "http", "localhost:9980", "address to serve API on") - rootCmd.StringVar(&dir, "dir", ".", "directory to store node state in") - rootCmd.StringVar(&network, "network", "mainnet", "network to connect to") - rootCmd.BoolVar(&upnp, "upnp", true, "attempt to forward ports and discover IP with UPnP") - rootCmd.BoolVar(&bootstrap, "bootstrap", true, "attempt to bootstrap the network") - rootCmd.StringVar(&seed, "seed", "", "testnet seed") - rootCmd.StringVar(&indexModeStr, "index", "partial", "address index mode (full, partial, off)") + rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") + rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") + + rootCmd.StringVar(&cfg.Consensus.GatewayAddress, "addr", cfg.Consensus.GatewayAddress, "p2p address to listen on") + rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") + rootCmd.BoolVar(&cfg.Consensus.EnableUPNP, "upnp", cfg.Consensus.EnableUPNP, "attempt to forward ports and discover IP with UPnP") + rootCmd.BoolVar(&cfg.Consensus.Bootstrap, "bootstrap", cfg.Consensus.Bootstrap, "attempt to bootstrap the network") + + rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (full, partial, none)") + rootCmd.IntVar(&cfg.Index.BatchSize, "batch-size", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") + versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) + mineCmd := flagg.New("mine", mineUsage) mineCmd.IntVar(&minerBlocks, "n", -1, "mine this many blocks. If negative, mine indefinitely") mineCmd.StringVar(&minerAddrStr, "addr", "", "address to send block rewards to (required)") @@ -100,7 +243,6 @@ func main() { }, }) - log.Println("walletd", build.Version()) switch cmd { case rootCmd: if len(cmd.Args()) != 0 { @@ -108,67 +250,143 @@ func main() { return } - if err := os.MkdirAll(dir, 0700); err != nil { - log.Fatal(err) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) + defer cancel() + + go func() { + t := time.NewTicker(time.Minute) + defer t.Stop() + + dir := filepath.Join(cfg.Directory, "profiles") + if err := os.MkdirAll(dir, 0755); err != nil { + stdoutFatalError("failed to create profiles directory: " + err.Error()) + } + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + err := func() error { + f, err := os.Create(filepath.Join(dir, "heap-"+time.Now().Format("2006-01-02T150405")+".pprof")) + if err != nil { + return fmt.Errorf("failed to create heap profile: %w", err) + } + defer f.Close() + + if err := pprof.WriteHeapProfile(f); err != nil { + return fmt.Errorf("failed to write heap profile: %w", err) + } else if err := f.Sync(); err != nil { + return fmt.Errorf("failed to sync heap profile: %w", err) + } else if err := f.Close(); err != nil { + return fmt.Errorf("failed to close heap profile: %w", err) + } + return nil + }() + if err != nil { + stdoutError(err.Error()) + } + } + } + }() + + if err := os.MkdirAll(cfg.Directory, 0700); err != nil { + stdoutFatalError("failed to create directory: " + err.Error()) } apiPassword := getAPIPassword() - l, err := net.Listen("tcp", apiAddr) + l, err := net.Listen("tcp", cfg.HTTP.Address) if err != nil { - log.Fatal(err) + stdoutFatalError("failed to start HTTP server: " + err.Error()) } - // configure console logging note: this is configured before anything else - // to have consistent logging. File logging will be added after the cli - // flags and config is parsed - consoleCfg := zap.NewProductionEncoderConfig() - consoleCfg.TimeKey = "" // prevent duplicate timestamps - consoleCfg.EncodeTime = zapcore.RFC3339TimeEncoder - consoleCfg.EncodeDuration = zapcore.StringDurationEncoder - consoleCfg.EncodeLevel = zapcore.CapitalColorLevelEncoder - consoleCfg.StacktraceKey = "" - consoleCfg.CallerKey = "" - consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg) - - // only log info messages to console unless stdout logging is enabled - consoleCore := zapcore.NewCore(consoleEncoder, zapcore.Lock(os.Stdout), zap.NewAtomicLevelAt(zap.DebugLevel)) - logger := zap.New(consoleCore, zap.AddCaller()) - defer logger.Sync() + var logCores []zapcore.Core + if cfg.Log.StdOut.Enabled { + // if no log level is set for stdout, use the global log level + if cfg.Log.StdOut.Level == "" { + cfg.Log.StdOut.Level = cfg.Log.Level + } + + var encoder zapcore.Encoder + switch cfg.Log.StdOut.Format { + case "json": + encoder = jsonEncoder() + default: // stdout defaults to human + encoder = humanEncoder(cfg.Log.StdOut.EnableANSI) + } + + // create the stdout logger + level := parseLogLevel(cfg.Log.StdOut.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), level)) + } + + if cfg.Log.File.Enabled { + // if no log level is set for file, use the global log level + if cfg.Log.File.Level == "" { + cfg.Log.File.Level = cfg.Log.Level + } + + // normalize log path + if cfg.Log.File.Path == "" { + cfg.Log.File.Path = filepath.Join(cfg.Directory, "walletd.log") + } + + // configure file logging + var encoder zapcore.Encoder + switch cfg.Log.File.Format { + case "human": + encoder = humanEncoder(false) // disable colors in file log + default: // log file defaults to JSON + encoder = jsonEncoder() + } + + fileWriter, closeFn, err := zap.Open(cfg.Log.File.Path) + if err != nil { + stdoutFatalError("failed to open log file: " + err.Error()) + return + } + defer closeFn() + + // create the file logger + level := parseLogLevel(cfg.Log.File.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(fileWriter), level)) + } + + var log *zap.Logger + if len(logCores) == 1 { + log = zap.New(logCores[0], zap.AddCaller()) + } else { + log = zap.New(zapcore.NewTee(logCores...), zap.AddCaller()) + } + defer log.Sync() + // redirect stdlib log to zap - zap.RedirectStdLog(logger.Named("stdlib")) - - var indexMode wallet.IndexMode - switch indexModeStr { - case "full": - indexMode = wallet.IndexModeFull - case "partial": - indexMode = wallet.IndexModePartial - case "off": - indexMode = wallet.IndexModeNone + zap.RedirectStdLog(log.Named("stdlib")) + + if err := cfg.Index.Mode.UnmarshalText([]byte(indexModeStr)); err != nil { + log.Fatal("failed to parse index mode", zap.Error(err)) } - n, err := newNode(gatewayAddr, dir, network, upnp, bootstrap, indexMode, logger) + n, err := newNode(cfg, log) if err != nil { - log.Fatal(err) + log.Fatal("failed to create node", zap.Error(err)) } defer n.Close() - log.Println("p2p: Listening on", n.s.Addr()) stop := n.Start() - log.Println("api: Listening on", l.Addr()) go startWeb(l, n, apiPassword) - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt) - <-signalCh - log.Println("Shutting down...") + log.Info("walletd started", zap.String("version", build.Version()), zap.String("network", cfg.Consensus.Network), zap.String("commit", build.Commit()), zap.Time("buildDate", build.Time())) + <-ctx.Done() + log.Info("shutting down") stop() case versionCmd: if len(cmd.Args()) != 0 { cmd.Usage() return } - log.Println("Commit Hash:", build.Commit()) - log.Println("Commit Date:", build.Time()) + fmt.Println("walletd", build.Version()) + fmt.Println("Commit:", build.Commit()) + fmt.Println("Build Date:", build.Time()) case seedCmd: if len(cmd.Args()) != 0 { cmd.Usage() @@ -194,7 +412,7 @@ func main() { log.Fatal(err) } - c := api.NewClient("http://"+apiAddr+"/api", getAPIPassword()) + c := api.NewClient("http://"+cfg.HTTP.Address+"/api", getAPIPassword()) runCPUMiner(c, minerAddr, minerBlocks) } } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index cc55770..f12be24 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -15,6 +15,7 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" + "go.sia.tech/walletd/config" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" "go.uber.org/zap" @@ -100,11 +101,11 @@ func (n *node) Close() error { return n.store.Close() } -func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, indexMode wallet.IndexMode, log *zap.Logger) (*node, error) { +func newNode(cfg config.Config, log *zap.Logger) (*node, error) { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string - switch chainNetwork { + switch cfg.Consensus.Network { case "mainnet": network, genesisBlock = chain.Mainnet() bootstrapPeers = mainnetBootstrap @@ -118,7 +119,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) if err != nil { return nil, fmt.Errorf("failed to open consensus database: %w", err) } @@ -128,18 +129,18 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, } cm := chain.NewManager(dbstore, tipState) - l, err := net.Listen("tcp", addr) + l, err := net.Listen("tcp", cfg.Consensus.GatewayAddress) if err != nil { return nil, err } syncerAddr := l.Addr().String() - if useUPNP { + if cfg.Consensus.EnableUPNP { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if d, err := upnp.Discover(ctx); err != nil { log.Debug("couldn't discover UPnP router", zap.Error(err)) } else { - _, portStr, _ := net.SplitHostPort(addr) + _, portStr, _ := net.SplitHostPort(cfg.Consensus.GatewayAddress) port, _ := strconv.Atoi(portStr) if !d.IsForwarded(uint16(port), "TCP") { if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { @@ -162,17 +163,22 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, syncerAddr = net.JoinHostPort("127.0.0.1", port) } - store, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { return nil, fmt.Errorf("failed to open wallet database: %w", err) } - if useBootstrap { + if cfg.Consensus.Bootstrap { for _, peer := range bootstrapPeers { if err := store.AddPeer(peer); err != nil { return nil, fmt.Errorf("failed to add bootstrap peer '%s': %w", peer, err) } } + for _, peer := range cfg.Consensus.Peers { + if err := store.AddPeer(peer); err != nil { + return nil, fmt.Errorf("failed to add peer '%s': %w", peer, err) + } + } } ps, err := sqlite.NewPeerStore(store) @@ -187,7 +193,7 @@ func newNode(addr, dir string, chainNetwork string, useUPNP, useBootstrap bool, } s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) - wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(indexMode)) + wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(cfg.Index.Mode), wallet.WithSyncBatchSize(cfg.Index.BatchSize)) if err != nil { return nil, fmt.Errorf("failed to create wallet manager: %w", err) } diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..b8a26ad --- /dev/null +++ b/config/config.go @@ -0,0 +1,62 @@ +package config + +import "go.sia.tech/walletd/wallet" + +type ( + // HTTP contains the configuration for the HTTP server. + HTTP struct { + Address string `yaml:"address,omitempty"` + Password string `yaml:"password,omitempty"` + } + + // Consensus contains the configuration for the consensus set. + Consensus struct { + Network string `yaml:"network,omitempty"` + GatewayAddress string `yaml:"gatewayAddress,omitempty"` + Bootstrap bool `yaml:"bootstrap,omitempty"` + Peers []string `yaml:"peers,omitempty"` + EnableUPNP bool `yaml:"enableUPNP,omitempty"` + } + + // Index contains the configuration for the blockchain indexer + Index struct { + Mode wallet.IndexMode `yaml:"mode,omitempty"` + BatchSize int `yaml:"batchSize,omitempty"` + } + + // LogFile configures the file output of the logger. + LogFile struct { + Enabled bool `yaml:"enabled,omitempty"` + Level string `yaml:"level,omitempty"` // override the file log level + Format string `yaml:"format,omitempty"` + // Path is the path of the log file. + Path string `yaml:"path,omitempty"` + } + + // StdOut configures the standard output of the logger. + StdOut struct { + Level string `yaml:"level,omitempty"` // override the stdout log level + Enabled bool `yaml:"enabled,omitempty"` + Format string `yaml:"format,omitempty"` + EnableANSI bool `yaml:"enableANSI,omitempty"` //nolint:tagliatelle + } + + // Log contains the configuration for the logger. + Log struct { + Level string `yaml:"level,omitempty"` // global log level + StdOut StdOut `yaml:"stdout,omitempty"` + File LogFile `yaml:"file,omitempty"` + } + + // Config contains the configuration for the host. + Config struct { + Name string `yaml:"name,omitempty"` + Directory string `yaml:"directory,omitempty"` + AutoOpenWebUI bool `yaml:"autoOpenWebUI,omitempty"` + + HTTP HTTP `yaml:"http,omitempty"` + Consensus Consensus `yaml:"consensus,omitempty"` + Log Log `yaml:"log,omitempty"` + Index Index `yaml:"index,omitempty"` + } +) diff --git a/go.mod b/go.mod index bad7170..620b0c4 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( go.sia.tech/web/walletd v0.20.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.20.0 + gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 lukechampine.com/upnp v0.3.0 diff --git a/go.sum b/go.sum index 337625e..ff14054 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= diff --git a/wallet/manager.go b/wallet/manager.go index 94f342d..0568de9 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -33,7 +33,7 @@ const ( IndexModeNone ) -const syncBatchSize = 250 +const defaultSyncBatchSize = 64 type ( // An IndexMode determines the chain state that the wallet manager stores. @@ -78,14 +78,15 @@ type ( // A Manager manages wallets. Manager struct { - indexMode IndexMode + indexMode IndexMode + syncBatchSize int chain ChainManager store Store log *zap.Logger tg *threadgroup.ThreadGroup - mu sync.Mutex + mu sync.Mutex // protects the fields below used map[types.Hash256]bool } ) @@ -104,6 +105,21 @@ func (i IndexMode) String() string { } } +// UnmarshalText implements the encoding.TextUnmarshaler interface. +func (i *IndexMode) UnmarshalText(buf []byte) error { + switch string(buf) { + case "partial": + *i = IndexModePartial + case "full": + *i = IndexModeFull + case "none": + *i = IndexModeNone + default: + return fmt.Errorf("unknown index mode %q", buf) + } + return nil +} + // MarshalText implements the encoding.TextMarshaler interface. func (i IndexMode) MarshalText() ([]byte, error) { return []byte(i.String()), nil @@ -219,7 +235,7 @@ func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { m.mu.Lock() defer m.mu.Unlock() - return syncStore(ctx, m.store, m.chain, index) + return syncStore(ctx, m.store, m.chain, index, m.syncBatchSize) } // IndexMode returns the index mode of the wallet manager. @@ -233,14 +249,14 @@ func (m *Manager) Close() error { return nil } -func syncStore(ctx context.Context, store Store, cm ChainManager, index types.ChainIndex) error { +func syncStore(ctx context.Context, store Store, cm ChainManager, index types.ChainIndex, batchSize int) error { for index != cm.Tip() { select { case <-ctx.Done(): return ctx.Err() default: } - crus, caus, err := cm.UpdatesSince(index, syncBatchSize) + crus, caus, err := cm.UpdatesSince(index, batchSize) if err != nil { return fmt.Errorf("failed to subscribe to chain manager: %w", err) } else if err := store.UpdateChainState(crus, caus); err != nil { @@ -254,7 +270,8 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch // NewManager creates a new wallet manager. func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { m := &Manager{ - indexMode: IndexModePartial, + indexMode: IndexModePartial, + syncBatchSize: defaultSyncBatchSize, chain: cm, store: store, @@ -306,7 +323,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) lastTip, err := store.LastCommittedIndex() if err != nil { log.Panic("failed to get last committed index", zap.Error(err)) - } else if err := syncStore(ctx, store, cm, lastTip); err != nil && !errors.Is(err, context.Canceled) { + } else if err := syncStore(ctx, store, cm, lastTip, m.syncBatchSize); err != nil && !errors.Is(err, context.Canceled) { log.Panic("failed to sync store", zap.Error(err)) } m.mu.Unlock() diff --git a/wallet/options.go b/wallet/options.go index 7033321..79075e1 100644 --- a/wallet/options.go +++ b/wallet/options.go @@ -18,3 +18,12 @@ func WithIndexMode(mode IndexMode) Option { m.indexMode = mode } } + +// WithSyncBatchSize sets the number of blocks to batch when scanning +// the blockchain. The default is 64. Increasing this value can +// improve performance at the cost of memory usage. +func WithSyncBatchSize(size int) Option { + return func(m *Manager) { + m.syncBatchSize = size + } +} From 8c383c3693ad28d8d35f384b1a1906c0108c5c33 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 14 May 2024 08:09:20 -0700 Subject: [PATCH 181/630] cmd: consistent flags --- cmd/walletd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index d499fc2..7f9ff7b 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -225,7 +225,7 @@ func main() { rootCmd.BoolVar(&cfg.Consensus.Bootstrap, "bootstrap", cfg.Consensus.Bootstrap, "attempt to bootstrap the network") rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (full, partial, none)") - rootCmd.IntVar(&cfg.Index.BatchSize, "batch-size", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") + rootCmd.IntVar(&cfg.Index.BatchSize, "index.batch", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) From 6b699ac3b956de4c468569090d0db61f92977145 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 14 May 2024 08:09:25 -0700 Subject: [PATCH 182/630] config: fix lint --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index b8a26ad..b8d3f14 100644 --- a/config/config.go +++ b/config/config.go @@ -15,7 +15,7 @@ type ( GatewayAddress string `yaml:"gatewayAddress,omitempty"` Bootstrap bool `yaml:"bootstrap,omitempty"` Peers []string `yaml:"peers,omitempty"` - EnableUPNP bool `yaml:"enableUPNP,omitempty"` + EnableUPNP bool `yaml:"enableUPnP,omitempty"` } // Index contains the configuration for the blockchain indexer From ae7922ee0a7e29c2d5f74fc141963940ffdd26fd Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 14 May 2024 08:09:33 -0700 Subject: [PATCH 183/630] docs: update README --- README.md | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/README.md b/README.md index 91e1bc5..52dd8ef 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![GoDoc](https://godoc.org/go.sia.tech/walletd?status.svg)](https://godoc.org/go.sia.tech/walletd) +## Overview + `walletd` is the flagship Sia wallet, suitable for miners, exchanges, and everyday hodlers. Its client-server architecture gives you the flexibility to access your funds from anywhere, on any device, without compromising the @@ -11,3 +13,141 @@ Ledger hardware wallet, or another preferred method. Like other Foundation node software, `walletd` ships with a slick embedded UI, but developers can easily build headless integrations leveraging its powerful JSON API. Whether you're using a single address or millions, `walletd` scales to your needs. + +Setup guides are available at https://docs.sia.tech + +### Index Mode +`walletd` supports three different index modes for different use cases. + +**Partial** + +In partial index mode, `walletd` will only index addresses that are registered in the +wallet. This mode is recommended for most users, as it provides a good balance between +comprehensiveness and resource usage. This is the default mode for `walletd`. + +When adding existing addresses with history on chain, users will need to manually +initiate a rescan to index the new transactions. This can take some to complete, +depending on the number of blocks that need to be scanned. + +**Full** + +In full index mode, `walletd` will index the entire blockchain including all addresses +and UTXOs. This is the most comprehensive mode, but it also requires the most +resources. This mode is recommended for exchanges or wallet builders that need +to support a large or unknown number of addresses. + +**None** + +In "none" index mode, `walletd` will treat the database as read-only and not +index any new data. This mode is only useful in situations where another process +is managing the database and `walletd` is only being used to read data. + +## Configuration + +`walletd` can be configured in multiple ways. Some settings, like the API password, +can be configured via environment variable. Others, like the API port, and data +directory, can be set via command line flags. To simplify more complex configurations, +`walletd` can also be configured via a YAML file. + +The priority of configuration settings is as follows: +1. Command line flags +2. YAML file +3. Environment variables + +### Default Ports ++ `9980` UI and API ++ `9981` Sia consensus + +### Environment Variables ++ `WALLETD_API_PASSWORD` - The password required to access the API + +### Command Line Flags +``` +-addr string + p2p address to listen on (default ":9981") +-bootstrap + attempt to bootstrap the network (default true) +-dir string + directory to store node state in (default "/Users/n8maninger/Downloads/walletd-tmp") +-http string + address to serve API on (default "localhost:9980") +-index.batch int + max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 64) +-index.mode string + address index mode (full, partial, none) (default "full") +-network string + network to connect to (default "mainnet") +-upnp + attempt to forward ports and discover IP with UPnP +``` + +### YAML +All configuration settings can be set in a YAML file. The file should be named +`walletd.yaml` in the working directory. All fields are optional. +```yaml +directory: /etc/walletd +autoOpenWebUI: true +http: + address: :9980 + password: sia is cool +consensus: + network: mainnet + gatewayAddress: :9981 + bootstrap: false + enableUPnP: false +index: + mode: partial # full, partial, none (full index will index the entire blockchain, partial will only index addresses that are registered in the wallet, none will treat the database as read-only and not index any new data) + batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) +log: + level: info # global log level + stdout: + enabled: true # enable logging to stdout + level: debug # override the global log level for stdout + enableANSI: false + format: human # human or JSON + file: + enabled: true # enable logging to a file + level: debug # override the global log level for the file + path: /var/log/walletd.log + format: json # human or JSON +``` + +## Building +`walletd` uses SQLite for its persistence. A gcc toolchain is required to build `walletd` + +```sh +go generate ./... +CGO_ENABLED=1 go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w' ./cmd/walletd +``` + +## Docker Image +`walletd` includes a Dockerfile for building a Docker image. For building and +running `walletd` within a Docker container. The image can also be pulled from `ghcr.io/siafoundation/walletd`. + +```sh +docker run -d \ + --name walletd \ + -p 127.0.0.1:9980:9980 \ + -p 9981:9981 \ + -v /data:/data \ + ghcr.io/siafoundation/walletd:latest +``` + +### Docker Compose +```yml +services: + walletd: + image: ghcr.io/siafoundation/walletd:latest + ports: + - 127.0.0.1:9980:9980/tcp + - 9981:9981/tcp + volumes: + - /data:/data + restart: unless-stopped +``` + +### Building + +```sh +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/siafoundation/walletd:master . +``` \ No newline at end of file From f5f3231be0ed61ed43cfc555d8e0b415eff7355c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 06:51:30 -0700 Subject: [PATCH 184/630] cmd: remove pprof --- cmd/walletd/main.go | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 7f9ff7b..ede02d6 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -9,9 +9,7 @@ import ( "os/signal" "path/filepath" "runtime" - "runtime/pprof" "syscall" - "time" "go.sia.tech/core/types" cwallet "go.sia.tech/coreutils/wallet" @@ -24,8 +22,6 @@ import ( "golang.org/x/term" "gopkg.in/yaml.v3" "lukechampine.com/flagg" - - _ "net/http/pprof" ) const ( @@ -253,43 +249,6 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) defer cancel() - go func() { - t := time.NewTicker(time.Minute) - defer t.Stop() - - dir := filepath.Join(cfg.Directory, "profiles") - if err := os.MkdirAll(dir, 0755); err != nil { - stdoutFatalError("failed to create profiles directory: " + err.Error()) - } - - for { - select { - case <-ctx.Done(): - return - case <-t.C: - err := func() error { - f, err := os.Create(filepath.Join(dir, "heap-"+time.Now().Format("2006-01-02T150405")+".pprof")) - if err != nil { - return fmt.Errorf("failed to create heap profile: %w", err) - } - defer f.Close() - - if err := pprof.WriteHeapProfile(f); err != nil { - return fmt.Errorf("failed to write heap profile: %w", err) - } else if err := f.Sync(); err != nil { - return fmt.Errorf("failed to sync heap profile: %w", err) - } else if err := f.Close(); err != nil { - return fmt.Errorf("failed to close heap profile: %w", err) - } - return nil - }() - if err != nil { - stdoutError(err.Error()) - } - } - } - }() - if err := os.MkdirAll(cfg.Directory, 0700); err != nil { stdoutFatalError("failed to create directory: " + err.Error()) } From 31cceeced8d8c801518909139e2fb42e33da1c03 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 07:40:43 -0700 Subject: [PATCH 185/630] docker: support YAML file --- Dockerfile | 1 + cmd/walletd/main.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e803063..315ff3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,4 +38,5 @@ EXPOSE 9981/tcp USER ${PUID}:${PGID} +ENV WALLETD_CONFIG_FILE=/data/walletd.yml ENTRYPOINT [ "walletd", "--dir", "/data", "--http", ":9980" ] \ No newline at end of file diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index ede02d6..d9f3bc2 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -125,7 +125,7 @@ func stdoutError(msg string) { } } -// tryLoadConfig loads the config file specified by the WALLETD_CONFIG_PATH. If +// tryLoadConfig loads the config file specified by the WALLETD_CONFIG_FILE. If // the config file does not exist, it will not be loaded. func tryLoadConfig() { configPath := "walletd.yml" From 7f2d46c493601910a66dee9117d21bfc6bb1a445 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 07:49:44 -0700 Subject: [PATCH 186/630] ci: update publish to use upload-artifact@v4 --- .github/workflows/publish.yml | 166 +++++++++++++++++++++++----------- 1 file changed, 114 insertions(+), 52 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 41e5f50..5ac8d24 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,14 +6,25 @@ on: push: branches: - master - - its-happening tags: - 'v[0-9]+.[0-9]+.[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+-**' jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' + - name: Test + uses: ./.github/actions/test docker: runs-on: ubuntu-latest + needs: [ test ] permissions: packages: write contents: read @@ -30,7 +41,7 @@ jobs: name: generate tags id: meta with: - images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + images: ghcr.io/${{ github.repository }} tags: | type=ref,event=branch type=sha,prefix= @@ -41,52 +52,53 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max build-linux: runs-on: ubuntu-latest + needs: [ test ] + strategy: + matrix: + go-arch: [amd64, arm64] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 'stable' + go-version: '1.21' - name: Setup run: | sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu go generate ./... - - name: Build amd64 - env: - CGO_ENABLED: 1 - GOOS: linux - GOARCH: amd64 - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - zip -qj $ZIP_OUTPUT bin/* - - name: Build arm64 + if [ ${{ matrix.go-arch }} == "arm64" ]; then + sudo apt install -y gcc-aarch64-linux-gnu + echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV + fi + - name: Build ${{ matrix.go-arch }} env: CGO_ENABLED: 1 GOOS: linux - GOARCH: arm64 - CC: aarch64-linux-gnu-gcc + GOARCH: ${{ matrix.go-arch }} run: | mkdir -p release ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd + go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd cp README.md LICENSE bin/ zip -qj $ZIP_OUTPUT bin/* - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: walletd - path: release/ + name: walletd-linux-${{ matrix.go-arch }} + path: release/* build-mac: runs-on: macos-latest + needs: [ test ] + strategy: + matrix: + go-arch: [amd64, arm64] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 'stable' + go-version: '1.21' - name: Setup env: APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} @@ -120,27 +132,10 @@ jobs: # generate go generate ./... - - name: Build amd64 - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - CGO_ENABLED: 1 - GOOS: darwin - GOARCH: amd64 - run: | - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - mkdir -p release - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/walletd - ditto -ck bin $ZIP_OUTPUT - xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - name: Build arm64 + + # resync system clock https://github.com/actions/runner/issues/2996#issuecomment-1833103110 + sudo sntp -sS time.windows.com + - name: Build ${{ matrix.go-arch }} env: APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} @@ -151,7 +146,7 @@ jobs: APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} CGO_ENABLED: 1 GOOS: darwin - GOARCH: arm64 + GOARCH: ${{ matrix.go-arch }} run: | ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip mkdir -p release @@ -160,17 +155,18 @@ jobs: /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/walletd ditto -ck bin $ZIP_OUTPUT xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: walletd - path: release/ + name: walletd-darwin-${{ matrix.go-arch }} + path: release/* build-windows: runs-on: windows-latest + needs: [ test ] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: 'stable' + go-version: '1.21' - name: Setup shell: bash run: | @@ -189,7 +185,73 @@ jobs: azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/walletd.exe cp README.md LICENSE bin/ 7z a $ZIP_OUTPUT ./bin/* - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 + with: + name: walletd-windows-amd64 + path: release/* + combine-release-assets: + runs-on: ubuntu-latest + needs: [ build-linux, build-mac, build-windows ] + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: walletd-release-assets-combined + + dispatch-homebrew: # only runs on full releases + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') + needs: [ build-mac ] + runs-on: ubuntu-latest + steps: + - name: Extract Tag Name + id: get_tag + run: echo "::set-output name=tag_name::${GITHUB_REF#refs/tags/}" + + - name: Dispatch + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} + repository: siafoundation/homebrew-sia + event-type: release-tagged + client-payload: > + { + "description": "walletd: The Next-Gen Sia Wallet", + "tag": "${{ steps.get_tag.outputs.tag_name }}", + "project": "walletd", + "workflow_id": "${{ github.run_id }}" + } + dispatch-linux: # always runs + needs: [ build-linux ] + runs-on: ubuntu-latest + steps: + - name: Build Dispatch Payload + id: get_payload + uses: actions/github-script@v7 + with: + script: | + const isRelease = context.ref.startsWith('refs/tags/v'), + isBeta = isRelease && context.ref.includes('-beta'), + tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master'; + + let component = 'nightly'; + if (isBeta) { + component = 'beta'; + } else if (isRelease) { + component = 'main'; + } + + return { + description: "walletd: The Next-Gen Sia Wallet", + tag: tag, + project: "walletd", + workflow_id: context.runId, + component: component + }; + + - name: Dispatch + uses: peter-evans/repository-dispatch@v3 with: - name: walletd - path: release/ + token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} + repository: siafoundation/linux + event-type: release-tagged + client-payload: ${{ steps.get_payload.outputs.result }} \ No newline at end of file From 9778b5fc5300368487d6598c7fd9efe6d9009dea Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 08:27:11 -0700 Subject: [PATCH 187/630] cmd,sqlite, wallet: Partial -> Personal --- README.md | 8 ++++---- cmd/walletd/main.go | 4 ++-- persist/sqlite/consensus.go | 2 +- wallet/manager.go | 16 ++++++++-------- wallet/wallet_test.go | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 52dd8ef..dbbf0cf 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ Setup guides are available at https://docs.sia.tech ### Index Mode `walletd` supports three different index modes for different use cases. -**Partial** +**Personal** -In partial index mode, `walletd` will only index addresses that are registered in the +In personal index mode, `walletd` will only index addresses that are registered in the wallet. This mode is recommended for most users, as it provides a good balance between comprehensiveness and resource usage. This is the default mode for `walletd`. @@ -74,7 +74,7 @@ The priority of configuration settings is as follows: -index.batch int max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 64) -index.mode string - address index mode (full, partial, none) (default "full") + address index mode (personal, full, none) (default "full") -network string network to connect to (default "mainnet") -upnp @@ -96,7 +96,7 @@ consensus: bootstrap: false enableUPnP: false index: - mode: partial # full, partial, none (full index will index the entire blockchain, partial will only index addresses that are registered in the wallet, none will treat the database as read-only and not index any new data) + mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) log: level: info # global log level diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index d9f3bc2..b71f92c 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -66,7 +66,7 @@ var cfg = config.Config{ Bootstrap: true, }, Index: config.Index{ - Mode: wallet.IndexModePartial, + Mode: wallet.IndexModePersonal, BatchSize: 64, }, Log: config.Log{ @@ -220,7 +220,7 @@ func main() { rootCmd.BoolVar(&cfg.Consensus.EnableUPNP, "upnp", cfg.Consensus.EnableUPNP, "attempt to forward ports and discover IP with UPnP") rootCmd.BoolVar(&cfg.Consensus.Bootstrap, "bootstrap", cfg.Consensus.Bootstrap, "attempt to bootstrap the network") - rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (full, partial, none)") + rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (personal, full, none)") rootCmd.IntVar(&cfg.Index.BatchSize, "index.batch", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") versionCmd := flagg.New("version", versionUsage) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 92761a6..af8e0fc 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -121,7 +121,7 @@ func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) er func (ut *updateTx) UpdateStateTree(changes []wallet.TreeNodeUpdate) error { if ut.indexMode != wallet.IndexModeFull { - panic("UpdateStateTree called in partial index mode") + panic("UpdateStateTree called in personal index mode") } stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES ($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value`) diff --git a/wallet/manager.go b/wallet/manager.go index 0568de9..3e664fa 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -16,7 +16,7 @@ import ( // IndexMode represents the index mode of the wallet manager. The index mode // determines how the wallet manager stores the consensus state. // -// IndexModePartial - The wallet manager scans the blockchain starting at +// IndexModePersonal - The wallet manager scans the blockchain starting at // genesis. Only state from addresses that are registered with a // wallet will be stored. If an address is added to a wallet after the // scan completes, the manager will need to rescan. @@ -28,7 +28,7 @@ import ( // useful for multiple nodes sharing the same database. None should only be used // when connecting to a database that is in "Full" mode. const ( - IndexModePartial IndexMode = iota + IndexModePersonal IndexMode = iota IndexModeFull IndexModeNone ) @@ -94,8 +94,8 @@ type ( // String returns the string representation of the index mode. func (i IndexMode) String() string { switch i { - case IndexModePartial: - return "partial" + case IndexModePersonal: + return "personal" case IndexModeFull: return "full" case IndexModeNone: @@ -108,8 +108,8 @@ func (i IndexMode) String() string { // UnmarshalText implements the encoding.TextUnmarshaler interface. func (i *IndexMode) UnmarshalText(buf []byte) error { switch string(buf) { - case "partial": - *i = IndexModePartial + case "personal": + *i = IndexModePersonal case "full": *i = IndexModeFull case "none": @@ -223,7 +223,7 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { // Scan rescans the chain starting from the given index. The scan will complete // when the chain manager reaches the current tip or the context is canceled. func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { - if m.indexMode != IndexModePartial { + if m.indexMode != IndexModePersonal { return fmt.Errorf("scans are disabled in index mode %s", m.indexMode) } @@ -270,7 +270,7 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch // NewManager creates a new wallet manager. func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { m := &Manager{ - indexMode: IndexModePartial, + indexMode: IndexModePersonal, syncBatchSize: defaultSyncBatchSize, chain: cm, diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 76fbfb0..6c34c8a 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -303,8 +303,8 @@ func TestReorg(t *testing.T) { } } - t.Run("IndexModePartial", func(t *testing.T) { - state, db, cm, w := setupNode(t, wallet.IndexModePartial) + t.Run("IndexModePersonal", func(t *testing.T) { + state, db, cm, w := setupNode(t, wallet.IndexModePersonal) testReorg(t, state, db, cm, w) }) From 79ee243bb209210fcc37752e355ebf6be8ea9475 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 10:01:12 -0700 Subject: [PATCH 188/630] update readme --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dbbf0cf..5924dc5 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,19 @@ Setup guides are available at https://docs.sia.tech **Personal** -In personal index mode, `walletd` will only index addresses that are registered in the +In "personal" index mode, `walletd` will only index addresses that are registered in the wallet. This mode is recommended for most users, as it provides a good balance between -comprehensiveness and resource usage. This is the default mode for `walletd`. +comprehensiveness and resource usage for personal wallets. This is the default +mode for `walletd`. -When adding existing addresses with history on chain, users will need to manually +When adding addresses with existing history on chain, users will need to manually initiate a rescan to index the new transactions. This can take some to complete, -depending on the number of blocks that need to be scanned. +depending on the number of blocks that need to be scanned. When adding addresses +with no existing history, a rescan is not necessary. **Full** -In full index mode, `walletd` will index the entire blockchain including all addresses +In "full" index mode, `walletd` will index the entire blockchain including all addresses and UTXOs. This is the most comprehensive mode, but it also requires the most resources. This mode is recommended for exchanges or wallet builders that need to support a large or unknown number of addresses. From b754c62bf751e99a66080bbb1f7f7c228d2f8879 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 15 May 2024 13:57:59 -0700 Subject: [PATCH 189/630] sqlite: clearer log keys --- persist/sqlite/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index af8e0fc..140130c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -225,7 +225,7 @@ func (ut *updateTx) RevertIndex(index types.ChainIndex, state wallet.RevertedSta // UpdateChainState implements chain.Subscriber func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { - log := s.log.Named("UpdateChainState").With(zap.Int("reverted", len(reverted)), zap.Int("applied", len(applied))) + log := s.log.Named("UpdateChainState").With(zap.Int("revertedUpdates", len(reverted)), zap.Int("appliedUpdates", len(applied))) return s.transaction(func(tx *txn) error { utx := &updateTx{ indexMode: s.indexMode, From 6314b95747529ea7c15980fc09f9acd54a41f8f6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 16 May 2024 07:49:08 -0700 Subject: [PATCH 190/630] ci: underscores for release assets --- .github/workflows/publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5ac8d24..a5f3cbd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -86,7 +86,7 @@ jobs: zip -qj $ZIP_OUTPUT bin/* - uses: actions/upload-artifact@v4 with: - name: walletd-linux-${{ matrix.go-arch }} + name: walletd_linux_${{ matrix.go-arch }} path: release/* build-mac: runs-on: macos-latest @@ -157,7 +157,7 @@ jobs: xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - uses: actions/upload-artifact@v4 with: - name: walletd-darwin-${{ matrix.go-arch }} + name: walletd_darwin_${{ matrix.go-arch }} path: release/* build-windows: runs-on: windows-latest @@ -187,7 +187,7 @@ jobs: 7z a $ZIP_OUTPUT ./bin/* - uses: actions/upload-artifact@v4 with: - name: walletd-windows-amd64 + name: walletd_windows_amd64 path: release/* combine-release-assets: runs-on: ubuntu-latest @@ -196,7 +196,7 @@ jobs: - name: Merge Artifacts uses: actions/upload-artifact/merge@v4 with: - name: walletd-release-assets-combined + name: walletd dispatch-homebrew: # only runs on full releases if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') From d01a53a77caaf5a4c4594f91444fe4ac484f02f4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 16 May 2024 07:49:37 -0700 Subject: [PATCH 191/630] ci: serial publish workflow --- .github/workflows/publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5f3cbd..20d650f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,6 +10,9 @@ on: - 'v[0-9]+.[0-9]+.[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+-**' +concurrency: + group: ${{ github.workflow }} + jobs: test: runs-on: ubuntu-latest From cde82a87411f07385999d6250ae4ad2820343afa Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 17 May 2024 13:58:43 -0700 Subject: [PATCH 192/630] sqlite,wallet: remove event data interface, add Type field to Event --- persist/sqlite/consensus.go | 2 +- persist/sqlite/wallet.go | 7 ++- wallet/manager.go | 2 +- wallet/wallet.go | 90 +++++-------------------------------- wallet/wallet_test.go | 80 ++++++++++++++++----------------- 5 files changed, 55 insertions(+), 126 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 140130c..fb2be02 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1043,7 +1043,7 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { } var eventID int64 - err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Data.EventType(), buf.String(), indexID).Scan(&eventID) + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, buf.String(), indexID).Scan(&eventID) if errors.Is(err, sql.ErrNoRows) { continue // skip if the event already exists } else if err != nil { diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 8e01e99..32f4481 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -448,15 +448,14 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e } func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { - var eventType string var eventBuf []byte - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &eventType, &eventBuf) + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) if err != nil { return } - switch eventType { + switch ev.Type { case wallet.EventTypeTransaction: var tx wallet.EventTransaction if err = json.Unmarshal(eventBuf, &tx); err != nil { @@ -482,7 +481,7 @@ func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { } ev.Data = &m default: - return wallet.Event{}, 0, fmt.Errorf("unknown event type: %s", eventType) + return wallet.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) } return } diff --git a/wallet/manager.go b/wallet/manager.go index 3e664fa..9d45b5f 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -33,7 +33,7 @@ const ( IndexModeNone ) -const defaultSyncBatchSize = 64 +const defaultSyncBatchSize = 1 type ( // An IndexMode determines the chain state that the wallet manager stores. diff --git a/wallet/wallet.go b/wallet/wallet.go index a80958b..5ed34c7 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -3,7 +3,6 @@ package wallet import ( "encoding/json" "errors" - "fmt" "strconv" "time" @@ -197,10 +196,6 @@ func Annotate(txn types.Transaction, ownsAddress func(types.Address) bool) PoolT return ptxn } -type eventData interface { - EventType() string -} - // An Event is something interesting that happened on the Sia blockchain. type Event struct { ID types.Hash256 `json:"id"` @@ -208,74 +203,8 @@ type Event struct { Timestamp time.Time `json:"timestamp"` MaturityHeight uint64 `json:"maturityHeight"` Relevant []types.Address `json:"relevant"` - Data eventData `json:"data"` -} - -// EventType implements Event. -func (*EventTransaction) EventType() string { return EventTypeTransaction } - -// EventType implements Event. -func (*EventMinerPayout) EventType() string { return EventTypeMinerPayout } - -// EventType implements Event. -func (*EventFoundationSubsidy) EventType() string { return EventTypeFoundationSubsidy } - -// EventType implements Event. -func (*EventContractPayout) EventType() string { return EventTypeContractPayout } - -// MarshalJSON implements json.Marshaler. -func (e Event) MarshalJSON() ([]byte, error) { - val, _ := json.Marshal(e.Data) - return json.Marshal(struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - MaturityHeight uint64 `json:"maturityHeight"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Data json.RawMessage `json:"data"` - }{ - ID: e.ID, - Timestamp: e.Timestamp, - Index: e.Index, - MaturityHeight: e.MaturityHeight, - Relevant: e.Relevant, - Type: e.Data.EventType(), - Data: val, - }) -} - -// UnmarshalJSON implements json.Unarshaler. -func (e *Event) UnmarshalJSON(data []byte) error { - var s struct { - ID types.Hash256 `json:"id"` - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - MaturityHeight uint64 `json:"maturityHeight"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Data json.RawMessage `json:"data"` - } - if err := json.Unmarshal(data, &s); err != nil { - return err - } - e.ID = s.ID - e.Timestamp = s.Timestamp - e.Index = s.Index - e.MaturityHeight = s.MaturityHeight - e.Relevant = s.Relevant - switch s.Type { - case (*EventTransaction)(nil).EventType(): - e.Data = new(EventTransaction) - case (*EventMinerPayout)(nil).EventType(): - e.Data = new(EventMinerPayout) - case (*EventContractPayout)(nil).EventType(): - e.Data = new(EventContractPayout) - } - if e.Data == nil { - return fmt.Errorf("unknown event type %q", s.Type) - } - return json.Unmarshal(s.Data, e.Data) + Type string `json:"type"` + Data any `json:"data"` } // A HostAnnouncement represents a host announcement within an EventTransaction. @@ -349,7 +278,7 @@ type ChainUpdate interface { // AppliedEvents extracts a list of relevant events from a chain update. func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { var events []Event - addEvent := func(id types.Hash256, maturityHeight uint64, v eventData, relevant []types.Address) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v any, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] @@ -366,6 +295,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f Index: cs.Index, MaturityHeight: maturityHeight, Relevant: unique, + Type: eventType, Data: v, }) } @@ -529,7 +459,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f e.Fee = e.Fee.Add(txn.MinerFees[i]) } - addEvent(types.Hash256(txn.ID()), cs.Index.Height, e, relevant) // transaction maturity height is the current block height + addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeTransaction, e, relevant) // transaction maturity height is the current block height } // handle v2 transactions @@ -599,7 +529,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } e.Fee = txn.MinerFee - addEvent(types.Hash256(txid), cs.Index.Height, e, relevant) // transaction maturity height is the current block height + addEvent(types.Hash256(txid), cs.Index.Height, EventTypeTransaction, e, relevant) // transaction maturity height is the current block height } // handle missed contracts @@ -615,7 +545,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } outputID := types.FileContractID(fce.ID).ValidOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeContractPayout, &EventContractPayout{ FileContract: fce, SiacoinOutput: sces[outputID], Missed: false, @@ -628,7 +558,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } outputID := types.FileContractID(fce.ID).MissedOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventContractPayout{ + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeContractPayout, &EventContractPayout{ FileContract: fce, SiacoinOutput: sces[outputID], Missed: true, @@ -641,7 +571,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { outputID := cs.Index.ID.MinerOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventMinerPayout{ + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeMinerPayout, &EventMinerPayout{ SiacoinOutput: sces[outputID], }, []types.Address{b.MinerPayouts[i].Address}) } @@ -652,7 +582,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f outputID := cs.Index.ID.FoundationOutputID() sce, ok := sces[outputID] if ok { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), &EventFoundationSubsidy{ + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeFoundationSubsidy, &EventFoundationSubsidy{ SiacoinOutput: sce, }, []types.Address{cs.FoundationPrimaryAddress}) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 6c34c8a..85b4b6e 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -169,8 +169,8 @@ func TestReorg(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created @@ -239,8 +239,8 @@ func TestReorg(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created @@ -377,8 +377,8 @@ func TestEphemeralBalance(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } else if events[0].ID != types.Hash256(minerPayoutID) { t.Fatalf("expected %v, got %v", minerPayoutID, events[0].ID) } @@ -469,12 +469,12 @@ func TestEphemeralBalance(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { // 1 payout, 2 transactions t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[2].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected miner payout event, got %v", events[2].Data.EventType()) - } else if events[1].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[1].Data.EventType()) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[2].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[2].Type) + } else if events[1].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[1].Type) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) } else if events[0].ID != types.Hash256(txn.ID()) { // child txn second @@ -508,8 +508,8 @@ func TestEphemeralBalance(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } } @@ -1013,8 +1013,8 @@ func TestOrphans(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created @@ -1194,8 +1194,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } // mine a block and send the payout to the first address @@ -1211,8 +1211,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected miner payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) } assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) @@ -1230,8 +1230,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected miner payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) } assertBalance(t, addr, expectedBalance1, types.ZeroCurrency, 0) @@ -1275,8 +1275,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } // check the events for the second address @@ -1284,8 +1284,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) @@ -1326,8 +1326,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } // check the events for the first address @@ -1335,8 +1335,8 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } } @@ -1400,8 +1400,8 @@ func TestV2(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // mine until the payout matures @@ -1454,8 +1454,8 @@ func TestV2(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeTransaction { - t.Fatalf("expected transaction event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events[0].Relevant[0] != addr { t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) } @@ -1712,8 +1712,8 @@ func TestReorgV2(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created @@ -1782,8 +1782,8 @@ func TestReorgV2(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created @@ -1955,8 +1955,8 @@ func TestOrphansV2(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Data.EventType() != wallet.EventTypeMinerPayout { - t.Fatalf("expected payout event, got %v", events[0].Data.EventType()) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) } // check that the utxo was created From dc3c388a730154b0b5c9eba60a7d58031c015e12 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Mon, 20 May 2024 14:55:56 +0000 Subject: [PATCH 193/630] ui: v0.21.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 620b0c4..6b2347d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.3 go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.20.0 + go.sia.tech/web/walletd v0.21.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.20.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ff14054..7ec56d5 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.20.0 h1:XqauYxR8AKGaUROUIVwlzWP535NQjP8+2PdrFKaFv3Y= -go.sia.tech/web/walletd v0.20.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.21.0 h1:w5LCl8AWhl0G2HuExgr/AVXJWQlgVzGLDXskbb7pNZs= +go.sia.tech/web/walletd v0.21.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= From 0ae6cbc6ee150ed5c5656bf7a79aff781f7c02c9 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 1 Jun 2024 11:05:31 -0500 Subject: [PATCH 194/630] update core and coreutils --- go.mod | 8 +++--- go.sum | 16 ++++++------ persist/sqlite/consensus.go | 24 ++++++++++++++--- persist/sqlite/consensus_test.go | 8 +++++- wallet/manager.go | 8 +++++- wallet/wallet_test.go | 45 +++++++++++++++++++++++++------- 6 files changed, 82 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 6b2347d..e0e7733 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.21.8 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.3 - go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 + go.sia.tech/core v0.2.5 + go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.21.0 go.uber.org/zap v1.27.0 @@ -21,11 +21,11 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - go.etcd.io/bbolt v1.3.9 // indirect + go.etcd.io/bbolt v1.3.10 // indirect go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/tools v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 7ec56d5..845df7b 100644 --- a/go.sum +++ b/go.sum @@ -10,12 +10,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= -go.sia.tech/core v0.2.3 h1:k+10zeV1V4bYFCGFaUiubbwRxlyV96WXForVlKnc8Rc= -go.sia.tech/core v0.2.3/go.mod h1:24liZWimivGQF+h3d14ly9oEpMIYxHPSgEMKmunxxi0= -go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7 h1:5AuiglkLdoBenrg41cJXJ4wTxkVTo85Asj9SPljnmiE= -go.sia.tech/coreutils v0.0.4-0.20240318195004-c73e571336f7/go.mod h1:QvsXghS4wqhJosQq3AkMjA2mJ6pbDB7PgG+w5b09/z0= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.sia.tech/core v0.2.5 h1:uGyaFQNPbbMOg6YpvSevx+FXoQn9Itfsq6rbB6MMq34= +go.sia.tech/core v0.2.5/go.mod h1:tf07w3f/8XRtK1XoHl8C/MXzzUp/MEUuZE22L5uoCDQ= +go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179 h1:afLw23p5JUJSeotJlICnh2fPFKlhNk4IxeWZozbuiO8= +go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179/go.mod h1:WTWYeFbAS4lBZhfXgot9DxZYzJ7sSulbzhEHKVfxDK0= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -30,8 +30,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index fb2be02..8701c6f 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/wallet" @@ -225,6 +226,10 @@ func (ut *updateTx) RevertIndex(index types.ChainIndex, state wallet.RevertedSta // UpdateChainState implements chain.Subscriber func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { + if len(applied) == 0 && len(reverted) == 0 { + return nil + } + log := s.log.Named("UpdateChainState").With(zap.Int("revertedUpdates", len(reverted)), zap.Int("appliedUpdates", len(applied))) return s.transaction(func(tx *txn) error { utx := &updateTx{ @@ -234,14 +239,27 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. relevantAddresses: make(map[types.Address]bool), } - state := applied[len(applied)-1].State - if err := wallet.UpdateChainState(utx, reverted, applied, s.indexMode, log); err != nil { return err - } else if err := setGlobalState(tx, state.Index, state.Elements.NumLeaves); err != nil { + } + + var state consensus.State + switch { + case len(applied) > 0: + state = applied[len(applied)-1].State + case len(reverted) > 0: + state = reverted[len(reverted)-1].State + } + + if err := setGlobalState(tx, state.Index, state.Elements.NumLeaves); err != nil { return fmt.Errorf("failed to set last committed index: %w", err) } + // skip pruning if there are no applied updates + if len(applied) == 0 { + return nil + } + if state.Index.Height > spentElementRetentionBlocks { pruneHeight := state.Index.Height - spentElementRetentionBlocks diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index af40db9..a93c89d 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -38,7 +38,13 @@ func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { } else if err := store.UpdateChainState(crus, caus); err != nil { tb.Fatalf("failed to update chain state: %v", err) } - index = caus[len(caus)-1].State.Index + + switch { + case len(caus) > 0: + index = caus[len(caus)-1].State.Index + case len(crus) > 0: + index = crus[len(crus)-1].State.Index + } } } diff --git a/wallet/manager.go b/wallet/manager.go index 9d45b5f..0c9dbe1 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -262,7 +262,13 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch } else if err := store.UpdateChainState(crus, caus); err != nil { return fmt.Errorf("failed to update chain state: %w", err) } - index = caus[len(caus)-1].State.Index + + switch { + case len(caus) > 0: + index = caus[len(caus)-1].State.Index + case len(crus) > 0: + index = crus[len(crus)-1].State.Index + } } return nil } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 85b4b6e..9adc05d 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -719,14 +719,22 @@ func TestScan(t *testing.T) { } expectedBalance1 := cm.TipState().BlockReward() + // mine a block to fund the first address - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr)}); err != nil { + b, ok := coreutils.MineBlock(cm, addr, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } - // mine a block to fund the second address expectedBalance2 := cm.TipState().BlockReward() - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr2)}); err != nil { + + // mine a block to fund the second address + b, ok = coreutils.MineBlock(cm, addr2, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } @@ -737,7 +745,9 @@ func TestScan(t *testing.T) { // mine until the first payout matures for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } } @@ -774,7 +784,10 @@ func TestScan(t *testing.T) { } // mine a block to mature the second payout - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + b, ok = coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } @@ -890,7 +903,11 @@ func TestSiafunds(t *testing.T) { if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { t.Fatal(err) - } else if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + } + + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } else if err := checkBalance(w1.ID, sendAmount); err != nil { t.Fatal(err) @@ -1538,13 +1555,17 @@ func TestScanV2(t *testing.T) { expectedBalance1 := cm.TipState().BlockReward() // mine a block to fund the first address - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr)}); err != nil { + if b, ok := coreutils.MineBlock(cm, addr, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } // mine a block to fund the second address expectedBalance2 := cm.TipState().BlockReward() - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, addr2)}); err != nil { + if b, ok := coreutils.MineBlock(cm, addr2, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } @@ -1555,7 +1576,9 @@ func TestScanV2(t *testing.T) { // mine until the first payout matures for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } } @@ -1592,7 +1615,9 @@ func TestScanV2(t *testing.T) { } // mine a block to mature the second payout - if err := cm.AddBlocks([]types.Block{testutil.MineBlock(cm, types.VoidAddress)}); err != nil { + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { t.Fatal(err) } From a554aa3cacb6536a3178c9bf91e5900cba80424e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 3 Jun 2024 08:14:12 -0700 Subject: [PATCH 195/630] deps: group dependabot updates --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cd88554..1996ae0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + groups: + all-dependencies: + patterns: + - "*" \ No newline at end of file From e46ce2c33a056c181bae284e1a84c63bb96a4fc4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 10 Jun 2024 09:33:41 -0700 Subject: [PATCH 196/630] go.mod: update deps --- .github/actions/test/action.yml | 2 +- go.mod | 20 ++++++++--------- go.sum | 40 ++++++++++++++++----------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 4a0a3e3..0b51e44 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: git config --global core.autocrlf false - name: Lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: skip-cache: true # - name: Analyze diff --git a/go.mod b/go.mod index e0e7733..bff3e0b 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module go.sia.tech/walletd -go 1.21.7 +go 1.21.8 -toolchain go1.21.8 +toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.5 - go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179 + go.sia.tech/core v0.2.6 + go.sia.tech/coreutils v0.0.5 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.21.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.20.0 + golang.org/x/term v0.21.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 @@ -23,9 +23,9 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect go.etcd.io/bbolt v1.3.10 // indirect go.sia.tech/mux v1.2.0 // indirect - go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/tools v0.7.0 // indirect + go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 845df7b..12e50b3 100644 --- a/go.sum +++ b/go.sum @@ -12,37 +12,37 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.2.5 h1:uGyaFQNPbbMOg6YpvSevx+FXoQn9Itfsq6rbB6MMq34= -go.sia.tech/core v0.2.5/go.mod h1:tf07w3f/8XRtK1XoHl8C/MXzzUp/MEUuZE22L5uoCDQ= -go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179 h1:afLw23p5JUJSeotJlICnh2fPFKlhNk4IxeWZozbuiO8= -go.sia.tech/coreutils v0.0.5-0.20240531191724-1de8f76a2179/go.mod h1:WTWYeFbAS4lBZhfXgot9DxZYzJ7sSulbzhEHKVfxDK0= +go.sia.tech/core v0.2.6 h1:JrbZwW4cPHCB2Q6TeqCsAj2GAFXrRqLq0q/GNevtnPU= +go.sia.tech/core v0.2.6/go.mod h1:B7ooFH3F6cLjxQz6IX33kgjOY392Ava7pzpPeccagac= +go.sia.tech/coreutils v0.0.5 h1:Jj03VrqAayYHgA9fwV13+X88WB+Wr1p8wuLw2B8d2FI= +go.sia.tech/coreutils v0.0.5/go.mod h1:SkSpHeq3tBh2ff4HXuBk2WtlhkYQQtdcvU4Yv1Rd2bU= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= -go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= -go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.21.0 h1:w5LCl8AWhl0G2HuExgr/AVXJWQlgVzGLDXskbb7pNZs= go.sia.tech/web/walletd v0.21.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 764b6421d5d1cdd1ff04fccdee3227438049480a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 10 Jun 2024 10:12:33 -0700 Subject: [PATCH 197/630] cmd: increase scan batch size --- cmd/walletd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index b71f92c..cf88218 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -67,7 +67,7 @@ var cfg = config.Config{ }, Index: config.Index{ Mode: wallet.IndexModePersonal, - BatchSize: 64, + BatchSize: 1000, }, Log: config.Log{ Level: "info", From e5cc77c76de5f3d7672649680d382005c7aecc21 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 11 Jun 2024 10:07:09 -0700 Subject: [PATCH 198/630] api,sqlite,wallet: add events endpoint --- api/server.go | 23 +++- persist/sqlite/events.go | 45 +++++++ wallet/manager.go | 11 +- wallet/wallet_test.go | 266 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 324 insertions(+), 21 deletions(-) create mode 100644 persist/sqlite/events.go diff --git a/api/server.go b/api/server.go index 23b5603..9525893 100644 --- a/api/server.go +++ b/api/server.go @@ -60,7 +60,7 @@ type ( AddAddress(id wallet.ID, addr wallet.Address) error RemoveAddress(id wallet.ID, addr types.Address) error Addresses(id wallet.ID) ([]wallet.Address, error) - Events(id wallet.ID, offset, limit int) ([]wallet.Event, error) + WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) WalletBalance(id wallet.ID) (wallet.Balance, error) @@ -71,6 +71,8 @@ type ( AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + Events(eventIDs []types.Hash256) ([]wallet.Event, error) + Reserve(ids []types.Hash256, duration time.Duration) error } ) @@ -382,7 +384,7 @@ func (s *server) walletsEventsHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - events, err := s.wm.Events(id, offset, limit) + events, err := s.wm.WalletEvents(id, offset, limit) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) return @@ -693,6 +695,21 @@ func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { jc.Encode(utxos) } +func (s *server) eventsHandlerGET(jc jape.Context) { + var eventID types.Hash256 + if jc.DecodeParam("id", &eventID) != nil { + return + } + events, err := s.wm.Events([]types.Hash256{eventID}) + if jc.Check("couldn't load events", err) != nil { + return + } else if len(events) == 0 { + jc.Error(errors.New("event not found"), http.StatusNotFound) + return + } + jc.Encode(events[0]) +} + // NewServer returns an HTTP handler that serves the walletd API. func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { srv := server{ @@ -742,5 +759,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /addresses/:addr/events": srv.addressesAddrEventsHandler, "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, + + "GET /events/:id": srv.eventsHandlerGET, }) } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go new file mode 100644 index 0000000..c030700 --- /dev/null +++ b/persist/sqlite/events.go @@ -0,0 +1,45 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// Events returns the events with the given event IDs. If an event is not found, +// it is skipped. +func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + // sqlite doesn't have easy support for IN clauses, use a statement since + // the number of event IDs is likely to be small instead of dynamically + // building the query + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + FROM events ev + INNER JOIN event_addresses ea ON (ev.id = ea.event_id) + INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) + INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) + WHERE ev.event_id = $1` + + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]wallet.Event, 0, len(eventIDs)) + for _, id := range eventIDs { + event, _, err := scanEvent(stmt.QueryRow(encode(id))) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return fmt.Errorf("failed to query transaction %q: %w", id, err) + } + events = append(events, event) + } + return nil + }) + return +} diff --git a/wallet/manager.go b/wallet/manager.go index 0c9dbe1..7adb6f4 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -72,6 +72,8 @@ type ( AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + Events(eventIDs []types.Hash256) ([]Event, error) + SetIndexMode(IndexMode) error LastCommittedIndex() (types.ChainIndex, error) } @@ -165,8 +167,8 @@ func (m *Manager) Addresses(walletID ID) ([]Address, error) { return m.store.WalletAddresses(walletID) } -// Events returns the events of the given wallet. -func (m *Manager) Events(walletID ID, offset, limit int) ([]Event, error) { +// WalletEvents returns the events of the given wallet. +func (m *Manager) WalletEvents(walletID ID, offset, limit int) ([]Event, error) { return m.store.WalletEvents(walletID, offset, limit) } @@ -191,6 +193,11 @@ func (m *Manager) WalletBalance(walletID ID) (Balance, error) { return m.store.WalletBalance(walletID) } +// Events returns the events with the given IDs. +func (m *Manager) Events(eventIDs []types.Hash256) ([]Event, error) { + return m.store.Events(eventIDs) +} + // Reserve reserves the given ids for the given duration. func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { m.mu.Lock() diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 9adc05d..25ae2e7 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "path/filepath" + "reflect" "testing" "time" @@ -164,7 +165,7 @@ func TestReorg(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -205,7 +206,7 @@ func TestReorg(t *testing.T) { } // check that the payout event was reverted - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 0 { @@ -234,7 +235,7 @@ func TestReorg(t *testing.T) { } // check that a payout event was recorded - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -372,7 +373,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -464,7 +465,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that both transactions were added - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 3 { // 1 payout, 2 transactions @@ -503,7 +504,7 @@ func TestEphemeralBalance(t *testing.T) { } // check that only the payout event remains - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1025,7 +1026,7 @@ func TestOrphans(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1085,7 +1086,7 @@ func TestOrphans(t *testing.T) { } // check that the transaction event was recorded - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 2 { @@ -1128,7 +1129,7 @@ func TestOrphans(t *testing.T) { } // check that the transaction event was reverted - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1357,6 +1358,237 @@ func TestFullIndex(t *testing.T) { } } +func TestEvents(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(addr2) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + assertBalance := func(t *testing.T, address types.Address, siacoin, immature types.Currency, siafund uint64) { + t.Helper() + + b, err := wm.AddressBalance(address) + if err != nil { + t.Fatal(err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if b.Siafunds != siafund { + t.Fatalf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } + } + + // check the events are empty for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // assert that the airdropped siafunds are on the second address + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + // check the events for the air dropped siafunds + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + // mine a block and send the payout to the first address + expectedBalance1 := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check the payout was received + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], events[0]) { + t.Fatalf("expected event %v to match %v", events[0], events2) + } + + assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // check that the events did not change + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } + + assertBalance(t, addr, expectedBalance1, types.ZeroCurrency, 0) + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + + // send half siacoins to the second address + utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], events[0]) { + t.Fatalf("expected event %v to match %v", events[0], events2) + } + + // check the events for the second address + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], events[0]) { + t.Fatalf("expected event %v to match %v", events[0], events2) + } + + sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + + // send the siafunds to the first address + policy = types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk2.PublicKey())) + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sf[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + ClaimAddress: addr2, // claim address shouldn't create an event since the value is 0 + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sf[0].SiafundOutput.Value}, + }, + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk2.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], events[0]) { + t.Fatalf("expected event %v to match %v", events[0], events2) + } + + // check the events for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeTransaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], events[0]) { + t.Fatalf("expected event %v to match %v", events[0], events2) + } +} + func TestV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -1412,7 +1644,7 @@ func TestV2(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1466,7 +1698,7 @@ func TestV2(t *testing.T) { } // check that a transaction event was recorded - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 2 { @@ -1732,7 +1964,7 @@ func TestReorgV2(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1773,7 +2005,7 @@ func TestReorgV2(t *testing.T) { } // check that the payout event was reverted - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 0 { @@ -1802,7 +2034,7 @@ func TestReorgV2(t *testing.T) { } // check that a payout event was recorded - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -1975,7 +2207,7 @@ func TestOrphansV2(t *testing.T) { } // check that a payout event was recorded - events, err := wm.Events(w.ID, 0, 100) + events, err := wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { @@ -2026,7 +2258,7 @@ func TestOrphansV2(t *testing.T) { } // check that the transaction event was recorded - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 2 { @@ -2069,7 +2301,7 @@ func TestOrphansV2(t *testing.T) { } // check that the transaction event was reverted - events, err = wm.Events(w.ID, 0, 100) + events, err = wm.WalletEvents(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(events) != 1 { From 4a5e0c9d76a364bcf404991d66eac01f017166ba Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 11 Jun 2024 13:37:57 -0700 Subject: [PATCH 199/630] cmd: add config command --- cmd/walletd/config.go | 255 ++++++++++++++++++++++++++++++++++++++++++ cmd/walletd/main.go | 32 ++---- 2 files changed, 264 insertions(+), 23 deletions(-) create mode 100644 cmd/walletd/config.go diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go new file mode 100644 index 0000000..adfadfd --- /dev/null +++ b/cmd/walletd/config.go @@ -0,0 +1,255 @@ +package main + +import ( + "bufio" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + + "go.sia.tech/walletd/wallet" + "golang.org/x/term" + "gopkg.in/yaml.v3" +) + +// readPasswordInput reads a password from stdin. +func readPasswordInput(context string) string { + fmt.Printf("%s: ", context) + input, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + stdoutFatalError("Could not read input: " + err.Error()) + } + fmt.Println("") + return string(input) +} + +func readInput(context string) string { + fmt.Printf("%s: ", context) + r := bufio.NewReader(os.Stdin) + input, err := r.ReadString('\n') + if err != nil { + stdoutFatalError("Could not read input: " + err.Error()) + } + return strings.TrimSpace(input) +} + +// wrapANSI wraps the output in ANSI escape codes if enabled. +func wrapANSI(prefix, output, suffix string) string { + if cfg.Log.StdOut.EnableANSI { + return prefix + output + suffix + } + return output +} + +func humanList(s []string, sep string) string { + if len(s) == 0 { + return "" + } else if len(s) == 1 { + return fmt.Sprintf(`%q`, s[0]) + } else if len(s) == 2 { + return fmt.Sprintf(`%q %s %q`, s[0], sep, s[1]) + } + + var sb strings.Builder + for i, v := range s { + if i != 0 { + sb.WriteString(", ") + } + if i == len(s)-1 { + sb.WriteString("or ") + } + sb.WriteString(`"`) + sb.WriteString(v) + sb.WriteString(`"`) + } + return sb.String() +} + +func promptQuestion(question string, answers []string) string { + for { + input := readInput(fmt.Sprintf("%s (%s)", question, strings.Join(answers, "/"))) + for _, answer := range answers { + if strings.EqualFold(input, answer) { + return answer + } + } + fmt.Println(wrapANSI("\033[31m", fmt.Sprintf("Answer must be %s", humanList(answers, "or")), "\033[0m")) + } +} + +func promptYesNo(question string) bool { + answer := promptQuestion(question, []string{"yes", "no"}) + return strings.EqualFold(answer, "yes") +} + +// stdoutFatalError prints an error message to stdout and exits with a 1 exit code. +func stdoutFatalError(msg string) { + stdoutError(msg) + os.Exit(1) +} + +// stdoutError prints an error message to stdout +func stdoutError(msg string) { + if cfg.Log.StdOut.EnableANSI { + fmt.Println(wrapANSI("\033[31m", msg, "\033[0m")) + } else { + fmt.Println(msg) + } +} + +func setAPIPassword() { + // retry until a valid API password is entered + for { + fmt.Println("Please choose a password to unlock walletd.") + fmt.Println("This password will be required to access the admin UI in your web browser.") + fmt.Println("(The password must be at least 4 characters.)") + cfg.HTTP.Password = readPasswordInput("Enter password") + if len(cfg.HTTP.Password) >= 4 { + break + } + + fmt.Println(wrapANSI("\033[31m", "Password must be at least 4 characters!", "\033[0m")) + fmt.Println("") + } +} + +func setDataDirectory() { + if cfg.Directory == "" { + cfg.Directory = "." + } + + dir, err := filepath.Abs(cfg.Directory) + if err != nil { + stdoutFatalError("Could not get absolute path of data directory: " + err.Error()) + } + + fmt.Println("The data directory is where walletd will store its metadata and consensus data.") + fmt.Println("This directory should be on a fast, reliable storage device, preferably an SSD.") + fmt.Println("") + + _, existsErr := os.Stat(filepath.Join(cfg.Directory, "walletd.sqlite3")) + dataExists := existsErr == nil + if dataExists { + fmt.Println(wrapANSI("\033[33m", "There is existing data in the data directory.", "\033[0m")) + fmt.Println(wrapANSI("\033[33m", "If you change your data directory, you will need to manually move consensus, gateway, tpool, and walletd.sqlite3 to the new directory.", "\033[0m")) + } + + if !promptYesNo("Would you like to change the data directory? (Current: " + dir + ")") { + return + } + cfg.Directory = readInput("Enter data directory") +} + +func setListenAddress(context string, value *string) { + // will continue to prompt until a valid value is entered + for { + input := readInput(fmt.Sprintf("%s (currently %q)", context, *value)) + if input == "" { + return + } + + host, port, err := net.SplitHostPort(input) + if err != nil { + stdoutError(fmt.Sprintf("Invalid %s port %q: %s", context, input, err.Error())) + continue + } + + n, err := strconv.Atoi(port) + if err != nil { + stdoutError(fmt.Sprintf("Invalid %s port %q: %s", context, input, err.Error())) + continue + } else if n < 0 || n > 65535 { + stdoutError(fmt.Sprintf("Invalid %s port %q: must be between 0 and 65535", context, input)) + continue + } + *value = net.JoinHostPort(host, port) + return + } +} + +func setAdvancedConfig() { + if !promptYesNo("Would you like to configure advanced settings?") { + return + } + + fmt.Println("") + fmt.Println("Advanced settings are used to configure walletd's behavior.") + fmt.Println("You can leave these settings blank to use the defaults.") + fmt.Println("") + + fmt.Println("The HTTP address is used to serve the host's admin API.") + fmt.Println("The admin API is used to configure the host.") + fmt.Println("It should not be exposed to the public internet without setting up a reverse proxy.") + setListenAddress("HTTP Address", &cfg.HTTP.Address) + + fmt.Println("") + fmt.Println("The gateway address is used to connect to the Sia network.") + fmt.Println("It should be reachable from other Sia nodes.") + setListenAddress("Gateway Address", &cfg.Consensus.GatewayAddress) + + fmt.Println("") + fmt.Println("Index mode determines how much of the blockchain to store.") + fmt.Println(`"personal" mode stores events only relevant to addresses associated with a wallet.`) + fmt.Println("To add new addresses, the wallet must be rescanned. This is the default mode.") + fmt.Println("") + fmt.Println(`"full" mode stores all blockchain events. This mode is useful for exchanges and shared wallet clients.`) + fmt.Println("This mode requires significantly more disk space, but does not require rescanning when adding new addresses.") + fmt.Println("") + fmt.Println("This cannot be changed later without resetting walletd.") + fmt.Printf("Currently %q\n", cfg.Index.Mode) + mode := readInput(`Enter index mode ("personal" or "full")`) + if strings.EqualFold(mode, "personal") { + cfg.Index.Mode = wallet.IndexModePersonal + } else if strings.EqualFold(mode, "full") { + cfg.Index.Mode = wallet.IndexModeFull + } else { + stdoutFatalError("Invalid index mode: " + mode) + } + + fmt.Println("") + fmt.Println("The network is the blockchain network that walletd will connect to.") + fmt.Println("Mainnet is the default network.") + fmt.Println("Zen is a production-like testnet.") + fmt.Println("This cannot be changed later without resetting walletd.") + fmt.Printf("Currently %q\n", cfg.Consensus.Network) + cfg.Consensus.Network = readInput(`Enter network ("mainnet" or "zen")`) +} + +func buildConfig() { + // write the config file + configPath := "walletd.yml" + if str := os.Getenv("WALLETD_CONFIG_FILE"); str != "" { + configPath = str + } + + if _, err := os.Stat(configPath); err == nil { + if !promptYesNo("walletd.yml already exists. Would you like to overwrite it?") { + return + } + } + + fmt.Println("") + setDataDirectory() + + fmt.Println("") + setAPIPassword() + + fmt.Println("") + setAdvancedConfig() + + // write the config file + f, err := os.Create(configPath) + if err != nil { + stdoutFatalError("failed to create config file: " + err.Error()) + return + } + defer f.Close() + + enc := yaml.NewEncoder(f) + if err := enc.Encode(cfg); err != nil { + stdoutFatalError("failed to encode config file: " + err.Error()) + return + } +} diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index cf88218..a0596f5 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -102,29 +102,6 @@ func getAPIPassword() string { return apiPassword } -// stdoutFatalError prints an error message to stdout and exits with a 1 exit code. -func stdoutFatalError(msg string) { - stdoutError(msg) - os.Exit(1) -} - -// wrapANSI wraps the output in ANSI escape codes if enabled. -func wrapANSI(prefix, output, suffix string) string { - if cfg.Log.StdOut.EnableANSI { - return prefix + output + suffix - } - return output -} - -// stdoutError prints an error message to stdout -func stdoutError(msg string) { - if cfg.Log.StdOut.EnableANSI { - fmt.Println(wrapANSI("\033[31m", msg, "\033[0m")) - } else { - fmt.Println(msg) - } -} - // tryLoadConfig loads the config file specified by the WALLETD_CONFIG_FILE. If // the config file does not exist, it will not be loaded. func tryLoadConfig() { @@ -225,6 +202,7 @@ func main() { versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) + configCmd := flagg.New("config", "interactively configure walletd") mineCmd := flagg.New("mine", mineUsage) mineCmd.IntVar(&minerBlocks, "n", -1, "mine this many blocks. If negative, mine indefinitely") @@ -233,6 +211,7 @@ func main() { cmd := flagg.Parse(flagg.Tree{ Cmd: rootCmd, Sub: []flagg.Tree{ + {Cmd: configCmd}, {Cmd: versionCmd}, {Cmd: seedCmd}, {Cmd: mineCmd}, @@ -360,6 +339,13 @@ func main() { fmt.Println("Recovery Phrase:", recoveryPhrase) fmt.Println("Address", addr) + case configCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + + buildConfig() case mineCmd: if len(cmd.Args()) != 0 { cmd.Usage() From 97b287b551e8b110210916ec2c36ef98ca1005a7 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 13 Jun 2024 16:28:20 -0700 Subject: [PATCH 200/630] api, sqlite, wallet: refactor events, fix unconfirmed events --- api/api_test.go | 4 +- api/client.go | 6 - api/server.go | 47 ++-- persist/sqlite/addresses.go | 127 +++++++++ persist/sqlite/events.go | 46 ++++ persist/sqlite/wallet.go | 262 +++++++++++++++---- wallet/addresses.go | 109 +++++++- wallet/events.go | 79 ++++++ wallet/manager.go | 42 ++- wallet/wallet.go | 500 ++++++++++-------------------------- wallet/wallet_test.go | 471 ++++++++++++++++++++++++++++++++- 11 files changed, 1225 insertions(+), 468 deletions(-) create mode 100644 wallet/events.go diff --git a/api/api_test.go b/api/api_test.go index a7222b8..584eb36 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -371,12 +371,12 @@ func TestWallet(t *testing.T) { t.Fatal("event history should be empty") } - tpool, err := wc.PoolTransactions() + /*tpool, err := wc.PoolTransactions() if err != nil { t.Fatal(err) } else if len(tpool) != 1 { t.Fatal("txpool should have one transaction") - } + }*/ cs := cm.TipState() b := types.Block{ diff --git a/api/client.go b/api/client.go index 8e6931c..ba87ded 100644 --- a/api/client.go +++ b/api/client.go @@ -190,12 +190,6 @@ func (c *WalletClient) Events(offset, limit int) (resp []wallet.Event, err error return } -// PoolTransactions returns all txpool transactions relevant to the wallet. -func (c *WalletClient) PoolTransactions() (resp []wallet.PoolTransaction, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/txpool", c.id), &resp) - return -} - // SiacoinOutputs returns the set of unspent outputs controlled by the wallet. func (c *WalletClient) SiacoinOutputs(offset, limit int) (sc []types.SiacoinElement, err error) { err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &sc) diff --git a/api/server.go b/api/server.go index 9525893..f8c2cad 100644 --- a/api/server.go +++ b/api/server.go @@ -61,15 +61,16 @@ type ( RemoveAddress(id wallet.ID, addr types.Address) error Addresses(id wallet.ID) ([]wallet.Address, error) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) + WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) WalletBalance(id wallet.ID) (wallet.Balance, error) - Annotate(id wallet.ID, pool []types.Transaction) ([]wallet.PoolTransaction, error) - AddressBalance(address types.Address) (balance wallet.Balance, err error) - AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) - AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) - AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + AddressBalance(address types.Address) (wallet.Balance, error) + AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) + AddressUnconfirmedEvents(address types.Address) ([]wallet.Event, error) + AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, error) + AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, error) Events(eventIDs []types.Hash256) ([]wallet.Event, error) @@ -394,19 +395,21 @@ func (s *server) walletsEventsHandler(jc jape.Context) { jc.Encode(events) } -func (s *server) walletsTxpoolHandler(jc jape.Context) { +func (s *server) walletsEventsUnconfirmedHandlerGET(jc jape.Context) { var id wallet.ID if jc.DecodeParam("id", &id) != nil { return } - pool, err := s.wm.Annotate(id, s.cm.PoolTransactions()) + + events, err := s.wm.WalletUnconfirmedEvents(id) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) return - } else if jc.Check("couldn't annotate pool", err) != nil { + } else if err != nil { + jc.Error(err, http.StatusInternalServerError) return } - jc.Encode(pool) + jc.Encode(events) } func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { @@ -641,7 +644,7 @@ func (s *server) addressesAddrBalanceHandler(jc jape.Context) { jc.Encode(BalanceResponse(b)) } -func (s *server) addressesAddrEventsHandler(jc jape.Context) { +func (s *server) addressesAddrEventsHandlerGET(jc jape.Context) { var addr types.Address if jc.DecodeParam("addr", &addr) != nil { return @@ -659,6 +662,19 @@ func (s *server) addressesAddrEventsHandler(jc jape.Context) { jc.Encode(events) } +func (s *server) addressesAddrEventsUnconfirmedHandlerGET(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + events, err := s.wm.AddressUnconfirmedEvents(addr) + if jc.Check("couldn't load events", err) != nil { + return + } + jc.Encode(events) +} + func (s *server) addressesAddrOutputsSCHandler(jc jape.Context) { var addr types.Address if jc.DecodeParam("addr", &addr) != nil { @@ -747,7 +763,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /wallets/:id/addresses": srv.walletsAddressesHandlerGET, "GET /wallets/:id/balance": srv.walletsBalanceHandler, "GET /wallets/:id/events": srv.walletsEventsHandler, - "GET /wallets/:id/txpool": srv.walletsTxpoolHandler, + "GET /wallets/:id/events/unconfirmed": srv.walletsEventsUnconfirmedHandlerGET, "GET /wallets/:id/outputs/siacoin": srv.walletsOutputsSiacoinHandler, "GET /wallets/:id/outputs/siafund": srv.walletsOutputsSiafundHandler, "POST /wallets/:id/reserve": srv.walletsReserveHandler, @@ -755,10 +771,11 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "POST /wallets/:id/fund": srv.walletsFundHandler, "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, - "GET /addresses/:addr/balance": srv.addressesAddrBalanceHandler, - "GET /addresses/:addr/events": srv.addressesAddrEventsHandler, - "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, - "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, + "GET /addresses/:addr/balance": srv.addressesAddrBalanceHandler, + "GET /addresses/:addr/events": srv.addressesAddrEventsHandlerGET, + "GET /addresses/:addr/events/unconfirmed": srv.addressesAddrEventsUnconfirmedHandlerGET, + "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, + "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, "GET /events/:id": srv.eventsHandlerGET, }) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index e2da36b..f395691 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "time" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" @@ -144,3 +145,129 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) }) return } + +// AnnotateV1Events annotates a list of unconfirmed transactions with +// relevant addresses and siacoin/siafund elements. +func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + siacoinElementStmt, err := tx.Prepare(`SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin statement: %w", err) + } + defer siacoinElementStmt.Close() + + siacoinElementCache := make(map[types.SiacoinOutputID]types.SiacoinElement) + fetchSiacoinElement := func(id types.SiacoinOutputID) (types.SiacoinElement, error) { + if se, ok := siacoinElementCache[id]; ok { + return se, nil + } + + se, err := scanSiacoinElement(siacoinElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiacoinElement{}, fmt.Errorf("failed to fetch siacoin element: %w", err) + } + siacoinElementCache[id] = se + return se, nil + } + + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siafund statement: %w", err) + } + defer siafundElementStmt.Close() + + siafundElementCache := make(map[types.SiafundOutputID]types.SiafundElement) + fetchSiafundElement := func(id types.SiafundOutputID) (types.SiafundElement, error) { + if se, ok := siafundElementCache[id]; ok { + return se, nil + } + + se, err := scanSiafundElement(siafundElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiafundElement{}, fmt.Errorf("failed to fetch siafund element: %w", err) + } + siafundElementCache[id] = se + return se, nil + } + + addEvent := func(id types.Hash256, data wallet.EventData) { + annotated = append(annotated, wallet.Event{ + ID: id, + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height, + Type: wallet.EventTypeV1Transaction, + Data: data, + }) + } + + for _, txn := range v1 { + var relevant bool + ev := wallet.EventV1Transaction{ + Transaction: txn, + } + + for _, input := range txn.SiacoinInputs { + // fetch the siacoin element + sce, err := fetchSiacoinElement(input.ParentID) + if errors.Is(err, sql.ErrNoRows) { + continue // ignore elements that are not found + } else if err != nil { + return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + } + ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) + relevant = true + } + + for i, output := range txn.SiacoinOutputs { + sce := types.SiacoinElement{ + StateElement: types.StateElement{ + ID: types.Hash256(txn.SiacoinOutputID(i)), + LeafIndex: types.EphemeralLeafIndex, + }, + SiacoinOutput: output, + } + siacoinElementCache[types.SiacoinOutputID(sce.StateElement.ID)] = sce + relevant = true + } + + for _, input := range txn.SiafundInputs { + // fetch the siafund element + sfe, err := fetchSiafundElement(input.ParentID) + if errors.Is(err, sql.ErrNoRows) { + continue // ignore elements that are not found + } else if err != nil { + return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + } + ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) + relevant = true + } + + for i, output := range txn.SiafundOutputs { + sfe := types.SiafundElement{ + StateElement: types.StateElement{ + ID: types.Hash256(txn.SiafundOutputID(i)), + LeafIndex: types.EphemeralLeafIndex, + }, + SiafundOutput: output, + } + siafundElementCache[types.SiafundOutputID(sfe.StateElement.ID)] = sfe + relevant = true + } + + if !relevant { + continue + } + + addEvent(types.Hash256(txn.ID()), ev) + } + return nil + }) + return +} diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index c030700..de7dc2d 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -2,6 +2,7 @@ package sqlite import ( "database/sql" + "encoding/json" "errors" "fmt" @@ -43,3 +44,48 @@ func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err err }) return } + +func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { + var eventBuf []byte + + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) + if err != nil { + return + } + + switch ev.Type { + case wallet.EventTypeV1Transaction: + var tx wallet.EventV1Transaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + ev.Data = tx + case wallet.EventTypeV2Transaction: + var tx wallet.EventV2Transaction + if err = json.Unmarshal(eventBuf, &tx); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) + } + ev.Data = tx + case wallet.EventTypeV1ContractResolution: + var r wallet.EventV1ContractResolution + if err = json.Unmarshal(eventBuf, &r); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) + } + ev.Data = r + case wallet.EventTypeV2ContractResolution: + var r wallet.EventV2ContractResolution + if err = json.Unmarshal(eventBuf, &r); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal file contract event: %w", err) + } + ev.Data = r + case wallet.EventTypeSiafundClaim, wallet.EventTypeMinerPayout, wallet.EventTypeFoundationSubsidy: + var p wallet.EventPayout + if err = json.Unmarshal(eventBuf, &p); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal event %q (%q): %w", ev.ID, ev.Type, err) + } + ev.Data = p + default: + return wallet.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) + } + return +} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 32f4481..d596578 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -2,7 +2,6 @@ package sqlite import ( "database/sql" - "encoding/json" "errors" "fmt" "math/bits" @@ -342,21 +341,21 @@ func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) return } -// Annotate annotates a list of transactions using the wallet's addresses. -func (s *Store) Annotate(id wallet.ID, txns []types.Transaction) (annotated []wallet.PoolTransaction, err error) { +// WalletUnconfirmedEvents annotates a list of unconfirmed transactions with +// relevant addresses and siacoin/siafund elements. +func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err } - const query = `SELECT sa.id FROM sia_addresses sa -INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) -WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` - stmt, err := tx.Prepare(query) + addrStmt, err := tx.Prepare(`SELECT sa.id FROM sia_addresses sa + INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) + WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to prepare address statement: %w", err) } - defer stmt.Close() + defer addrStmt.Close() // note: this would be more performant for small wallets to load all // addresses into memory. However, for larger wallets (> 10K addresses), @@ -364,20 +363,214 @@ WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1` // address. Monitor performance and consider changing this in the // future. From a memory perspective, it would be fine to lazy load all // addresses into memory. + checkedAddresses := make(map[types.Address]bool) ownsAddress := func(address types.Address) bool { + if relevant, ok := checkedAddresses[address]; ok { + return relevant + } + var dbID int64 - err := stmt.QueryRow(id, encode(address)).Scan(&dbID) + err := addrStmt.QueryRow(id, encode(address)).Scan(&dbID) if err != nil && !errors.Is(err, sql.ErrNoRows) { panic(err) // database error } - return err == nil + relevant := err == nil + checkedAddresses[address] = relevant + return relevant + } + + siacoinElementStmt, err := tx.Prepare(`SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin statement: %w", err) } + defer siacoinElementStmt.Close() - for _, txn := range txns { - ptxn := wallet.Annotate(txn, ownsAddress) - if ptxn.Type != "unrelated" { - annotated = append(annotated, ptxn) + siacoinElementCache := make(map[types.SiacoinOutputID]types.SiacoinElement) + fetchSiacoinElement := func(id types.SiacoinOutputID) (types.SiacoinElement, error) { + if se, ok := siacoinElementCache[id]; ok { + return se, nil + } + + se, err := scanSiacoinElement(siacoinElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiacoinElement{}, fmt.Errorf("failed to fetch siacoin element: %w", err) } + siacoinElementCache[id] = se + return se, nil + } + + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siafund statement: %w", err) + } + defer siafundElementStmt.Close() + + siafundElementCache := make(map[types.SiafundOutputID]types.SiafundElement) + fetchSiafundElement := func(id types.SiafundOutputID) (types.SiafundElement, error) { + if se, ok := siafundElementCache[id]; ok { + return se, nil + } + + se, err := scanSiafundElement(siafundElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiafundElement{}, fmt.Errorf("failed to fetch siafund element: %w", err) + } + siafundElementCache[id] = se + return se, nil + } + + addEvent := func(id types.Hash256, eventType string, data wallet.EventData, relevant []types.Address) { + annotated = append(annotated, wallet.Event{ + ID: id, + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height + 1, + Type: eventType, + Data: data, + Relevant: relevant, + }) + } + + for _, txn := range v1 { + var relevant []types.Address + seen := make(map[types.Address]bool) + ev := wallet.EventV1Transaction{ + Transaction: txn, + } + + for _, input := range txn.SiacoinInputs { + address := input.UnlockConditions.UnlockHash() + if !ownsAddress(address) { + continue + } + + if !seen[address] { + seen[address] = true + relevant = append(relevant, address) + } + + // fetch the siacoin element + sce, err := fetchSiacoinElement(input.ParentID) + if err != nil { + return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + } + ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) + } + + for i, output := range txn.SiacoinOutputs { + if !ownsAddress(output.Address) { + continue + } + + if !seen[output.Address] { + seen[output.Address] = true + relevant = append(relevant, output.Address) + } + + sce := types.SiacoinElement{ + StateElement: types.StateElement{ + ID: types.Hash256(txn.SiacoinOutputID(i)), + LeafIndex: types.EphemeralLeafIndex, + }, + SiacoinOutput: output, + } + siacoinElementCache[types.SiacoinOutputID(sce.StateElement.ID)] = sce + } + + for _, input := range txn.SiafundInputs { + address := input.UnlockConditions.UnlockHash() + if !ownsAddress(address) { + continue + } + + if !seen[address] { + seen[address] = true + relevant = append(relevant, address) + } + + // fetch the siafund element + sfe, err := fetchSiafundElement(input.ParentID) + if err != nil { + return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + } + ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) + } + + for i, output := range txn.SiafundOutputs { + if !ownsAddress(output.Address) { + continue + } + + if !seen[output.Address] { + seen[output.Address] = true + relevant = append(relevant, output.Address) + } + + sfe := types.SiafundElement{ + StateElement: types.StateElement{ + ID: types.Hash256(txn.SiafundOutputID(i)), + LeafIndex: types.EphemeralLeafIndex, + }, + SiafundOutput: output, + } + siafundElementCache[types.SiafundOutputID(sfe.StateElement.ID)] = sfe + } + + if len(relevant) == 0 { + continue + } + addEvent(types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, ev, relevant) + } + + // only need to check if the address is relevant for v2 transactions + // the inputs contain the necessary metadata for calculating value + for _, txn := range v2 { + var relevant []types.Address + seen := make(map[types.Address]bool) + + for _, sci := range txn.SiacoinInputs { + if !ownsAddress(sci.Parent.SiacoinOutput.Address) || seen[sci.Parent.SiacoinOutput.Address] { + continue + } + seen[sci.Parent.SiacoinOutput.Address] = true + relevant = append(relevant, sci.Parent.SiacoinOutput.Address) + } + + for _, sco := range txn.SiacoinOutputs { + if !ownsAddress(sco.Address) || seen[sco.Address] { + continue + } + seen[sco.Address] = true + relevant = append(relevant, sco.Address) + } + + for _, sfi := range txn.SiafundInputs { + if !ownsAddress(sfi.Parent.SiafundOutput.Address) || seen[sfi.Parent.SiafundOutput.Address] { + continue + } + seen[sfi.Parent.SiafundOutput.Address] = true + relevant = append(relevant, sfi.Parent.SiafundOutput.Address) + } + + for _, sfo := range txn.SiafundOutputs { + if !ownsAddress(sfo.Address) || seen[sfo.Address] { + continue + } + seen[sfo.Address] = true + relevant = append(relevant, sfo.Address) + } + + if len(relevant) == 0 { + continue + } + + addEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, wallet.EventV2Transaction(txn), relevant) } return nil }) @@ -447,45 +640,6 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e return } -func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { - var eventBuf []byte - - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) - if err != nil { - return - } - - switch ev.Type { - case wallet.EventTypeTransaction: - var tx wallet.EventTransaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - ev.Data = &tx - case wallet.EventTypeContractPayout: - var m wallet.EventContractPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - ev.Data = &m - case wallet.EventTypeMinerPayout: - var m wallet.EventMinerPayout - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal payout event: %w", err) - } - ev.Data = &m - case wallet.EventTypeFoundationSubsidy: - var m wallet.EventFoundationSubsidy - if err = json.Unmarshal(eventBuf, &m); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal foundation subsidy event: %w", err) - } - ev.Data = &m - default: - return wallet.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) - } - return -} - func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev diff --git a/wallet/addresses.go b/wallet/addresses.go index 85e22b1..841734c 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -1,17 +1,16 @@ package wallet -import "go.sia.tech/core/types" +import ( + "time" + + "go.sia.tech/core/types" +) // AddressBalance returns the balance of a single address. func (m *Manager) AddressBalance(address types.Address) (balance Balance, err error) { return m.store.AddressBalance(address) } -// AddressEvents returns the events of a single address. -func (m *Manager) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) { - return m.store.AddressEvents(address, offset, limit) -} - // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { return m.store.AddressSiacoinOutputs(address, offset, limit) @@ -21,3 +20,101 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) { return m.store.AddressSiafundOutputs(address, offset, limit) } + +// AddressEvents returns the events of a single address. +func (m *Manager) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) { + return m.store.AddressEvents(address, offset, limit) +} + +// AddressUnconfirmedEvents returns the unconfirmed events for a single address. +func (m *Manager) AddressUnconfirmedEvents(address types.Address) ([]Event, error) { + index := m.chain.Tip() + index.Height++ + index.ID = types.BlockID{} + timestamp := time.Now() + + v1, v2 := m.chain.PoolTransactions(), m.chain.V2PoolTransactions() + + relevantV1Txn := func(txn types.Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + return false + } + + relevantV1 := v1[:0] + for _, txn := range v1 { + if !relevantV1Txn(txn) { + continue + } + relevantV1 = append(relevantV1, txn) + } + + events, err := m.store.AnnotateV1Events(index, timestamp, relevantV1) + if err != nil { + return nil, err + } + + for i := range events { + events[i].Relevant = []types.Address{address} + } + + relevantV2Txn := func(txn types.V2Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.Parent.SiacoinOutput.Address == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.Parent.SiafundOutput.Address == address { + return true + } + } + return false + } + + // Annotate v2 transactions. + for _, txn := range v2 { + if !relevantV2Txn(txn) { + continue + } + + events = append(events, Event{ + ID: types.Hash256(txn.ID()), + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height, + Type: EventTypeV2Transaction, + Data: EventV2Transaction(txn), + Relevant: []types.Address{address}, + }) + } + return events, nil +} diff --git a/wallet/events.go b/wallet/events.go new file mode 100644 index 0000000..f9f7141 --- /dev/null +++ b/wallet/events.go @@ -0,0 +1,79 @@ +package wallet + +import ( + "time" + + "go.sia.tech/core/types" +) + +// event types indicate the source of an event. Events can +// either be created by sending Siacoins between addresses or they can be +// created by consensus (e.g. a miner payout, a siafund claim, or a contract). +const ( + EventTypeMinerPayout = "miner" + EventTypeFoundationSubsidy = "foundation" + + EventTypeV1Transaction = "v1Transaction" + EventTypeV1ContractResolution = "v1ContractResolution" + + EventTypeV2Transaction = "v2Transaction" + EventTypeV2ContractResolution = "v2ContractResolution" + + EventTypeSiafundClaim = "siafundClaim" +) + +type ( + EventData interface { + isEvent() bool + } + + // An Event is something interesting that happened on the Sia blockchain. + Event struct { + ID types.Hash256 `json:"id"` + Index types.ChainIndex `json:"index"` + Timestamp time.Time `json:"timestamp"` + MaturityHeight uint64 `json:"maturityHeight"` + Type string `json:"type"` + Data EventData `json:"data"` + Relevant []types.Address `json:"relevant,omitempty"` + } + + EventV1Transaction struct { + Transaction types.Transaction `json:"transaction"` + // v1 siacoin inputs do not describe the value of the spent utxo + SpentSiacoinElements []types.SiacoinElement `json:"spentSiacoinElements"` + // v1 siafund inputs do not describe the value of the spent utxo + SpentSiafundElements []types.SiafundElement `json:"spentSiafundElements"` + } + + EventV2Transaction types.V2Transaction + + // An EventPayout represents a payout from a siafund claim, a miner, or the + // foundation subsidy. + EventPayout struct { + SiacoinElement types.SiacoinElement `json:"siacoinElement"` + } + + // An EventV1ContractResolution represents a file contract payout from a v1 + // contract. + EventV1ContractResolution struct { + FileContract types.FileContractElement `json:"fileContract"` + SiacoinElement types.SiacoinElement `json:"siacoinElement"` + Missed bool `json:"missed"` + } + + // An EventV2ContractResolution represents a file contract payout from a v2 + // contract. + EventV2ContractResolution struct { + FileContract types.V2FileContractElement `json:"fileContract"` + Resolution types.V2FileContractResolutionType `json:"resolution"` + SiacoinElement types.SiacoinElement `json:"siacoinElement"` + Missed bool `json:"missed"` + } +) + +func (EventPayout) isEvent() bool { return true } +func (EventV1ContractResolution) isEvent() bool { return true } +func (EventV2ContractResolution) isEvent() bool { return true } +func (EventV1Transaction) isEvent() bool { return true } +func (EventV2Transaction) isEvent() bool { return true } diff --git a/wallet/manager.go b/wallet/manager.go index 7adb6f4..7355bc8 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -41,6 +41,9 @@ type ( // A ChainManager manages the consensus state ChainManager interface { + PoolTransactions() []types.Transaction + V2PoolTransactions() []types.V2Transaction + Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) @@ -52,6 +55,7 @@ type ( Store interface { UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error + WalletUnconfirmedEvents(id ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []Event, err error) WalletEvents(walletID ID, offset, limit int) ([]Event, error) AddWallet(Wallet) (Wallet, error) UpdateWallet(Wallet) (Wallet, error) @@ -65,14 +69,13 @@ type ( AddWalletAddress(walletID ID, address Address) error RemoveWalletAddress(walletID ID, address types.Address) error - Annotate(walletID ID, txns []types.Transaction) ([]PoolTransaction, error) - AddressBalance(address types.Address) (balance Balance, err error) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) Events(eventIDs []types.Hash256) ([]Event, error) + AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) SetIndexMode(IndexMode) error LastCommittedIndex() (types.ChainIndex, error) @@ -183,9 +186,12 @@ func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types return m.store.WalletSiafundOutputs(walletID, offset, limit) } -// Annotate annotates the given transactions with the wallet they belong to. -func (m *Manager) Annotate(walletID ID, pool []types.Transaction) ([]PoolTransaction, error) { - return m.store.Annotate(walletID, pool) +// WalletUnconfirmedEvents returns the unconfirmed events of the given wallet. +func (m *Manager) WalletUnconfirmedEvents(walletID ID) ([]Event, error) { + index := m.chain.Tip() + index.Height++ + index.ID = types.BlockID{} + return m.store.WalletUnconfirmedEvents(walletID, index, time.Now(), m.chain.PoolTransactions(), m.chain.V2PoolTransactions()) } // WalletBalance returns the balance of the given wallet. @@ -198,6 +204,32 @@ func (m *Manager) Events(eventIDs []types.Hash256) ([]Event, error) { return m.store.Events(eventIDs) } +// UnconfirmedEvents returns all unconfirmed events in the transaction pool. +func (m *Manager) UnconfirmedEvents() ([]Event, error) { + v1, v2 := m.chain.PoolTransactions(), m.chain.V2PoolTransactions() + + unconfirmedIndex := m.chain.Tip() + unconfirmedIndex.Height++ + unconfirmedIndex.ID = types.BlockID{} + timestamp := time.Now() + + events, err := m.store.AnnotateV1Events(unconfirmedIndex, timestamp, v1) + if err != nil { + return nil, err + } + + for _, txn := range v2 { + events = append(events, Event{ + ID: types.Hash256(txn.ID()), + Index: unconfirmedIndex, + Timestamp: timestamp, + Type: EventTypeV2Transaction, + Data: EventV2Transaction(txn), + }) + } + return events, nil +} + // Reserve reserves the given ids for the given duration. func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { m.mu.Lock() diff --git a/wallet/wallet.go b/wallet/wallet.go index 5ed34c7..f4fc8e9 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -10,15 +10,6 @@ import ( "go.sia.tech/core/types" ) -// event type constants -const ( - EventTypeTransaction = "transaction" - EventTypeMinerPayout = "miner payout" - EventTypeContractPayout = "contract payout" - EventTypeSiafundClaim = "siafund claim" - EventTypeFoundationSubsidy = "foundation subsidy" -) - type ( // Balance is a summary of a siacoin and siafund balance Balance struct { @@ -47,6 +38,14 @@ type ( SpendPolicy *types.SpendPolicy `json:"spendPolicy,omitempty"` Metadata json.RawMessage `json:"metadata"` } + + // A ChainUpdate is a set of changes to the consensus state. + ChainUpdate interface { + ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) + ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) + ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) + ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) + } ) // ErrNotFound is returned when a requested wallet or address is not found. @@ -92,193 +91,9 @@ func SignTransaction(cs consensus.State, txn *types.Transaction, sigIndex int, k tsig.Signature = sig[:] } -// A PoolTransaction summarizes the wallet-relevant data in a txpool -// transaction. -type PoolTransaction struct { - ID types.TransactionID `json:"id"` - Raw types.Transaction `json:"raw"` - Type string `json:"type"` - Sent types.Currency `json:"sent"` - Received types.Currency `json:"received"` - Locked types.Currency `json:"locked"` -} - -// Annotate annotates a txpool transaction. -func Annotate(txn types.Transaction, ownsAddress func(types.Address) bool) PoolTransaction { - ptxn := PoolTransaction{ID: txn.ID(), Raw: txn, Type: "unknown"} - - var totalValue types.Currency - for _, sco := range txn.SiacoinOutputs { - totalValue = totalValue.Add(sco.Value) - } - for _, fc := range txn.FileContracts { - totalValue = totalValue.Add(fc.Payout) - } - for _, fee := range txn.MinerFees { - totalValue = totalValue.Add(fee) - } - - var ownedIn, ownedOut int - for _, sci := range txn.SiacoinInputs { - if ownsAddress(sci.UnlockConditions.UnlockHash()) { - ownedIn++ - } - } - for _, sco := range txn.SiacoinOutputs { - if ownsAddress(sco.Address) { - ownedOut++ - } - } - var ins, outs string - switch { - case ownedIn == 0: - ins = "none" - case ownedIn < len(txn.SiacoinInputs): - ins = "some" - case ownedIn == len(txn.SiacoinInputs): - ins = "all" - } - switch { - case ownedOut == 0: - outs = "none" - case ownedOut < len(txn.SiacoinOutputs): - outs = "some" - case ownedOut == len(txn.SiacoinOutputs): - outs = "all" - } - - switch { - case ins == "none" && outs == "none": - ptxn.Type = "unrelated" - case ins == "all": - ptxn.Sent = totalValue - switch { - case outs == "all": - ptxn.Type = "redistribution" - case len(txn.FileContractRevisions) > 0: - ptxn.Type = "contract revision" - case len(txn.StorageProofs) > 0: - ptxn.Type = "storage proof" - case len(txn.ArbitraryData) > 0: - ptxn.Type = "announcement" - default: - ptxn.Type = "send" - } - case ins == "none" && outs != "none": - ptxn.Type = "receive" - for _, sco := range txn.SiacoinOutputs { - if ownsAddress(sco.Address) { - ptxn.Received = ptxn.Received.Add(sco.Value) - } - } - case ins == "some" && len(txn.FileContracts) > 0: - ptxn.Type = "contract" - for _, fc := range txn.FileContracts { - var validLocked, missedLocked types.Currency - for _, sco := range fc.ValidProofOutputs { - if ownsAddress(sco.Address) { - validLocked = validLocked.Add(fc.Payout) - } - } - for _, sco := range fc.MissedProofOutputs { - if ownsAddress(sco.Address) { - missedLocked = missedLocked.Add(fc.Payout) - } - } - if validLocked.Cmp(missedLocked) > 0 { - ptxn.Locked = ptxn.Locked.Add(validLocked) - } else { - ptxn.Locked = ptxn.Locked.Add(missedLocked) - } - } - } - - return ptxn -} - -// An Event is something interesting that happened on the Sia blockchain. -type Event struct { - ID types.Hash256 `json:"id"` - Index types.ChainIndex `json:"index"` - Timestamp time.Time `json:"timestamp"` - MaturityHeight uint64 `json:"maturityHeight"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Data any `json:"data"` -} - -// A HostAnnouncement represents a host announcement within an EventTransaction. -type HostAnnouncement struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` -} - -// A SiafundInput represents a siafund input within an EventTransaction. -type SiafundInput struct { - SiafundElement types.SiafundElement `json:"siafundElement"` - ClaimElement types.SiacoinElement `json:"claimElement"` -} - -// A FileContract represents a file contract within an EventTransaction. -type FileContract struct { - FileContract types.FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - ValidOutputs []types.SiacoinElement `json:"validOutputs,omitempty"` -} - -// A V2FileContract represents a v2 file contract within an EventTransaction. -type V2FileContract struct { - FileContract types.V2FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.V2FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - Resolution types.V2FileContractResolutionType `json:"resolution,omitempty"` - Outputs []types.SiacoinElement `json:"outputs,omitempty"` -} - -// An EventTransaction represents a transaction that affects the wallet. -type EventTransaction struct { - SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` - SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` - SiafundInputs []SiafundInput `json:"siafundInputs"` - SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` - FileContracts []FileContract `json:"fileContracts"` - V2FileContracts []V2FileContract `json:"v2FileContracts"` - HostAnnouncements []HostAnnouncement `json:"hostAnnouncements"` - Fee types.Currency `json:"fee"` -} - -// An EventMinerPayout represents a miner payout from a block. -type EventMinerPayout struct { - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` -} - -// EventFoundationSubsidy represents a foundation subsidy from a block. -type EventFoundationSubsidy struct { - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` -} - -// An EventContractPayout represents a file contract payout -type EventContractPayout struct { - FileContract types.FileContractElement `json:"fileContract"` - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` - Missed bool `json:"missed"` -} - -// A ChainUpdate is a set of changes to the consensus state. -type ChainUpdate interface { - ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) - ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) -} - // AppliedEvents extracts a list of relevant events from a chain update. -func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { - var events []Event - addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v any, relevant []types.Address) { +func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) (events []Event) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v EventData, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] @@ -339,197 +154,115 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f v2fces[types.FileContractID(fce.ID)] = fce }) - relevantTxn := func(txn types.Transaction) (addrs []types.Address) { + // handle v1 transactions + for _, txn := range b.Transactions { + addresses := make(map[types.Address]bool) + e := &EventV1Transaction{ + Transaction: txn, + SpentSiacoinElements: make([]types.SiacoinElement, 0, len(txn.SiacoinInputs)), + SpentSiafundElements: make([]types.SiafundElement, 0, len(txn.SiafundInputs)), + } + for _, sci := range txn.SiacoinInputs { - if sce := sces[sci.ParentID]; relevant(sce.SiacoinOutput.Address) { - addrs = append(addrs, sce.SiacoinOutput.Address) + sce, ok := sces[sci.ParentID] + if !ok { + continue + } + + e.SpentSiacoinElements = append(e.SpentSiacoinElements, sce) + if relevant(sce.SiacoinOutput.Address) { + addresses[sce.SiacoinOutput.Address] = true } } for _, sco := range txn.SiacoinOutputs { if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + addresses[sco.Address] = true } } + for _, sfi := range txn.SiafundInputs { - if sfe := sfes[sfi.ParentID]; relevant(sfe.SiafundOutput.Address) { - addrs = append(addrs, sfe.SiafundOutput.Address) - } - } - for _, sfo := range txn.SiafundOutputs { - if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) + sfe, ok := sfes[sfi.ParentID] + if !ok { + continue } - } - return - } - relevantV2Txn := func(txn types.V2Transaction) (addrs []types.Address) { - for _, sci := range txn.SiacoinInputs { - if relevant(sci.Parent.SiacoinOutput.Address) { - addrs = append(addrs, sci.Parent.SiacoinOutput.Address) - } - } - for _, sco := range txn.SiacoinOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + e.SpentSiafundElements = append(e.SpentSiafundElements, sfe) + if relevant(sfe.SiafundOutput.Address) { + addresses[sfe.SiafundOutput.Address] = true } - } - for _, sfi := range txn.SiafundInputs { - if relevant(sfi.Parent.SiafundOutput.Address) { - addrs = append(addrs, sfi.Parent.SiafundOutput.Address) + + outputID := sfi.ParentID.ClaimOutputID() + if sfo, ok := sces[outputID]; ok && relevant(sfi.ClaimAddress) { + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeSiafundClaim, EventPayout{ + SiacoinElement: sfo, + }, []types.Address{sfi.ClaimAddress}) } } for _, sfo := range txn.SiafundOutputs { if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) + addresses[sfo.Address] = true } } - return - } - // handle v1 transactions - for _, txn := range b.Transactions { - relevant := relevantTxn(txn) - if len(relevant) == 0 { + // skip transactions with no relevant addresses + if len(addresses) == 0 { continue } - e := &EventTransaction{ - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), - } - - for i := range txn.SiacoinInputs { - e.SiacoinInputs[i] = sces[txn.SiacoinInputs[i].ParentID] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(i)] - } - for i := range txn.SiafundInputs { - e.SiafundInputs[i] = SiafundInput{ - SiafundElement: sfes[txn.SiafundInputs[i].ParentID], - ClaimElement: sces[txn.SiafundClaimOutputID(i)], - } - } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(i)] - } - addContract := func(id types.FileContractID) *FileContract { - for i := range e.FileContracts { - if types.FileContractID(e.FileContracts[i].FileContract.ID) == id { - return &e.FileContracts[i] - } - } - e.FileContracts = append(e.FileContracts, FileContract{FileContract: fces[id]}) - return &e.FileContracts[len(e.FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.FileContractID(i)) - } - for i := range txn.FileContractRevisions { - fc := addContract(txn.FileContractRevisions[i].ParentID) - rev := txn.FileContractRevisions[i].FileContract - fc.Revision = &rev - } - for i := range txn.StorageProofs { - fc := addContract(txn.StorageProofs[i].ParentID) - fc.ValidOutputs = make([]types.SiacoinElement, len(fc.FileContract.FileContract.ValidProofOutputs)) - for i := range fc.ValidOutputs { - fc.ValidOutputs[i] = sces[types.FileContractID(fc.FileContract.ID).ValidOutputID(i)] - } - } - for _, arb := range txn.ArbitraryData { - var prefix types.Specifier - var uk types.UnlockKey - d := types.NewBufDecoder(arb) - prefix.DecodeFrom(d) - netAddress := d.ReadString() - uk.DecodeFrom(d) - if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") && - uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: *(*types.PublicKey)(uk.Key), - NetAddress: netAddress, - }) - } - } - for i := range txn.MinerFees { - e.Fee = e.Fee.Add(txn.MinerFees[i]) + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) } - addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeTransaction, e, relevant) // transaction maturity height is the current block height + addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeV1Transaction, e, relevant) // transaction maturity height is the current block height } // handle v2 transactions for _, txn := range b.V2Transactions() { - relevant := relevantV2Txn(txn) - if len(relevant) == 0 { - continue - } - - txid := txn.ID() - e := &EventTransaction{ - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), - } - for i := range txn.SiacoinInputs { - // NOTE: here (and elsewhere), we fetch the element from our maps, - // rather than using the parent directly, because our copy has its - // Merkle proof nil'd out - e.SiacoinInputs[i] = sces[types.SiacoinOutputID(txn.SiacoinInputs[i].Parent.ID)] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(txid, i)] - } - for i := range txn.SiafundInputs { - sfoid := types.SiafundOutputID(txn.SiafundInputs[i].Parent.ID) - e.SiafundInputs[i] = SiafundInput{ - SiafundElement: sfes[sfoid], - ClaimElement: sces[sfoid.ClaimOutputID()], + addresses := make(map[types.Address]bool) + for _, sci := range txn.SiacoinInputs { + if !relevant(sci.Parent.SiacoinOutput.Address) { + continue } + addresses[sci.Parent.SiacoinOutput.Address] = true } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(txid, i)] - } - addContract := func(id types.FileContractID) *V2FileContract { - for i := range e.V2FileContracts { - if types.FileContractID(e.V2FileContracts[i].FileContract.ID) == id { - return &e.V2FileContracts[i] - } + for _, sco := range txn.SiacoinOutputs { + if !relevant(sco.Address) { + continue } - e.V2FileContracts = append(e.V2FileContracts, V2FileContract{FileContract: v2fces[id]}) - return &e.V2FileContracts[len(e.V2FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.V2FileContractID(txid, i)) + addresses[sco.Address] = true } - for _, fcr := range txn.FileContractRevisions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Revision = &fcr.Revision - } - for _, fcr := range txn.FileContractResolutions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Resolution = fcr.Resolution - fc.Outputs = []types.SiacoinElement{ - sces[types.FileContractID(fcr.Parent.ID).V2RenterOutputID()], - sces[types.FileContractID(fcr.Parent.ID).V2HostOutputID()], + for _, sfi := range txn.SiafundInputs { + if !relevant(sfi.Parent.SiafundOutput.Address) { + continue + } + addresses[sfi.Parent.SiafundOutput.Address] = true + + outputID := types.SiafundOutputID(sfi.Parent.ID).ClaimOutputID() + if sfo, ok := sces[outputID]; ok && relevant(sfi.ClaimAddress) { + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeSiafundClaim, EventPayout{ + SiacoinElement: sfo, + }, []types.Address{sfi.ClaimAddress}) } } - for _, a := range txn.Attestations { - if a.Key == "HostAnnouncement" { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: a.PublicKey, - NetAddress: string(a.Value), - }) + for _, sco := range txn.SiafundOutputs { + if !relevant(sco.Address) { + continue } + addresses[sco.Address] = true } - e.Fee = txn.MinerFee - addEvent(types.Hash256(txid), cs.Index.Height, EventTypeTransaction, e, relevant) // transaction maturity height is the current block height + // skip transactions with no relevant addresses + if len(addresses) == 0 { + continue + } + + ev := EventV2Transaction(txn) + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) + } + addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeV2Transaction, ev, relevant) // transaction maturity height is the current block height } // handle missed contracts @@ -540,39 +273,72 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if valid { for i := range fce.FileContract.ValidProofOutputs { - if !relevant(fce.FileContract.ValidProofOutputs[i].Address) { + address := fce.FileContract.ValidProofOutputs[i].Address + if !relevant(address) { continue } outputID := types.FileContractID(fce.ID).ValidOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeContractPayout, &EventContractPayout{ - FileContract: fce, - SiacoinOutput: sces[outputID], - Missed: false, - }, []types.Address{fce.FileContract.ValidProofOutputs[i].Address}) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ + FileContract: fce, + SiacoinElement: sces[outputID], + Missed: false, + }, []types.Address{address}) } } else { for i := range fce.FileContract.MissedProofOutputs { - if !relevant(fce.FileContract.MissedProofOutputs[i].Address) { + address := fce.FileContract.MissedProofOutputs[i].Address + if !relevant(address) { continue } outputID := types.FileContractID(fce.ID).MissedOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeContractPayout, &EventContractPayout{ - FileContract: fce, - SiacoinOutput: sces[outputID], - Missed: true, - }, []types.Address{fce.FileContract.MissedProofOutputs[i].Address}) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ + FileContract: fce, + SiacoinElement: sces[outputID], + Missed: true, + }, []types.Address{address}) } } }) + cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + if res == nil { + return + } + + var missed bool + if _, ok := res.(*types.V2FileContractExpiration); ok { + missed = true + } + + if relevant(fce.V2FileContract.HostOutput.Address) { + outputID := types.FileContractID(fce.ID).V2HostOutputID() + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ + FileContract: fce, + Resolution: res, + SiacoinElement: sces[outputID], + Missed: missed, + }, []types.Address{fce.V2FileContract.HostOutput.Address}) + } + + if relevant(fce.V2FileContract.RenterOutput.Address) { + outputID := types.FileContractID(fce.ID).V2RenterOutputID() + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ + FileContract: fce, + Resolution: res, + SiacoinElement: sces[outputID], + Missed: missed, + }, []types.Address{fce.V2FileContract.RenterOutput.Address}) + } + }) + // handle block rewards for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { outputID := cs.Index.ID.MinerOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeMinerPayout, &EventMinerPayout{ - SiacoinOutput: sces[outputID], + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeMinerPayout, EventPayout{ + SiacoinElement: sces[outputID], }, []types.Address{b.MinerPayouts[i].Address}) } } @@ -582,9 +348,9 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f outputID := cs.Index.ID.FoundationOutputID() sce, ok := sces[outputID] if ok { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeFoundationSubsidy, &EventFoundationSubsidy{ - SiacoinOutput: sce, - }, []types.Address{cs.FoundationPrimaryAddress}) + addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeFoundationSubsidy, EventPayout{ + SiacoinElement: sce, + }, []types.Address{sce.SiacoinOutput.Address}) } } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 25ae2e7..fa670c9 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -472,9 +472,9 @@ func TestEphemeralBalance(t *testing.T) { t.Fatalf("expected 3 events, got %v", len(events)) } else if events[2].Type != wallet.EventTypeMinerPayout { t.Fatalf("expected miner payout event, got %v", events[2].Type) - } else if events[1].Type != wallet.EventTypeTransaction { + } else if events[1].Type != wallet.EventTypeV1Transaction { t.Fatalf("expected transaction event, got %v", events[1].Type) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV1Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) @@ -906,6 +906,19 @@ func TestSiafunds(t *testing.T) { t.Fatal(err) } + // check that the transaction made it into the pool + if events, err := wm.WalletUnconfirmedEvents(w1.ID); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected %v, got %v", addr1, events[0].Relevant[0]) + } + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { t.Fatal("failed to mine block") } else if err := cm.AddBlocks([]types.Block{b}); err != nil { @@ -1212,7 +1225,7 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV1Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1293,7 +1306,7 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1302,7 +1315,7 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 event, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1344,7 +1357,7 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1353,7 +1366,7 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } } @@ -1423,7 +1436,7 @@ func TestEvents(t *testing.T) { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 event, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV1Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1508,7 +1521,7 @@ func TestEvents(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { t.Fatalf("expected to get event: %v", err) @@ -1521,7 +1534,7 @@ func TestEvents(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 event, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { t.Fatalf("expected to get event: %v", err) @@ -1567,7 +1580,7 @@ func TestEvents(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { t.Fatalf("expected to get event: %v", err) @@ -1580,7 +1593,7 @@ func TestEvents(t *testing.T) { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { t.Fatalf("expected to get event: %v", err) @@ -1589,6 +1602,438 @@ func TestEvents(t *testing.T) { } } +func TestWalletUnconfirmedEvents(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + // mine a block sending the payout to the wallet + b, ok := coreutils.MineBlock(cm, addr1, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // mine until the payout matures + maturityHeight := cm.TipState().MaturityHeight() + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + utxos, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + // generate a second address to send the payout to + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a transaction that splits the payout + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr1, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: utxos[0].ID, + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event was recorded + events, err := wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant address, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } + + txnData := events[0].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != utxos[0].ID { + t.Fatalf("expected siacoin output %v, got %v", utxos[0].ID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != utxos[0].SiacoinOutput.Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // add the second address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event's relevant addresses were updated + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if len(events[0].Relevant) != 2 { + t.Fatalf("expected 2 relevant addresses, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } else if events[0].Relevant[1] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[0].Relevant[1]) + } + + // spend the ephemeral output + ephemeralOutputID := txn.SiacoinOutputID(0) + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: ephemeralOutputID, + UnlockConditions: types.StandardUnlockConditions(pk2.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: txn.SiacoinOutputs[0].Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(ephemeralOutputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash = cm.TipState().WholeSigHash(txn2, txn2.Signatures[0].ParentID, 0, 0, nil) + sig2 := pk2.SignHash(sigHash) + txn2.Signatures[0].Signature = sig2[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn, txn2}); err != nil { + t.Fatal(err) + } + + // check that the new unconfirmed event was recorded + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[1].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[1].Relevant) != 1 { // second event is only relevant to the second address + t.Fatalf("expected 1 relevant addresses, got %v", len(events[1].Relevant)) + } + + txnData = events[1].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != types.Hash256(ephemeralOutputID) { + t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // mine the transactions + b, ok = coreutils.MineBlock(cm, types.VoidAddress, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed events were removed + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } +} + +func TestAddressUnconfirmedEvents(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + // mine a block sending the payout to the wallet + b, ok := coreutils.MineBlock(cm, addr1, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // mine until the payout matures + maturityHeight := cm.TipState().MaturityHeight() + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + utxos, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + // generate a second address to send the payout to + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a transaction that splits the payout + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr1, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: utxos[0].ID, + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event was recorded + events, err := wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant address, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } + + txnData := events[0].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != utxos[0].ID { + t.Fatalf("expected siacoin output %v, got %v", utxos[0].ID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != utxos[0].SiacoinOutput.Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // add the second address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // check that the address now shows an unconfirmed event + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant addresses, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[0].Relevant[1]) + } + + // spend the ephemeral output + ephemeralOutputID := txn.SiacoinOutputID(0) + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: ephemeralOutputID, + UnlockConditions: types.StandardUnlockConditions(pk2.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: txn.SiacoinOutputs[0].Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(ephemeralOutputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + + sigHash = cm.TipState().WholeSigHash(txn2, txn2.Signatures[0].ParentID, 0, 0, nil) + sig2 := pk2.SignHash(sigHash) + txn2.Signatures[0].Signature = sig2[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn, txn2}); err != nil { + t.Fatal(err) + } + + // check that the first address still shows only one unconfirmed event + events, err = wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } + + // check that the second address now shows two unconfirmed events + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[1].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[1].Relevant) != 1 { // second event is only relevant to the second address + t.Fatalf("expected 1 relevant addresses, got %v", len(events[1].Relevant)) + } else if events[1].Relevant[0] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[1].Relevant[0]) + } + + txnData = events[1].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != types.Hash256(ephemeralOutputID) { + t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // mine the transactions + b, ok = coreutils.MineBlock(cm, types.VoidAddress, time.Minute) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed events were removed + events, err = wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } + + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } +} + func TestV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -1703,7 +2148,7 @@ func TestV2(t *testing.T) { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeTransaction { + } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events[0].Relevant[0] != addr { t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) From ccf952cb864e4c823d29cdebc0badce7db4acbcd Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 13 Jun 2024 17:15:28 -0700 Subject: [PATCH 201/630] api: add client methods, uncomment test --- api/api_test.go | 6 +++--- api/client.go | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 584eb36..50862ce 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -371,12 +371,12 @@ func TestWallet(t *testing.T) { t.Fatal("event history should be empty") } - /*tpool, err := wc.PoolTransactions() + unconfirmed, err := wc.UnconfirmedEvents() if err != nil { t.Fatal(err) - } else if len(tpool) != 1 { + } else if len(unconfirmed) != 1 { t.Fatal("txpool should have one transaction") - }*/ + } cs := cm.TipState() b := types.Block{ diff --git a/api/client.go b/api/client.go index ba87ded..6000dc2 100644 --- a/api/client.go +++ b/api/client.go @@ -140,6 +140,12 @@ func (c *Client) AddressEvents(addr types.Address, offset, limit int) (resp []wa return } +// AddressUnconfirmedEvents returns the unconfirmed events for a single address. +func (c *Client) AddressUnconfirmedEvents(addr types.Address) (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/events/unconfirmed", addr), &resp) + return +} + // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) (resp []types.SiacoinElement, err error) { err = c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) @@ -152,6 +158,12 @@ func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) (r return } +// Event returns the event with the specified ID. +func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/events/%v", id), &resp) + return +} + // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { @@ -190,6 +202,12 @@ func (c *WalletClient) Events(offset, limit int) (resp []wallet.Event, err error return } +// UnconfirmedEvents returns all unconfirmed events relevant to the wallet. +func (c *WalletClient) UnconfirmedEvents() (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/events/unconfirmed", c.id), &resp) + return +} + // SiacoinOutputs returns the set of unspent outputs controlled by the wallet. func (c *WalletClient) SiacoinOutputs(offset, limit int) (sc []types.SiacoinElement, err error) { err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &sc) From 511b5cab7e2d1ca2c0f7ae14b1ff11a5285b6f0c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 13 Jun 2024 17:15:37 -0700 Subject: [PATCH 202/630] wallet: fix lint --- wallet/events.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wallet/events.go b/wallet/events.go index f9f7141..b6e415a 100644 --- a/wallet/events.go +++ b/wallet/events.go @@ -23,6 +23,7 @@ const ( ) type ( + // EventData provides type safety for the Data field of an Event. EventData interface { isEvent() bool } @@ -38,6 +39,8 @@ type ( Relevant []types.Address `json:"relevant,omitempty"` } + // An EventV1Transaction pairs a v1 transaction with its spent siacoin and + // siafund elements. EventV1Transaction struct { Transaction types.Transaction `json:"transaction"` // v1 siacoin inputs do not describe the value of the spent utxo @@ -46,6 +49,7 @@ type ( SpentSiafundElements []types.SiafundElement `json:"spentSiafundElements"` } + // An EventV2Transaction is a v2 transaction. EventV2Transaction types.V2Transaction // An EventPayout represents a payout from a siafund claim, a miner, or the From 6db022876fd5cf01c0215e8e1a457881e5e33d5a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 13 Jun 2024 19:15:07 -0700 Subject: [PATCH 203/630] wallet: fix api client marshalling --- wallet/events.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/wallet/events.go b/wallet/events.go index b6e415a..b305fe5 100644 --- a/wallet/events.go +++ b/wallet/events.go @@ -1,6 +1,8 @@ package wallet import ( + "encoding/json" + "fmt" "time" "go.sia.tech/core/types" @@ -81,3 +83,53 @@ func (EventV1ContractResolution) isEvent() bool { return true } func (EventV2ContractResolution) isEvent() bool { return true } func (EventV1Transaction) isEvent() bool { return true } func (EventV2Transaction) isEvent() bool { return true } + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (e *Event) UnmarshalJSON(b []byte) error { + var je struct { + ID types.Hash256 `json:"id"` + Index types.ChainIndex `json:"index"` + Timestamp time.Time `json:"timestamp"` + MaturityHeight uint64 `json:"maturityHeight"` + Type string `json:"type"` + Data json.RawMessage `json:"data"` + Relevant []types.Address `json:"relevant,omitempty"` + } + if err := json.Unmarshal(b, &je); err != nil { + return err + } + + e.ID = je.ID + e.Index = je.Index + e.Timestamp = je.Timestamp + e.MaturityHeight = je.MaturityHeight + e.Type = je.Type + e.Relevant = je.Relevant + + var err error + switch je.Type { + case EventTypeMinerPayout, EventTypeFoundationSubsidy, EventTypeSiafundClaim: + var data EventPayout + err = json.Unmarshal(je.Data, &data) + e.Data = data + case EventTypeV1ContractResolution: + var data EventV1ContractResolution + err = json.Unmarshal(je.Data, &data) + e.Data = data + case EventTypeV2ContractResolution: + var data EventV2ContractResolution + err = json.Unmarshal(je.Data, &data) + e.Data = data + case EventTypeV1Transaction: + var data EventV1Transaction + err = json.Unmarshal(je.Data, &data) + e.Data = data + case EventTypeV2Transaction: + var data EventV2Transaction + err = json.Unmarshal(je.Data, &data) + e.Data = data + default: + return fmt.Errorf("unknown event type: %v", je.Type) + } + return err +} From 24635f44a34e990650fb9d9537e41c4b97fde45d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:01:56 +0000 Subject: [PATCH 204/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.2.6 to 0.2.7 - [Commits](https://github.com/SiaFoundation/core/compare/v0.2.6...v0.2.7) Updates `go.sia.tech/coreutils` from 0.0.5 to 0.0.6 - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.0.5...v0.0.6) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bff3e0b..df2638a 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.6 - go.sia.tech/coreutils v0.0.5 + go.sia.tech/core v0.2.7 + go.sia.tech/coreutils v0.0.6 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.21.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 12e50b3..16492a2 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.2.6 h1:JrbZwW4cPHCB2Q6TeqCsAj2GAFXrRqLq0q/GNevtnPU= -go.sia.tech/core v0.2.6/go.mod h1:B7ooFH3F6cLjxQz6IX33kgjOY392Ava7pzpPeccagac= -go.sia.tech/coreutils v0.0.5 h1:Jj03VrqAayYHgA9fwV13+X88WB+Wr1p8wuLw2B8d2FI= -go.sia.tech/coreutils v0.0.5/go.mod h1:SkSpHeq3tBh2ff4HXuBk2WtlhkYQQtdcvU4Yv1Rd2bU= +go.sia.tech/core v0.2.7 h1:9Q/3BHL6ziAMPeiko863hhTD/Zs2s7OqEUiPKouDny8= +go.sia.tech/core v0.2.7/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/coreutils v0.0.6 h1:xLpv3JyvbOoXcX3gC6a6Y3zQ/MZn/fyFvuPIzv/e/Eg= +go.sia.tech/coreutils v0.0.6/go.mod h1:EiDcQk2qLuP32Qoj/XphY9fbjTXphJhJZMERoC4LF0c= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 6adc48159d64ec118f20d2e5af48a23ce6962fa1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 17 Jun 2024 14:09:39 -0700 Subject: [PATCH 205/630] sqlite: fix slice encoding --- persist/sqlite/consensus.go | 10 +++---- persist/sqlite/encoding.go | 52 +++++++------------------------------ persist/sqlite/wallet.go | 4 +-- 3 files changed, 16 insertions(+), 50 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 8701c6f..048d03b 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -66,7 +66,7 @@ func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) er for _, se := range elements { var dummy types.Hash256 - err := stmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) + err := stmt.QueryRow(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -112,7 +112,7 @@ func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) er for _, se := range elements { var dummy types.Hash256 - err := stmt.QueryRow(encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) + err := stmt.QueryRow(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -320,7 +320,7 @@ func (s *Store) SetIndexMode(mode wallet.IndexMode) error { } func scanStateElement(s scanner) (se types.StateElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof)) + err = s.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)) return } @@ -534,7 +534,7 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, se.MerkleProof = nil } - _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encodeSlice(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -808,7 +808,7 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, se.MerkleProof = nil } - _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encodeSlice(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encode(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } else if exists { diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 966c6b0..abe5ab9 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -19,6 +19,12 @@ func encode(obj any) any { binary.BigEndian.PutUint64(buf, obj.Hi) binary.BigEndian.PutUint64(buf[8:], obj.Lo) return buf + case []types.Hash256: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + types.EncodeSlice(e, obj) + e.Flush() + return buf.Bytes() case types.EncoderTo: var buf bytes.Buffer e := types.NewEncoder(&buf) @@ -61,6 +67,9 @@ func (d *decodable) Scan(src any) error { return dec.Err() case *uint64: *v = binary.LittleEndian.Uint64(src) + case *[]types.Hash256: + dec := types.NewBufDecoder(src) + types.DecodeSlice(dec, v) default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } @@ -83,46 +92,3 @@ func (d *decodable) Scan(src any) error { func decode(obj any) sql.Scanner { return &decodable{obj} } - -type decodableSlice[T any] struct { - v *[]T -} - -func (d *decodableSlice[T]) Scan(src any) error { - switch src := src.(type) { - case []byte: - dec := types.NewBufDecoder(src) - s := make([]T, dec.ReadPrefix()) - for i := range s { - dv, ok := any(&s[i]).(types.DecoderFrom) - if !ok { - panic(fmt.Errorf("cannot decode %T", s[i])) - } - dv.DecodeFrom(dec) - } - if err := dec.Err(); err != nil { - return err - } - *d.v = s - return nil - default: - return fmt.Errorf("cannot scan %T to []byte", src) - } -} - -func decodeSlice[T any](v *[]T) sql.Scanner { - return &decodableSlice[T]{v: v} -} - -func encodeSlice[T types.EncoderTo](v []T) []byte { - var buf bytes.Buffer - enc := types.NewEncoder(&buf) - enc.WritePrefix(len(v)) - for _, e := range v { - e.EncodeTo(enc) - } - if err := enc.Flush(); err != nil { - panic(err) - } - return buf.Bytes() -} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index d596578..e78045f 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -578,12 +578,12 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti } func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { - err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decodeSlice(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) return } func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decodeSlice(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + err = s.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) return } From 3ccd8213c1476b0ec6222d46d1023ecd018c5d9c Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 18 Jun 2024 09:30:55 -0700 Subject: [PATCH 206/630] wallet: change resolution event to use V2FileContractResolution for custom resolution type encoding --- wallet/events.go | 9 ++++----- wallet/wallet.go | 16 ++++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/wallet/events.go b/wallet/events.go index b305fe5..3151a0e 100644 --- a/wallet/events.go +++ b/wallet/events.go @@ -63,7 +63,7 @@ type ( // An EventV1ContractResolution represents a file contract payout from a v1 // contract. EventV1ContractResolution struct { - FileContract types.FileContractElement `json:"fileContract"` + Parent types.FileContractElement `json:"parent"` SiacoinElement types.SiacoinElement `json:"siacoinElement"` Missed bool `json:"missed"` } @@ -71,10 +71,9 @@ type ( // An EventV2ContractResolution represents a file contract payout from a v2 // contract. EventV2ContractResolution struct { - FileContract types.V2FileContractElement `json:"fileContract"` - Resolution types.V2FileContractResolutionType `json:"resolution"` - SiacoinElement types.SiacoinElement `json:"siacoinElement"` - Missed bool `json:"missed"` + types.V2FileContractResolution + SiacoinElement types.SiacoinElement `json:"siacoinElement"` + Missed bool `json:"missed"` } ) diff --git a/wallet/wallet.go b/wallet/wallet.go index f4fc8e9..c954f8f 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -280,7 +280,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f outputID := types.FileContractID(fce.ID).ValidOutputID(i) addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ - FileContract: fce, + Parent: fce, SiacoinElement: sces[outputID], Missed: false, }, []types.Address{address}) @@ -294,7 +294,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f outputID := types.FileContractID(fce.ID).MissedOutputID(i) addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ - FileContract: fce, + Parent: fce, SiacoinElement: sces[outputID], Missed: true, }, []types.Address{address}) @@ -315,8 +315,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.HostOutput.Address) { outputID := types.FileContractID(fce.ID).V2HostOutputID() addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ - FileContract: fce, - Resolution: res, + V2FileContractResolution: types.V2FileContractResolution{ + Parent: fce, + Resolution: res, + }, SiacoinElement: sces[outputID], Missed: missed, }, []types.Address{fce.V2FileContract.HostOutput.Address}) @@ -325,8 +327,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.RenterOutput.Address) { outputID := types.FileContractID(fce.ID).V2RenterOutputID() addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ - FileContract: fce, - Resolution: res, + V2FileContractResolution: types.V2FileContractResolution{ + Parent: fce, + Resolution: res, + }, SiacoinElement: sces[outputID], Missed: missed, }, []types.Address{fce.V2FileContract.RenterOutput.Address}) From 213b2810835ac75d893c674eb879b2ab5de3ceb7 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 18 Jun 2024 14:38:13 -0700 Subject: [PATCH 207/630] sqlite: add siacoin_elements index --- persist/sqlite/init.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 133cad1..a6220b1 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -28,6 +28,7 @@ CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); CREATE INDEX siacoin_elements_chain_index_id ON siacoin_elements (chain_index_id); CREATE INDEX siacoin_elements_spent_index_id ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id ON siacoin_elements(address_id, spent_index_id); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -42,6 +43,7 @@ CREATE TABLE siafund_elements ( CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id ON siafund_elements(address_id, spent_index_id); CREATE TABLE state_tree ( row INTEGER, From 81c6afa6b581bf28bf9f2f06f665c57f70ab8f7f Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Thu, 20 Jun 2024 20:40:18 +0000 Subject: [PATCH 208/630] ui: v0.22.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df2638a..e2c87b5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.7 go.sia.tech/coreutils v0.0.6 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.21.0 + go.sia.tech/web/walletd v0.22.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.21.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 16492a2..c3bc49a 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.21.0 h1:w5LCl8AWhl0G2HuExgr/AVXJWQlgVzGLDXskbb7pNZs= -go.sia.tech/web/walletd v0.21.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.22.0 h1:VB66+bLbnFBObCqn985QPf+r8Y08pOLbnKZXERNvKUQ= +go.sia.tech/web/walletd v0.22.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From d67db03d645b36593cbd42a5d825838b5f2c87a9 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Fri, 21 Jun 2024 20:18:47 +0000 Subject: [PATCH 209/630] ui: v0.22.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2c87b5..41bfa35 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.2.7 go.sia.tech/coreutils v0.0.6 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 - go.sia.tech/web/walletd v0.22.0 + go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 golang.org/x/term v0.21.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index c3bc49a..42768b9 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.22.0 h1:VB66+bLbnFBObCqn985QPf+r8Y08pOLbnKZXERNvKUQ= -go.sia.tech/web/walletd v0.22.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.22.1 h1:F9wxJD7whZFWhmefhLKlIq0IunUXdeP132DTAstv2Mc= +go.sia.tech/web/walletd v0.22.1/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From ba54fa0cd437af09b096f4505263a0323fda17c1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 21 Jun 2024 21:11:27 -0700 Subject: [PATCH 210/630] deps: update core --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 41bfa35..98a3fc4 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.7 + go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d go.sia.tech/coreutils v0.0.6 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.22.1 diff --git a/go.sum b/go.sum index 42768b9..4060d2d 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.2.7 h1:9Q/3BHL6ziAMPeiko863hhTD/Zs2s7OqEUiPKouDny8= go.sia.tech/core v0.2.7/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d h1:Y4FZH39Es+JFf2lQDubdTQRydyC4rY1uD2w0bZaxW1g= +go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= go.sia.tech/coreutils v0.0.6 h1:xLpv3JyvbOoXcX3gC6a6Y3zQ/MZn/fyFvuPIzv/e/Eg= go.sia.tech/coreutils v0.0.6/go.mod h1:EiDcQk2qLuP32Qoj/XphY9fbjTXphJhJZMERoC4LF0c= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= From f386e567c5787703debf4b9eed738b68dfdf43b4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 21 Jun 2024 21:21:20 -0700 Subject: [PATCH 211/630] wallet: fix v2 siafund claims --- wallet/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index c954f8f..fe79a3b 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -238,7 +238,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } addresses[sfi.Parent.SiafundOutput.Address] = true - outputID := types.SiafundOutputID(sfi.Parent.ID).ClaimOutputID() + outputID := types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID() if sfo, ok := sces[outputID]; ok && relevant(sfi.ClaimAddress) { addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeSiafundClaim, EventPayout{ SiacoinElement: sfo, From 1ce3f5166e2dca82fd6282b5c26e2c6d181a80f6 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 22 Jun 2024 09:35:13 -0700 Subject: [PATCH 212/630] wallet: fix siafund claim tests --- wallet/wallet_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index fa670c9..d4c5be6 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1355,9 +1355,9 @@ func TestFullIndex(t *testing.T) { // check the events for the transaction if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { t.Fatal(err) - } else if len(events) != 3 { - t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeV2Transaction { + } else if len(events) != 4 { + t.Fatalf("expected 4 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeSiafundClaim { t.Fatalf("expected transaction event, got %v", events[0].Type) } @@ -1578,9 +1578,9 @@ func TestEvents(t *testing.T) { // check the events for the transaction if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { t.Fatal(err) - } else if len(events) != 3 { - t.Fatalf("expected 3 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeV2Transaction { + } else if len(events) != 4 { + t.Fatalf("expected 4 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeSiafundClaim { t.Fatalf("expected transaction event, got %v", events[0].Type) } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { t.Fatalf("expected to get event: %v", err) From 1430b6a1247d2e6a1e89f1edcfc62683dcda769e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 16:35:10 +0000 Subject: [PATCH 213/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.2.8-0.20240621235715-5054a28dc43d to 0.2.8 - [Commits](https://github.com/SiaFoundation/core/commits/v0.2.8) Updates `go.sia.tech/coreutils` from 0.0.6 to 0.0.7 - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.0.6...v0.0.7) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 98a3fc4..9e660f2 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d - go.sia.tech/coreutils v0.0.6 + go.sia.tech/core v0.2.8 + go.sia.tech/coreutils v0.0.7 go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 4060d2d..8f5fec9 100644 --- a/go.sum +++ b/go.sum @@ -12,12 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.2.7 h1:9Q/3BHL6ziAMPeiko863hhTD/Zs2s7OqEUiPKouDny8= -go.sia.tech/core v0.2.7/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= -go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d h1:Y4FZH39Es+JFf2lQDubdTQRydyC4rY1uD2w0bZaxW1g= -go.sia.tech/core v0.2.8-0.20240621235715-5054a28dc43d/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= -go.sia.tech/coreutils v0.0.6 h1:xLpv3JyvbOoXcX3gC6a6Y3zQ/MZn/fyFvuPIzv/e/Eg= -go.sia.tech/coreutils v0.0.6/go.mod h1:EiDcQk2qLuP32Qoj/XphY9fbjTXphJhJZMERoC4LF0c= +go.sia.tech/core v0.2.8 h1:NQZw8gz9XWlkw9zr7HrLIA3xQnoatp8lYzyONS0IXJg= +go.sia.tech/core v0.2.8/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/coreutils v0.0.7 h1:+4nAMevcItpVUi7IMffvpU/O4D47pjDNF3LOv4y8Y/E= +go.sia.tech/coreutils v0.0.7/go.mod h1:Cke5eNMbvpjt+dX1u+m7/z1Du8jpwUaTGFafEMm0ZqE= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From e8655c032a0ad4ed85f4bc073b2f6dacec17ada0 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 24 Jun 2024 10:31:42 -0700 Subject: [PATCH 214/630] sqlite: improve wallet and address event query performance --- persist/sqlite/addresses.go | 2 +- persist/sqlite/init.sql | 29 ++++++++++++------------ persist/sqlite/migrations.go | 43 +++++++++++++++++++++++++++++++++++- persist/sqlite/wallet.go | 2 +- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index f395691..83e4b15 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -28,7 +28,7 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev + FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index a6220b1..ad6fd95 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -3,7 +3,7 @@ CREATE TABLE chain_indices ( block_id BLOB UNIQUE NOT NULL, height INTEGER UNIQUE NOT NULL ); -CREATE INDEX chain_indices_height ON chain_indices (block_id, height); +CREATE INDEX chain_indices_height_idx ON chain_indices (block_id, height); CREATE TABLE sia_addresses ( id INTEGER PRIMARY KEY, @@ -24,11 +24,11 @@ CREATE TABLE siacoin_elements ( chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ ); -CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); -CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); -CREATE INDEX siacoin_elements_chain_index_id ON siacoin_elements (chain_index_id); -CREATE INDEX siacoin_elements_spent_index_id ON siacoin_elements (spent_index_id); -CREATE INDEX siacoin_elements_address_id_spent_index_id ON siacoin_elements(address_id, spent_index_id); +CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id); CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, @@ -40,10 +40,10 @@ CREATE TABLE siafund_elements ( chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ ); -CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); -CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); -CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); -CREATE INDEX siafund_elements_address_id_spent_index_id ON siafund_elements(address_id, spent_index_id); +CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id); CREATE TABLE state_tree ( row INTEGER, @@ -61,7 +61,8 @@ CREATE TABLE events ( event_type TEXT NOT NULL, event_data BLOB NOT NULL ); -CREATE INDEX events_chain_index_id ON events (chain_index_id); +CREATE INDEX events_chain_index_id_idx ON events (chain_index_id); +CREATE INDEX events_maturity_height_id_idx ON events (maturity_height DESC, id DESC); CREATE TABLE event_addresses ( event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, @@ -88,8 +89,8 @@ CREATE TABLE wallet_addresses ( extra_data BLOB, UNIQUE (wallet_id, address_id) ); -CREATE INDEX wallet_addresses_wallet_id ON wallet_addresses (wallet_id); -CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); +CREATE INDEX wallet_addresses_wallet_id_idx ON wallet_addresses (wallet_id); +CREATE INDEX wallet_addresses_address_id_idx ON wallet_addresses (address_id); CREATE TABLE syncer_peers ( peer_address TEXT PRIMARY KEY NOT NULL, @@ -101,7 +102,7 @@ CREATE TABLE syncer_bans ( expiration INTEGER NOT NULL, reason TEXT NOT NULL ); -CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); +CREATE INDEX syncer_bans_expiration_index_idx ON syncer_bans (expiration); CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 99d01e1..bf8fde2 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -4,7 +4,48 @@ import ( "go.uber.org/zap" ) +// recreates indices and speeds up event queries +func migrateVersion2(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`DROP INDEX IF EXISTS chain_indices_height; +DROP INDEX IF EXISTS siacoin_elements_address_id; +DROP INDEX IF EXISTS siacoin_elements_maturity_height_matured; +DROP INDEX IF EXISTS siacoin_elements_chain_index_id; +DROP INDEX IF EXISTS siacoin_elements_spent_index_id; +DROP INDEX IF EXISTS siacoin_elements_address_id_spent_index_id; +DROP INDEX IF EXISTS siafund_elements_address_id; +DROP INDEX IF EXISTS siafund_elements_chain_index_id; +DROP INDEX IF EXISTS siafund_elements_spent_index_id; +DROP INDEX IF EXISTS siafund_elements_address_id_spent_index_id; +DROP INDEX IF EXISTS events_chain_index_id; +DROP INDEX IF EXISTS event_addresses_event_id_idx; +DROP INDEX IF EXISTS event_addresses_address_id_idx; +DROP INDEX IF EXISTS wallet_addresses_wallet_id; +DROP INDEX IF EXISTS wallet_addresses_address_id; +DROP INDEX IF EXISTS syncer_bans_expiration_index; + +CREATE INDEX IF NOT EXISTS chain_indices_height_idx ON chain_indices (block_id, height); +CREATE INDEX IF NOT EXISTS siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX IF NOT EXISTS siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX IF NOT EXISTS siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id); +CREATE INDEX IF NOT EXISTS events_chain_index_id_idx ON events (chain_index_id); +CREATE INDEX IF NOT EXISTS events_maturity_height_id_idx ON events (maturity_height DESC, id DESC); +CREATE INDEX IF NOT EXISTS event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX IF NOT EXISTS event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX IF NOT EXISTS wallet_addresses_wallet_id_idx ON wallet_addresses (wallet_id); +CREATE INDEX IF NOT EXISTS wallet_addresses_address_id_idx ON wallet_addresses (address_id); +CREATE INDEX IF NOT EXISTS syncer_bans_expiration_index_idx ON syncer_bans (expiration);`) + return err +} + // migrations is a list of functions that are run to migrate the database from // one version to the next. Migrations are used to update existing databases to // match the schema in init.sql. -var migrations = []func(tx *txn, log *zap.Logger) error{} +var migrations = []func(tx *txn, log *zap.Logger) error{ + migrateVersion2, +} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index e78045f..69e8f8d 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -642,7 +642,7 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev + FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) ORDER BY ev.maturity_height DESC, ev.id DESC From 57579c04a2731cbcefb3d0ca9dde8f390898e31e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 24 Jun 2024 12:34:14 -0700 Subject: [PATCH 215/630] update jape --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9e660f2..fcaea60 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.2.8 go.sia.tech/coreutils v0.0.7 - go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 + go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385 go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 golang.org/x/term v0.21.0 diff --git a/go.sum b/go.sum index 8f5fec9..2c05a30 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ go.sia.tech/core v0.2.8 h1:NQZw8gz9XWlkw9zr7HrLIA3xQnoatp8lYzyONS0IXJg= go.sia.tech/core v0.2.8/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= go.sia.tech/coreutils v0.0.7 h1:+4nAMevcItpVUi7IMffvpU/O4D47pjDNF3LOv4y8Y/E= go.sia.tech/coreutils v0.0.7/go.mod h1:Cke5eNMbvpjt+dX1u+m7/z1Du8jpwUaTGFafEMm0ZqE= -go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 h1:mSaJ622P7T/M97dAK8iPV+IRIC9M5vV28NHeceoWO3M= -go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= +go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385 h1:Gho1g6pkv56o6Ut9cez/Yu5o4xlA8WNkDbPn6RWXL7g= +go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= From 1691189b2755f3004588d82c1867cb1a93aec710 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 25 Jun 2024 07:40:16 -0700 Subject: [PATCH 216/630] update core, coreutils, jape --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index fcaea60..37a1c77 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.8 - go.sia.tech/coreutils v0.0.7 - go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385 + go.sia.tech/core v0.2.9 + go.sia.tech/coreutils v0.0.8 + go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 golang.org/x/term v0.21.0 diff --git a/go.sum b/go.sum index 2c05a30..a13518a 100644 --- a/go.sum +++ b/go.sum @@ -12,12 +12,12 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.2.8 h1:NQZw8gz9XWlkw9zr7HrLIA3xQnoatp8lYzyONS0IXJg= -go.sia.tech/core v0.2.8/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= -go.sia.tech/coreutils v0.0.7 h1:+4nAMevcItpVUi7IMffvpU/O4D47pjDNF3LOv4y8Y/E= -go.sia.tech/coreutils v0.0.7/go.mod h1:Cke5eNMbvpjt+dX1u+m7/z1Du8jpwUaTGFafEMm0ZqE= -go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385 h1:Gho1g6pkv56o6Ut9cez/Yu5o4xlA8WNkDbPn6RWXL7g= -go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= +go.sia.tech/core v0.2.9 h1:UnO+wXQ3w2dMaU3ULA95fLYspllxaYPSfVW08fFIxQU= +go.sia.tech/core v0.2.9/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/coreutils v0.0.8 h1:eHrzR5s2J4Q1cX+VHLibjNuPTKNGRq/D6jhR+wSRHFE= +go.sia.tech/coreutils v0.0.8/go.mod h1:7VSwhyxoE3DqVFbcLUsN9zC4AnBZ9/QSz/ff+uEzGgc= +go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= +go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= From 693436e34c1033e698ea8fcc73f3d711ef5dfe7e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 25 Jun 2024 10:59:48 -0700 Subject: [PATCH 217/630] api: use zero value for start time --- api/server.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/server.go b/api/server.go index f8c2cad..42c2e4a 100644 --- a/api/server.go +++ b/api/server.go @@ -273,9 +273,6 @@ func (s *server) rescanHandlerGET(jc jape.Context) { s.scanMu.Lock() defer s.scanMu.Unlock() - if s.scanInfo.StartTime.IsZero() { - s.scanInfo.StartTime = s.startTime - } s.scanInfo.Index = index jc.Encode(s.scanInfo) } From e4d70cd7c4f6666bc369740e06830b68d1b0f475 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 25 Jun 2024 11:08:51 -0700 Subject: [PATCH 218/630] sqlite: defer foreign key checks during migration --- persist/sqlite/init.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index 329e9a8..9579e04 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -37,18 +37,12 @@ func (s *Store) upgradeDatabase(current, target int64) error { log := s.log.Named("migrations") log.Info("migrating database", zap.Int64("current", current), zap.Int64("target", target)) - // disable foreign key constraints during migration - if _, err := s.db.Exec("PRAGMA foreign_keys = OFF"); err != nil { - return fmt.Errorf("failed to disable foreign key constraints: %w", err) - } - defer func() { - // re-enable foreign key constraints - if _, err := s.db.Exec("PRAGMA foreign_keys = ON"); err != nil { - log.Panic("failed to enable foreign key constraints", zap.Error(err)) + return s.transaction(func(tx *txn) error { + // defer foreign key constraints until commit + if _, err := tx.Exec("PRAGMA defer_foreign_keys=ON"); err != nil { + return fmt.Errorf("failed to enable foreign key deferral: %w", err) } - }() - return s.transaction(func(tx *txn) error { for _, fn := range migrations[current-1:] { current++ start := time.Now() From 3a0076c8102cdd7f93f324a1ad1beb1c7f1c9880 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 26 Jun 2024 09:53:14 -0700 Subject: [PATCH 219/630] sqlite,wallet: change utxo queries to only include matured Siacoin outputs --- persist/sqlite/addresses.go | 8 +++--- persist/sqlite/consensus_test.go | 2 +- persist/sqlite/wallet.go | 8 +++--- wallet/addresses.go | 2 +- wallet/manager.go | 13 ++++----- wallet/wallet_test.go | 46 ++++++++++---------------------- 6 files changed, 31 insertions(+), 48 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 83e4b15..7a64f16 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -56,15 +56,15 @@ func (s *Store) AddressEvents(address types.Address, offset, limit int) (events } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (s *Store) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address=$1 AND se.spent_index_id IS NULL - LIMIT $2 OFFSET $3` + WHERE sa.sia_address=$1 AND se.maturity_height <= $2 AND se.spent_index_id IS NULL + LIMIT $3 OFFSET $4` - rows, err := tx.Query(query, encode(address), limit, offset) + rows, err := tx.Query(query, encode(address), index.Height, limit, offset) if err != nil { return err } diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index a93c89d..cc77c4f 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -137,7 +137,7 @@ func TestPruneSiacoins(t *testing.T) { assertUTXOs(0, 1) // spend the utxo - utxos, err := db.WalletSiacoinOutputs(w.ID, 0, 100) + utxos, err := db.WalletSiacoinOutputs(w.ID, cm.Tip(), 0, 100) if err != nil { t.Fatalf("failed to get wallet siacoin outputs: %v", err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 69e8f8d..0b56101 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -208,7 +208,7 @@ WHERE wa.wallet_id=$1` } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) WalletSiacoinOutputs(id wallet.ID, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -217,10 +217,10 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) - LIMIT $2 OFFSET $3` + WHERE se.spent_index_id IS NULL AND se.maturity_height <= $1 AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$2) + LIMIT $3 OFFSET $4` - rows, err := tx.Query(query, id, limit, offset) + rows, err := tx.Query(query, index.Height, id, limit, offset) if err != nil { return err } diff --git a/wallet/addresses.go b/wallet/addresses.go index 841734c..2358631 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -13,7 +13,7 @@ func (m *Manager) AddressBalance(address types.Address) (balance Balance, err er // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { - return m.store.AddressSiacoinOutputs(address, offset, limit) + return m.store.AddressSiacoinOutputs(address, m.chain.Tip(), offset, limit) } // AddressSiafundOutputs returns the unspent siafund outputs for an address. diff --git a/wallet/manager.go b/wallet/manager.go index 7355bc8..6a5b428 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -61,7 +61,7 @@ type ( UpdateWallet(Wallet) (Wallet, error) DeleteWallet(walletID ID) error WalletBalance(walletID ID) (Balance, error) - WalletSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, error) + WalletSiacoinOutputs(walletID ID, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, error) WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) @@ -71,7 +71,7 @@ type ( AddressBalance(address types.Address) (balance Balance, err error) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) - AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) + AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) Events(eventIDs []types.Hash256) ([]Event, error) @@ -175,13 +175,14 @@ func (m *Manager) WalletEvents(walletID ID, offset, limit int) ([]Event, error) return m.store.WalletEvents(walletID, offset, limit) } -// UnspentSiacoinOutputs returns a paginated list of unspent siacoin outputs of -// the given wallet and the total number of unspent siacoin outputs. +// UnspentSiacoinOutputs returns a paginated list of matured siacoin outputs +// relevant to the wallet func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, error) { - return m.store.WalletSiacoinOutputs(walletID, offset, limit) + return m.store.WalletSiacoinOutputs(walletID, m.chain.Tip(), offset, limit) } -// UnspentSiafundOutputs returns the unspent siafund outputs of the given wallet +// UnspentSiafundOutputs returns a paginated list of siafund outputs relevant to +// the wallet func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) { return m.store.WalletSiafundOutputs(walletID, offset, limit) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index d4c5be6..66485d2 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -141,7 +141,6 @@ func TestReorg(t *testing.T) { } expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() // mine a block sending the payout to the wallet if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) @@ -174,16 +173,12 @@ func TestReorg(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Type) } - // check that the utxo was created + // check that the utxo has not matured utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) } // mine to trigger a reorg @@ -223,7 +218,7 @@ func TestReorg(t *testing.T) { // mine a new payout expectedPayout = cm.TipState().BlockReward() - maturityHeight = cm.TipState().MaturityHeight() + maturityHeight := cm.TipState().MaturityHeight() if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } @@ -244,16 +239,12 @@ func TestReorg(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Type) } - // check that the utxo was created + // check that the utxo has not matured utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) } // mine until the payout matures @@ -2385,7 +2376,6 @@ func TestReorgV2(t *testing.T) { } expectedPayout := cm.TipState().BlockReward() - maturityHeight := cm.TipState().MaturityHeight() // mine a block sending the payout to the wallet if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) @@ -2418,16 +2408,12 @@ func TestReorgV2(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Type) } - // check that the utxo was created + // check that the utxo has not matured utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) } // mine to trigger a reorg @@ -2467,7 +2453,7 @@ func TestReorgV2(t *testing.T) { // mine a new payout expectedPayout = cm.TipState().BlockReward() - maturityHeight = cm.TipState().MaturityHeight() + maturityHeight := cm.TipState().MaturityHeight() if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { t.Fatal(err) } @@ -2488,16 +2474,12 @@ func TestReorgV2(t *testing.T) { t.Fatalf("expected payout event, got %v", events[0].Type) } - // check that the utxo was created + // check that the utxo has not matured utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) - } else if len(utxos) != 1 { - t.Fatalf("expected 1 output, got %v", len(utxos)) - } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { - t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) - } else if utxos[0].MaturityHeight != maturityHeight { - t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) } // mine until the payout matures From 5d2651022dcbd2a87da85a6f48eb1ceca977d574 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 27 Jun 2024 07:37:04 -0700 Subject: [PATCH 220/630] disable flaky test --- api/api_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index 50862ce..5f0a077 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -893,6 +893,8 @@ func TestV2(t *testing.T) { } func TestP2P(t *testing.T) { + t.Skip("flaky test") // TODO refactor + logger := zaptest.NewLogger(t) n, genesisBlock := testNetwork() // gift primary wallet some coins From c1262248d936efc326736c9b37b0969c65ddbaf9 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 27 Jun 2024 08:02:11 -0700 Subject: [PATCH 221/630] sqlite,wallet: update coreutils --- go.mod | 4 ++-- go.sum | 8 ++++---- persist/sqlite/addresses.go | 4 ++-- persist/sqlite/wallet.go | 4 ++-- wallet/update.go | 37 ++++++++----------------------------- wallet/wallet.go | 24 ++++++++++++------------ 6 files changed, 30 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 37a1c77..8b9b1a0 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.2.9 - go.sia.tech/coreutils v0.0.8 + go.sia.tech/core v0.3.0 + go.sia.tech/coreutils v0.1.0 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index a13518a..c78a348 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.2.9 h1:UnO+wXQ3w2dMaU3ULA95fLYspllxaYPSfVW08fFIxQU= -go.sia.tech/core v0.2.9/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= -go.sia.tech/coreutils v0.0.8 h1:eHrzR5s2J4Q1cX+VHLibjNuPTKNGRq/D6jhR+wSRHFE= -go.sia.tech/coreutils v0.0.8/go.mod h1:7VSwhyxoE3DqVFbcLUsN9zC4AnBZ9/QSz/ff+uEzGgc= +go.sia.tech/core v0.3.0 h1:PDfAQh9z8PYD+oeVS7rS9SEnTMOZzwwFfAH45yktmko= +go.sia.tech/core v0.3.0/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/coreutils v0.1.0 h1:WQL7iT+jK1BiMx87bASXrZJZf4N2fbQkIOW8rS7wkh4= +go.sia.tech/coreutils v0.1.0/go.mod h1:ybaFgewKXrlxFW71LqsyQlxjG6yWL6BSePrbZYnrprU= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 7a64f16..497742f 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -229,7 +229,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 sce := types.SiacoinElement{ StateElement: types.StateElement{ ID: types.Hash256(txn.SiacoinOutputID(i)), - LeafIndex: types.EphemeralLeafIndex, + LeafIndex: types.UnassignedLeafIndex, }, SiacoinOutput: output, } @@ -253,7 +253,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 sfe := types.SiafundElement{ StateElement: types.StateElement{ ID: types.Hash256(txn.SiafundOutputID(i)), - LeafIndex: types.EphemeralLeafIndex, + LeafIndex: types.UnassignedLeafIndex, }, SiafundOutput: output, } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 0b56101..0353dc6 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -476,7 +476,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti sce := types.SiacoinElement{ StateElement: types.StateElement{ ID: types.Hash256(txn.SiacoinOutputID(i)), - LeafIndex: types.EphemeralLeafIndex, + LeafIndex: types.UnassignedLeafIndex, }, SiacoinOutput: output, } @@ -515,7 +515,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti sfe := types.SiafundElement{ StateElement: types.StateElement{ ID: types.Hash256(txn.SiafundOutputID(i)), - LeafIndex: types.EphemeralLeafIndex, + LeafIndex: types.UnassignedLeafIndex, }, SiafundOutput: output, } diff --git a/wallet/update.go b/wallet/update.go index 83852b0..4f458dd 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -116,30 +116,9 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e NumLeaves: cau.State.Elements.NumLeaves, } - // determine which siacoin and siafund elements are ephemeral - // - // note: I thought we could use LeafIndex == EphemeralLeafIndex, but - // it seems to be set before the subscriber is called. - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cau.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - } - // add new siacoin elements to the store - cau.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { + cau.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { + if created && spent { return } @@ -157,8 +136,8 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e } }) - cau.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { + cau.ForEachSiafundElement(func(se types.SiafundElement, created, spent bool) { + if created && spent { return } @@ -221,8 +200,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } } - cru.ForEachSiacoinElement(func(se types.SiacoinElement, spent bool) { - if ephemeral[se.ID] { + cru.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { + if created && spent { return } @@ -242,8 +221,8 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } }) - cru.ForEachSiafundElement(func(se types.SiafundElement, spent bool) { - if ephemeral[se.ID] { + cru.ForEachSiafundElement(func(se types.SiafundElement, created, spent bool) { + if created && spent { return } diff --git a/wallet/wallet.go b/wallet/wallet.go index fe79a3b..00d267f 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -41,10 +41,10 @@ type ( // A ChainUpdate is a set of changes to the consensus state. ChainUpdate interface { - ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) - ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) + ForEachSiacoinElement(func(sce types.SiacoinElement, created, spent bool)) + ForEachSiafundElement(func(sfe types.SiafundElement, created, spent bool)) + ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool)) + ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) } ) @@ -116,12 +116,12 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } anythingRelevant := func() (ok bool) { - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { if ok || relevant(sce.SiacoinOutput.Address) { ok = true } }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + cu.ForEachSiafundElement(func(sfe types.SiafundElement, _, _ bool) { if ok || relevant(sfe.SiafundOutput.Address) { ok = true } @@ -137,19 +137,19 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sfes := make(map[types.SiafundOutputID]types.SiafundElement) fces := make(map[types.FileContractID]types.FileContractElement) v2fces := make(map[types.FileContractID]types.V2FileContractElement) - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { sce.MerkleProof = nil sces[types.SiacoinOutputID(sce.ID)] = sce }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + cu.ForEachSiafundElement(func(sfe types.SiafundElement, _, _ bool) { sfe.MerkleProof = nil sfes[types.SiafundOutputID(sfe.ID)] = sfe }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + cu.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { fce.MerkleProof = nil fces[types.FileContractID(fce.ID)] = fce }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { fce.MerkleProof = nil v2fces[types.FileContractID(fce.ID)] = fce }) @@ -266,7 +266,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } // handle missed contracts - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + cu.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { if !resolved { return } @@ -302,7 +302,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if res == nil { return } From 1fe129bdc40ac9025636f526a74b098e0c684857 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 28 Jun 2024 09:26:47 -0700 Subject: [PATCH 222/630] sqlite: improve wallet events query when wallet has no events --- persist/sqlite/init.sql | 2 ++ persist/sqlite/migrations.go | 11 ++++++++++- persist/sqlite/wallet.go | 23 ++++++++++++++++++++--- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index ad6fd95..3517b44 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -71,6 +71,7 @@ CREATE TABLE event_addresses ( ); CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (event_id, address_id); CREATE TABLE wallets ( id INTEGER PRIMARY KEY, @@ -91,6 +92,7 @@ CREATE TABLE wallet_addresses ( ); CREATE INDEX wallet_addresses_wallet_id_idx ON wallet_addresses (wallet_id); CREATE INDEX wallet_addresses_address_id_idx ON wallet_addresses (address_id); +CREATE INDEX wallet_addresses_wallet_id_address_id_idx ON wallet_addresses (wallet_id, address_id); CREATE TABLE syncer_peers ( peer_address TEXT PRIMARY KEY NOT NULL, diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index bf8fde2..79544ec 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -4,7 +4,15 @@ import ( "go.uber.org/zap" ) -// recreates indices and speeds up event queries +// migrateVersion3 adds additional indices to event_addresses and wallet_addresses +// to improve query performance. +func migrateVersion3(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (event_id, address_id); +CREATE INDEX wallet_addresses_wallet_id_address_id_idx ON wallet_addresses (wallet_id, address_id);`) + return err +} + +// migrateVersion2 recreates indices and speeds up event queries func migrateVersion2(tx *txn, _ *zap.Logger) error { _, err := tx.Exec(`DROP INDEX IF EXISTS chain_indices_height; DROP INDEX IF EXISTS siacoin_elements_address_id; @@ -48,4 +56,5 @@ CREATE INDEX IF NOT EXISTS syncer_bans_expiration_index_idx ON syncer_bans (expi // match the schema in init.sql. var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion2, + migrateVersion3, } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 0353dc6..6204848 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -641,14 +641,31 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e } func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + // the events query can be slow in full index mode for wallets with no + // events. Check if the wallet has events first. + const hasEventsQuery = `SELECT EXISTS ( + SELECT 1 + FROM event_addresses ea + INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id + WHERE wa.wallet_id=$1 +) AS has_events;` + var hasEvents bool + if err := tx.QueryRow(hasEventsQuery, id).Scan(&hasEvents); err != nil { + return nil, nil, err + } else if !hasEvents { + return nil, nil, nil + } + + const eventsQuery = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts + INNER JOIN event_addresses ea ON (ev.id = ea.event_id) + INNER JOIN wallet_addresses wa ON (ea.address_id = wa.address_id) INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) - WHERE ev.id IN (SELECT event_id FROM event_addresses WHERE address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1)) + WHERE wa.wallet_id=$1 ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` - rows, err := tx.Query(query, id, limit, offset) + rows, err := tx.Query(eventsQuery, id, limit, offset) if err != nil { return nil, nil, err } From 8a821dc589467662bc83344f107debf3e86eec8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:35:42 +0000 Subject: [PATCH 223/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.21.0 to 0.22.0 - [Commits](https://github.com/golang/term/compare/v0.21.0...v0.22.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 8b9b1a0..59e85ef 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.1 go.uber.org/zap v1.27.0 - golang.org/x/term v0.21.0 + golang.org/x/term v0.22.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 @@ -26,6 +26,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index c78a348..10028d9 100644 --- a/go.sum +++ b/go.sum @@ -37,10 +37,10 @@ golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 255bbbad2a0d2c83469b1c58066cabc1978882e3 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Wed, 10 Jul 2024 21:30:38 +0000 Subject: [PATCH 224/630] ui: v0.22.2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 59e85ef..59f3b64 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.3.0 go.sia.tech/coreutils v0.1.0 go.sia.tech/jape v0.12.0 - go.sia.tech/web/walletd v0.22.1 + go.sia.tech/web/walletd v0.22.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.22.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 10028d9..ff0dac8 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.22.1 h1:F9wxJD7whZFWhmefhLKlIq0IunUXdeP132DTAstv2Mc= -go.sia.tech/web/walletd v0.22.1/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.22.2 h1:98MJ4I6chgFUQ2ZHj/96JYXnS75gnJdtW5nZHc0sqk0= +go.sia.tech/web/walletd v0.22.2/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 4f55a455434b7b4b27eb0564ddfc82abe8c37dce Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Fri, 12 Jul 2024 14:37:54 +0000 Subject: [PATCH 225/630] ui: v0.22.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 59f3b64..859f9d8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.3.0 go.sia.tech/coreutils v0.1.0 go.sia.tech/jape v0.12.0 - go.sia.tech/web/walletd v0.22.2 + go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 golang.org/x/term v0.22.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ff0dac8..3ab1a66 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.22.2 h1:98MJ4I6chgFUQ2ZHj/96JYXnS75gnJdtW5nZHc0sqk0= -go.sia.tech/web/walletd v0.22.2/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.22.3 h1:I8og0NN2AW1VC2Oi2Kp/e6/Io14PFNiumELju8Hh2dU= +go.sia.tech/web/walletd v0.22.3/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 64fe08655eacdff4fa1a6e142ce389cb0991a6b2 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jul 2024 14:39:08 -0700 Subject: [PATCH 226/630] wallet: fix maturity height off-by-one, fix v2 resolution json encoding --- wallet/events.go | 6 +- wallet/wallet.go | 64 ++-- wallet/wallet_test.go | 815 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 851 insertions(+), 34 deletions(-) diff --git a/wallet/events.go b/wallet/events.go index 3151a0e..a42ab7e 100644 --- a/wallet/events.go +++ b/wallet/events.go @@ -71,9 +71,9 @@ type ( // An EventV2ContractResolution represents a file contract payout from a v2 // contract. EventV2ContractResolution struct { - types.V2FileContractResolution - SiacoinElement types.SiacoinElement `json:"siacoinElement"` - Missed bool `json:"missed"` + Resolution types.V2FileContractResolution `json:"resolution"` + SiacoinElement types.SiacoinElement `json:"siacoinElement"` + Missed bool `json:"missed"` } ) diff --git a/wallet/wallet.go b/wallet/wallet.go index 00d267f..4806bcf 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -3,6 +3,7 @@ package wallet import ( "encoding/json" "errors" + "log" "strconv" "time" @@ -191,10 +192,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f addresses[sfe.SiafundOutput.Address] = true } - outputID := sfi.ParentID.ClaimOutputID() - if sfo, ok := sces[outputID]; ok && relevant(sfi.ClaimAddress) { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeSiafundClaim, EventPayout{ - SiacoinElement: sfo, + sce, ok := sces[sfi.ParentID.ClaimOutputID()] + if ok && relevant(sce.SiacoinOutput.Address) { + addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, EventPayout{ + SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } } @@ -238,10 +239,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } addresses[sfi.Parent.SiafundOutput.Address] = true - outputID := types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID() - if sfo, ok := sces[outputID]; ok && relevant(sfi.ClaimAddress) { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeSiafundClaim, EventPayout{ - SiacoinElement: sfo, + sce, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] + if ok && relevant(sfi.ClaimAddress) { + addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, EventPayout{ + SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } } @@ -265,7 +266,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeV2Transaction, ev, relevant) // transaction maturity height is the current block height } - // handle missed contracts + // handle contracts cu.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { if !resolved { return @@ -278,10 +279,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f continue } - outputID := types.FileContractID(fce.ID).ValidOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ + element := sces[types.FileContractID(fce.ID).ValidOutputID(i)] + addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, EventV1ContractResolution{ Parent: fce, - SiacoinElement: sces[outputID], + SiacoinElement: element, Missed: false, }, []types.Address{address}) } @@ -292,10 +293,10 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f continue } - outputID := types.FileContractID(fce.ID).MissedOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV1ContractResolution, EventV1ContractResolution{ + element := sces[types.FileContractID(fce.ID).MissedOutputID(i)] + addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, EventV1ContractResolution{ Parent: fce, - SiacoinElement: sces[outputID], + SiacoinElement: element, Missed: true, }, []types.Address{address}) } @@ -313,25 +314,27 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } if relevant(fce.V2FileContract.HostOutput.Address) { - outputID := types.FileContractID(fce.ID).V2HostOutputID() - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ - V2FileContractResolution: types.V2FileContractResolution{ + element := sces[types.FileContractID(fce.ID).V2HostOutputID()] + log.Println("HOST", element.ID, fce.V2FileContract.HostOutput.Address) + addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ + Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, }, - SiacoinElement: sces[outputID], + SiacoinElement: element, Missed: missed, }, []types.Address{fce.V2FileContract.HostOutput.Address}) } if relevant(fce.V2FileContract.RenterOutput.Address) { - outputID := types.FileContractID(fce.ID).V2RenterOutputID() - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeV2ContractResolution, EventV2ContractResolution{ - V2FileContractResolution: types.V2FileContractResolution{ + element := sces[types.FileContractID(fce.ID).V2RenterOutputID()] + log.Println("RENTER", element.ID, fce.V2FileContract.RenterOutput.Address) + addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ + Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, }, - SiacoinElement: sces[outputID], + SiacoinElement: element, Missed: missed, }, []types.Address{fce.V2FileContract.RenterOutput.Address}) } @@ -340,21 +343,20 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f // handle block rewards for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { - outputID := cs.Index.ID.MinerOutputID(i) - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeMinerPayout, EventPayout{ - SiacoinElement: sces[outputID], + element := sces[cs.Index.ID.MinerOutputID(i)] + addEvent(element.ID, element.MaturityHeight, EventTypeMinerPayout, EventPayout{ + SiacoinElement: element, }, []types.Address{b.MinerPayouts[i].Address}) } } // handle foundation subsidy if relevant(cs.FoundationPrimaryAddress) { - outputID := cs.Index.ID.FoundationOutputID() - sce, ok := sces[outputID] + element, ok := sces[cs.Index.ID.FoundationOutputID()] if ok { - addEvent(types.Hash256(outputID), cs.MaturityHeight(), EventTypeFoundationSubsidy, EventPayout{ - SiacoinElement: sce, - }, []types.Address{sce.SiacoinOutput.Address}) + addEvent(element.ID, element.MaturityHeight, EventTypeFoundationSubsidy, EventPayout{ + SiacoinElement: element, + }, []types.Address{element.SiacoinOutput.Address}) } } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 66485d2..e0d600b 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -5,8 +5,11 @@ import ( "context" "encoding/json" "fmt" + "math" + "math/bits" "path/filepath" "reflect" + "sort" "testing" "time" @@ -17,6 +20,7 @@ import ( "go.sia.tech/coreutils/testutil" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -2821,3 +2825,814 @@ func TestDeleteWallet(t *testing.T) { t.Fatal(err) } } + +// NOTE: due to a bug in the transaction validation code, calculating payouts +// is way harder than it needs to be. Tax is calculated on the post-tax +// contract payout (instead of the sum of the renter and host payouts). So the +// equation for the payout is: +// +// payout = renterPayout + hostPayout + payout*tax +// ∴ payout = (renterPayout + hostPayout) / (1 - tax) +// +// This would work if 'tax' were a simple fraction, but because the tax must +// be evenly distributed among siafund holders, 'tax' is actually a function +// that multiplies by a fraction and then rounds down to the nearest multiple +// of the siafund count. Thus, when inverting the function, we have to make an +// initial guess and then fix the rounding error. +func taxAdjustedPayout(target types.Currency) types.Currency { + // compute initial guess as target * (1 / 1-tax); since this does not take + // the siafund rounding into account, the guess will be up to + // types.SiafundCount greater than the actual payout value. + guess := target.Mul64(1000).Div64(961) + + // now, adjust the guess to remove the rounding error. We know that: + // + // (target % types.SiafundCount) == (payout % types.SiafundCount) + // + // therefore, we can simply adjust the guess to have this remainder as + // well. The only wrinkle is that, since we know guess >= payout, if the + // guess remainder is smaller than the target remainder, we must subtract + // an extra types.SiafundCount. + // + // for example, if target = 87654321 and types.SiafundCount = 10000, then: + // + // initial_guess = 87654321 * (1 / (1 - tax)) + // = 91211572 + // target % 10000 = 4321 + // adjusted_guess = 91204321 + + mod64 := func(c types.Currency, v uint64) types.Currency { + var r uint64 + if c.Hi < v { + _, r = bits.Div64(c.Hi, c.Lo, v) + } else { + _, r = bits.Div64(0, c.Hi, v) + _, r = bits.Div64(r, c.Lo, v) + } + return types.NewCurrency64(r) + } + sfc := (consensus.State{}).SiafundCount() + tm := mod64(target, sfc) + gm := mod64(guess, sfc) + if gm.Cmp(tm) < 0 { + guess = guess.Sub(types.NewCurrency64(sfc)) + } + return guess.Add(tm).Sub(gm) +} + +func TestEventTypes(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zap.NewNop() + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // create a new test network with the Siafund airdrop going to the wallet address + network, genesisBlock := testV2Network(addr) + // raise the require height to test v1 events + network.HardforkV2.RequireHeight = 250 + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + // helper to mine blocks + mineBlock := func(n int, addr types.Address) { + t.Helper() + for i := 0; i < n; i++ { + b, ok := coreutils.MineBlock(cm, addr, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + } + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + spendableSiacoinUTXOs := func() []types.SiacoinElement { + t.Helper() + + sces, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + filtered := sces[:0] + height := cm.Tip().Height + for _, sce := range sces { + if sce.MaturityHeight > height { + continue + } + filtered = append(filtered, sce) + } + sort.Slice(filtered, func(i, j int) bool { + return filtered[i].SiacoinOutput.Value.Cmp(filtered[j].SiacoinOutput.Value) < 0 + }) + return filtered + } + + assertEvent := func(id types.Hash256, eventType string, expectedInflow, expectedOutflow types.Currency, maturityHeight uint64) { + t.Helper() + + events, err := wm.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + for _, event := range events { + if event.ID == id { + t.Log(id, eventType) + if event.Type != eventType { + t.Fatalf("expected %v event, got %v", eventType, event.Type) + } else if event.MaturityHeight != maturityHeight { + t.Fatalf("expected maturity height %v, got %v", maturityHeight, event.MaturityHeight) + } + + var inflowSum, outflowSum types.Currency + switch ev := event.Data.(type) { + case wallet.EventV1Transaction: + for _, sce := range ev.SpentSiacoinElements { + if sce.SiacoinOutput.Address == addr { + outflowSum = outflowSum.Add(sce.SiacoinOutput.Value) + } + } + for _, sce := range ev.Transaction.SiacoinOutputs { + if sce.Address == addr { + inflowSum = inflowSum.Add(sce.Value) + } + } + case wallet.EventV1ContractResolution: + if ev.SiacoinElement.SiacoinOutput.Address == addr { + inflowSum = ev.SiacoinElement.SiacoinOutput.Value + } + case wallet.EventPayout: + if ev.SiacoinElement.SiacoinOutput.Address == addr { + inflowSum = ev.SiacoinElement.SiacoinOutput.Value + } + case wallet.EventV2ContractResolution: + if ev.SiacoinElement.SiacoinOutput.Address == addr { + inflowSum = ev.SiacoinElement.SiacoinOutput.Value + } + case wallet.EventV2Transaction: + for _, sce := range ev.SiacoinInputs { + if sce.Parent.SiacoinOutput.Address == addr { + outflowSum = outflowSum.Add(sce.Parent.SiacoinOutput.Value) + } + } + for _, sce := range ev.SiacoinOutputs { + if sce.Address == addr { + inflowSum = inflowSum.Add(sce.Value) + } + } + default: + t.Fatalf("unexpected event type %T", ev) + } + + if !inflowSum.Equals(expectedInflow) { + t.Fatalf("expected inflow %v, got %v", expectedInflow, inflowSum) + } else if !outflowSum.Equals(expectedOutflow) { + t.Fatalf("expected outflow %v, got %v", expectedOutflow, outflowSum) + } + return + } + } + t.Fatalf("event not found") + } + + // miner payout event + { + mineBlock(1, addr) + assertEvent(types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) + } + + // mine until the payout matures + mineBlock(int(cm.TipState().MaturityHeight()), types.VoidAddress) + + // v1 transaction + { + sce := spendableSiacoinUTXOs() + + // v1 only supports unlock conditions + uc := types.StandardUnlockConditions(pk.PublicKey()) + + // create a transaction + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + {ParentID: types.SiacoinOutputID(sce[0].ID), UnlockConditions: uc}, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1000)}, + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000))}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: sce[0].ID, + PublicKeyIndex: 0, + Timelock: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + + // sign the transaction + sigHash := cm.TipState().WholeSigHash(txn, sce[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + } + + // v1 contract resolution - only one type of resolution is supported. + // The only difference is `missed == true` or `missed == false` + { + sce := spendableSiacoinUTXOs() + + uc := types.StandardUnlockConditions(pk.PublicKey()) + + // create a storage contract + contractPayout := types.Siacoins(10000) + fc := types.FileContract{ + WindowStart: cm.TipState().Index.Height + 10, + WindowEnd: cm.TipState().Index.Height + 20, + Payout: taxAdjustedPayout(contractPayout), + ValidProofOutputs: []types.SiacoinOutput{ + {Address: addr, Value: contractPayout}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: addr, Value: contractPayout}, + }, + } + + // create a transaction with the contract + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + {ParentID: types.SiacoinOutputID(sce[0].ID), UnlockConditions: uc}, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(fc.Payout)}, // return the remainder to the wallet + }, + FileContracts: []types.FileContract{fc}, + Signatures: []types.TransactionSignature{ + { + ParentID: sce[0].ID, + PublicKeyIndex: 0, + Timelock: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, sce[0].ID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + txn.FileContractID(0).MissedOutputID(0) + + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + // mine until the contract expires to trigger the resolution event + blocksRemaining := int(fc.WindowEnd - cm.Tip().Height) + mineBlock(blocksRemaining, types.VoidAddress) + assertEvent(types.Hash256(txn.FileContractID(0).MissedOutputID(0)), wallet.EventTypeV1ContractResolution, contractPayout, types.ZeroCurrency, fc.WindowEnd+144) + } + + // v2 transaction + { + sce := spendableSiacoinUTXOs() + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1000)}, + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000))}, + }, + } + sigHash := cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + } + + // v2 contract resolution - expired + { + sce := spendableSiacoinUTXOs() + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract expires + mineBlock(int(fc.ExpirationHeight-cm.Tip().Height), types.VoidAddress) + + // this is kind of annoying because we have to keep the file contract + // proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + var fce types.V2FileContractElement + applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { + fce = ele + }) + for _, cau := range applied { + cau.UpdateElementProof(&fce.StateElement) + } + + resolutionTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &types.V2FileContractExpiration{}, + }, + }, + } + // broadcast the expire resolution + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the resolution + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + } + + // v2 contract resolution - storage proof + { + sce := spendableSiacoinUTXOs() + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + HostOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract proof window + mineBlock(int(fc.ProofHeight-cm.Tip().Height), types.VoidAddress) + + // this is even more annoying because we have to keep the file contract + // proof and the chain index proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + var fce types.V2FileContractElement + applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { + fce = ele + }) + // update its proof + for _, cau := range applied { + cau.UpdateElementProof(&fce.StateElement) + } + // get the proof index element + indexElement := applied[len(applied)-1].ChainIndexElement() + + resolutionTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &types.V2StorageProof{ + ProofIndex: indexElement, + // proof is nil since there's no data + }, + }, + }, + } + + // broadcast the expire resolution + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { + t.Fatal(err) + } + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(types.FileContractID(fce.ID).V2HostOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + } + + // v2 contract resolution - renewal + { + sces := spendableSiacoinUTXOs() + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract proof window + mineBlock(1, types.VoidAddress) + + // this is even more annoying because we have to keep the file contract + // proof and the chain index proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + var fce types.V2FileContractElement + applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { + fce = ele + }) + for _, cau := range applied { + cau.UpdateElementProof(&fce.StateElement) + } + + // finalize the contract + finalRevision := fce.V2FileContract + finalRevision.RevisionNumber = math.MaxUint64 + finalRevision.RenterSignature = types.Signature{} + finalRevision.HostSignature = types.Signature{} + // create a renewal + renewal := types.V2FileContractRenewal{ + FinalRevision: finalRevision, + NewContract: types.V2FileContract{ + RenterOutput: fc.RenterOutput, + ProofHeight: fc.ProofHeight + 10, + ExpirationHeight: fc.ExpirationHeight + 10, + + RenterPublicKey: fc.RenterPublicKey, + HostPublicKey: fc.HostPublicKey, + }, + } + + renewalSigHash := cm.TipState().RenewalSigHash(renewal) + renewalSig := pk.SignHash(renewalSigHash) + renewal.RenterSignature = renewalSig + renewal.HostSignature = renewalSig + + sces = spendableSiacoinUTXOs() + newContractValue := renterPayout.Add(cm.TipState().V2FileContractTax(renewal.NewContract)) + + // renewals can't have change outputs + setupTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: newContractValue}, + {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(newContractValue)}, + }, + } + setupSigHash := cm.TipState().InputSigHash(setupTxn) + setupTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(setupSigHash)} + + // create the renewal transaction + resolutionTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: setupTxn.EphemeralSiacoinOutput(0), + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &renewal, + }, + }, + } + resolutionTxnSigHash := cm.TipState().InputSigHash(resolutionTxn) + resolutionTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(resolutionTxnSigHash)} + + // broadcast the renewal + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{setupTxn, resolutionTxn}); err != nil { + t.Fatal(err) + } + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + } + + // v2 contract resolution - finalization + /*{ + sces := spendableSiacoinUTXOs() + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract proof window + mineBlock(1, types.VoidAddress) + + // this is even more annoying because we have to keep the file contract + // proof and the chain index proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + var fce types.V2FileContractElement + applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { + fce = ele + }) + for _, cau := range applied { + cau.UpdateElementProof(&fce.StateElement) + } + + // finalize the contract + fc = fce.V2FileContract + fc.RevisionNumber = types.MaxRevisionNumber + finalizationSigHash := cm.TipState().ContractSigHash(fc) + fc.RenterSignature = pk.SignHash(finalizationSigHash) + fc.HostSignature = pk.SignHash(finalizationSigHash) + finalization := types.V2FileContractFinalization(fc) + + // create the resolution transaction + finalizationTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &finalization, + }, + }, + } + + // broadcast the resolution + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{finalizationTxn}); err != nil { + t.Fatal(err) + } + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + }*/ + + // siafund claim + { + sfe, err := wm.AddressSiafundOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a transaction + txn := types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfe[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + ClaimAddress: addr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sfe[0].SiafundOutput.Value}, + }, + } + sigHash := cm.TipState().InputSigHash(txn) + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + claimValue := cm.TipState().SiafundPool + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) + } +} From eff9d6ff9283f0953374d51eaf179d038c521596 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jul 2024 14:39:36 -0700 Subject: [PATCH 227/630] wallet: enable broken finalization test --- wallet/wallet_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index e0d600b..a4d33aa 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3497,7 +3497,7 @@ func TestEventTypes(t *testing.T) { } // v2 contract resolution - finalization - /*{ + { sces := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3595,7 +3595,7 @@ func TestEventTypes(t *testing.T) { } mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - }*/ + } // siafund claim { From c293bb2c32413f43b6f03a7f042a24c4d11076c8 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jul 2024 14:42:24 -0700 Subject: [PATCH 228/630] wallet: fix lint --- wallet/wallet_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index a4d33aa..38d5bd2 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3018,10 +3018,8 @@ func TestEventTypes(t *testing.T) { } // miner payout event - { - mineBlock(1, addr) - assertEvent(types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) - } + mineBlock(1, addr) + assertEvent(types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) // mine until the payout matures mineBlock(int(cm.TipState().MaturityHeight()), types.VoidAddress) From 8a91e0461f4bc8b8852469eddab8f77675d1fece Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jul 2024 14:44:12 -0700 Subject: [PATCH 229/630] wallet: remove debug logging --- wallet/wallet.go | 3 --- wallet/wallet_test.go | 1 - 2 files changed, 4 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 4806bcf..4a8678a 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -3,7 +3,6 @@ package wallet import ( "encoding/json" "errors" - "log" "strconv" "time" @@ -315,7 +314,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.HostOutput.Address) { element := sces[types.FileContractID(fce.ID).V2HostOutputID()] - log.Println("HOST", element.ID, fce.V2FileContract.HostOutput.Address) addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, @@ -328,7 +326,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.RenterOutput.Address) { element := sces[types.FileContractID(fce.ID).V2RenterOutputID()] - log.Println("RENTER", element.ID, fce.V2FileContract.RenterOutput.Address) addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 38d5bd2..dcbba55 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -2959,7 +2959,6 @@ func TestEventTypes(t *testing.T) { for _, event := range events { if event.ID == id { - t.Log(id, eventType) if event.Type != eventType { t.Fatalf("expected %v event, got %v", eventType, event.Type) } else if event.MaturityHeight != maturityHeight { From c711d2348327abc9bc6f2ed795da23a578ac72a4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 12 Jul 2024 14:46:22 -0700 Subject: [PATCH 230/630] deps: update core and coreutils --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 859f9d8..229d67e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.3.0 - go.sia.tech/coreutils v0.1.0 + go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9 + go.sia.tech/coreutils v0.1.2 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 @@ -25,7 +25,7 @@ require ( go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 3ab1a66..c45e38c 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,12 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.3.0 h1:PDfAQh9z8PYD+oeVS7rS9SEnTMOZzwwFfAH45yktmko= go.sia.tech/core v0.3.0/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= +go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9 h1:rjIynknsIVM+BTN3lAVME3HxAWiACzOds+nABlqw33g= +go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= go.sia.tech/coreutils v0.1.0 h1:WQL7iT+jK1BiMx87bASXrZJZf4N2fbQkIOW8rS7wkh4= go.sia.tech/coreutils v0.1.0/go.mod h1:ybaFgewKXrlxFW71LqsyQlxjG6yWL6BSePrbZYnrprU= +go.sia.tech/coreutils v0.1.2 h1:U4WWSm4QOlcyVCDyPw66iDco0bFSYQGUwJLyu39VoSQ= +go.sia.tech/coreutils v0.1.2/go.mod h1:x1W8wuU1/z19FYT1wjeshe2hg18oXJOfjUXfGpTm6H8= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -32,6 +36,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= From 5c666fdeb18ff2c493b7e710a8b61e64948ee354 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 09:38:26 -0700 Subject: [PATCH 231/630] deps: update core and coreutils --- go.mod | 4 ++-- go.sum | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 229d67e..ec6cc1f 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9 - go.sia.tech/coreutils v0.1.2 + go.sia.tech/core v0.4.0 + go.sia.tech/coreutils v0.2.0 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index c45e38c..74b1f03 100644 --- a/go.sum +++ b/go.sum @@ -12,14 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.3.0 h1:PDfAQh9z8PYD+oeVS7rS9SEnTMOZzwwFfAH45yktmko= -go.sia.tech/core v0.3.0/go.mod h1:BMgT/reXtgv6XbDgUYTCPY7wSMbspDRDs7KMi1vL6Iw= -go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9 h1:rjIynknsIVM+BTN3lAVME3HxAWiACzOds+nABlqw33g= -go.sia.tech/core v0.3.2-0.20240710163411-d078002e33c9/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= -go.sia.tech/coreutils v0.1.0 h1:WQL7iT+jK1BiMx87bASXrZJZf4N2fbQkIOW8rS7wkh4= -go.sia.tech/coreutils v0.1.0/go.mod h1:ybaFgewKXrlxFW71LqsyQlxjG6yWL6BSePrbZYnrprU= -go.sia.tech/coreutils v0.1.2 h1:U4WWSm4QOlcyVCDyPw66iDco0bFSYQGUwJLyu39VoSQ= -go.sia.tech/coreutils v0.1.2/go.mod h1:x1W8wuU1/z19FYT1wjeshe2hg18oXJOfjUXfGpTm6H8= +go.sia.tech/core v0.4.0 h1:TlbVuiw1nk7wAybSvuZozRixnI4lpmcK0MVIlpJ9ApA= +go.sia.tech/core v0.4.0/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= +go.sia.tech/coreutils v0.2.0 h1:Tad3SPPyUM0gW/jwCxMFFgOa6YSkcwH0dsUv6muB4NA= +go.sia.tech/coreutils v0.2.0/go.mod h1:WpdAhWmtQ8gyqJfXnHhWLsnWn+j1eDkkWuvFVMDG4IU= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -34,8 +30,6 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= From 2fe5179a973aa3bb2e2a3e49baa1d369808caf22 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 09:38:48 -0700 Subject: [PATCH 232/630] wallet: add test structure --- wallet/wallet_test.go | 46 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index dcbba55..1f6914d 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3024,7 +3024,7 @@ func TestEventTypes(t *testing.T) { mineBlock(int(cm.TipState().MaturityHeight()), types.VoidAddress) // v1 transaction - { + t.Run("v1 transaction", func(t *testing.T) { sce := spendableSiacoinUTXOs() // v1 only supports unlock conditions @@ -3061,13 +3061,13 @@ func TestEventTypes(t *testing.T) { // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) - } + }) - // v1 contract resolution - only one type of resolution is supported. - // The only difference is `missed == true` or `missed == false` - { - sce := spendableSiacoinUTXOs() + t.Run("v1 contract resolution - missed", func(t *testing.T) { + // v1 contract resolution - only one type of resolution is supported. + // The only difference is `missed == true` or `missed == false` + sce := spendableSiacoinUTXOs() uc := types.StandardUnlockConditions(pk.PublicKey()) // create a storage contract @@ -3119,10 +3119,9 @@ func TestEventTypes(t *testing.T) { blocksRemaining := int(fc.WindowEnd - cm.Tip().Height) mineBlock(blocksRemaining, types.VoidAddress) assertEvent(types.Hash256(txn.FileContractID(0).MissedOutputID(0)), wallet.EventTypeV1ContractResolution, contractPayout, types.ZeroCurrency, fc.WindowEnd+144) - } + }) - // v2 transaction - { + t.Run("v2 transaction", func(t *testing.T) { sce := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3154,10 +3153,9 @@ func TestEventTypes(t *testing.T) { // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) - } + }) - // v2 contract resolution - expired - { + t.Run("v2 contract resolution - expired", func(t *testing.T) { sce := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3246,10 +3244,9 @@ func TestEventTypes(t *testing.T) { // mine a block to confirm the resolution mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - } + }) - // v2 contract resolution - storage proof - { + t.Run("v2 contract resolution - storage proof", func(t *testing.T) { sce := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3344,10 +3341,9 @@ func TestEventTypes(t *testing.T) { } mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.FileContractID(fce.ID).V2HostOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - } + }) - // v2 contract resolution - renewal - { + t.Run("v2 contract resolution - renewal", func(t *testing.T) { sces := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3491,10 +3487,11 @@ func TestEventTypes(t *testing.T) { } mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - } + }) + + t.Run("v2 contract resolution - finalization", func(t *testing.T) { + t.Skip("finalization currently errors with commitment hash mismatch") - // v2 contract resolution - finalization - { sces := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity @@ -3592,10 +3589,9 @@ func TestEventTypes(t *testing.T) { } mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - } + }) - // siafund claim - { + t.Run("siafund claim", func(t *testing.T) { sfe, err := wm.AddressSiafundOutputs(addr, 0, 100) if err != nil { t.Fatal(err) @@ -3631,5 +3627,5 @@ func TestEventTypes(t *testing.T) { // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) assertEvent(types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) - } + }) } From 4dc470558712abb5ea277c477d70d5fd43a9b78a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 09:45:42 -0700 Subject: [PATCH 233/630] sqlite: remove forced wallet index --- persist/sqlite/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 6204848..aac5ba8 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -657,7 +657,7 @@ func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet. } const eventsQuery = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts + FROM events ev INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN wallet_addresses wa ON (ea.address_id = wa.address_id) INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) From 3bb27e98a45b386d607faed9db3cbf721bf849a1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 09:53:38 -0700 Subject: [PATCH 234/630] sqlite: replace query placeholders with prepared statements --- persist/sqlite/peers.go | 27 +++++++++++++++++--------- persist/sqlite/sql.go | 27 -------------------------- persist/sqlite/wallet.go | 41 +++++++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 49 deletions(-) diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index a626908..daca41f 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -207,16 +207,25 @@ func (s *Store) Banned(peer string) (banned bool, _ error) { } err = s.transaction(func(tx *txn) error { - query := `SELECT net_cidr, expiration FROM syncer_bans WHERE net_cidr IN (` + queryPlaceHolders(len(checkSubnets)) + `) ORDER BY expiration DESC LIMIT 1` - - var subnet string - var expiration time.Time - err := tx.QueryRow(query, queryArgs(checkSubnets)...).Scan(&subnet, decode(&expiration)) - banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows - if err == nil && banned { - s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) + checkSubnetStmt, err := tx.Prepare(`SELECT expiration FROM syncer_bans WHERE net_cidr = $1 ORDER BY expiration DESC LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) } - return err + defer checkSubnetStmt.Close() + + for _, subnet := range checkSubnets { + var expiration time.Time + + err := checkSubnetStmt.QueryRow(subnet).Scan(decode(&expiration)) + banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to check ban status: %w", err) + } else if banned { + s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) + return nil + } + } + return nil }) if err != nil && !errors.Is(err, sql.ErrNoRows) { return false, fmt.Errorf("failed to check ban status: %w", err) diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index c9f990f..2ea2424 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "math/rand" - "strings" "time" _ "github.com/mattn/go-sqlite3" // import sqlite3 driver @@ -171,32 +170,6 @@ func (tx *txn) QueryRow(query string, args ...any) *row { return &row{r, tx.log.Named("row")} } -func queryPlaceHolders(n int) string { - if n == 0 { - return "" - } else if n == 1 { - return "?" - } - var b strings.Builder - b.Grow(((n - 1) * 2) + 1) // ?,? - for i := 0; i < n-1; i++ { - b.WriteString("?,") - } - b.WriteString("?") - return b.String() -} - -func queryArgs[T any](args []T) []any { - if len(args) == 0 { - return nil - } - out := make([]any, len(args)) - for i, arg := range args { - out[i] = arg - } - return out -} - // getDBVersion returns the current version of the database. func getDBVersion(db *sql.DB) (version int64) { // error is ignored -- the database may not have been initialized yet. diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index aac5ba8..95564b4 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -12,27 +12,42 @@ import ( ) func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.ID, eventIDs []int64) (map[int64][]types.Address, error) { - query := `SELECT ea.event_id, sa.sia_address + stmt, err := tx.Prepare(`SELECT sa.sia_address FROM event_addresses ea INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) -WHERE event_id IN (` + queryPlaceHolders(len(eventIDs)) + `) AND address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=?)` - - rows, err := tx.Query(query, append(queryArgs(eventIDs), id)...) +INNER JOIN wallet_addresses wa ON (ea.address_id = wa.address_id) +WHERE wa.wallet_id=? AND ea.event_id=?`) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + relevant := func(walletID wallet.ID, eventID int64) (addresses []types.Address, err error) { + rows, err := stmt.Query(walletID, eventID) + if err != nil { + return nil, fmt.Errorf("failed to query relevant addresses: %w", err) + } + defer rows.Close() + + for rows.Next() { + var address types.Address + if err := rows.Scan(decode(&address)); err != nil { + return nil, fmt.Errorf("failed to scan relevant address: %w", err) + } + addresses = append(addresses, address) + } + return addresses, rows.Err() } - defer rows.Close() relevantAddresses := make(map[int64][]types.Address) - for rows.Next() { - var eventID int64 - var address types.Address - if err := rows.Scan(&eventID, decode(&address)); err != nil { - return nil, fmt.Errorf("failed to scan relevant address: %w", err) + for _, eventID := range eventIDs { + addresses, err := relevant(id, eventID) + if err != nil { + return nil, err } - relevantAddresses[eventID] = append(relevantAddresses[eventID], address) + relevantAddresses[eventID] = addresses } - return relevantAddresses, rows.Err() + return relevantAddresses, nil } // WalletEvents returns the events relevant to a wallet, sorted by height descending. From c969c31d53510b42fb4568c8bccbc3837574d8ea Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 10:21:05 -0700 Subject: [PATCH 235/630] sqlite: add utxo query --- persist/sqlite/encoding.go | 1 + persist/sqlite/utxo.go | 72 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 persist/sqlite/utxo.go diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index abe5ab9..48b9e25 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -70,6 +70,7 @@ func (d *decodable) Scan(src any) error { case *[]types.Hash256: dec := types.NewBufDecoder(src) types.DecodeSlice(dec, v) + return dec.Err() default: return fmt.Errorf("cannot scan %T to %T", src, d.v) } diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go new file mode 100644 index 0000000..5bcf9bf --- /dev/null +++ b/persist/sqlite/utxo.go @@ -0,0 +1,72 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// SiacoinElement returns an unspent Siacoin UTXO by its ID. +func (s *Store) SiacoinElement(id types.SiacoinOutputID) (ele types.SiacoinElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address +FROM siacoin_elements se +INNER JOIN sia_addresses sa ON (se.address_id = sa.id) +WHERE se.id=$1 AND spent_index_id IS NULL` + + ele, err = scanSiacoinElement(tx.QueryRow(query, encode(id))) + if err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin element + if s.indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.LeafIndex}) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen + } + ele.MerkleProof = proof[0] + } + return nil + }) + if errors.Is(err, sql.ErrNoRows) { + err = wallet.ErrNotFound + } + return +} + +// SiafundElement returns an unspent Siafund UTXO by its ID. +func (s *Store) SiafundElement(id types.SiafundOutputID) (ele types.SiafundElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address +FROM siafund_elements se +INNER JOIN sia_addresses sa ON (se.address_id = sa.id) +WHERE se.id=$1 AND spent_index_id IS NULL` + + ele, err = scanSiafundElement(tx.QueryRow(query, encode(id))) + if err != nil { + return err + } + + // retrieve the merkle proofs for the siafund element + if s.indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.LeafIndex}) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen + } + ele.MerkleProof = proof[0] + } + return nil + }) + if errors.Is(err, sql.ErrNoRows) { + err = wallet.ErrNotFound + } + return +} From cef5d2f13f6c327ee6722541461c4bb0fcc2ad05 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 10:21:21 -0700 Subject: [PATCH 236/630] api,wallet: add siacoin and siafund utxo endpoints --- api/server.go | 32 ++++++++++++++++++++++++++++++++ wallet/manager.go | 13 +++++++++++++ wallet/wallet_test.go | 15 +++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/api/server.go b/api/server.go index 42c2e4a..0b236e4 100644 --- a/api/server.go +++ b/api/server.go @@ -74,6 +74,9 @@ type ( Events(eventIDs []types.Hash256) ([]wallet.Event, error) + SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) + SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + Reserve(ids []types.Hash256, duration time.Duration) error } ) @@ -723,6 +726,32 @@ func (s *server) eventsHandlerGET(jc jape.Context) { jc.Encode(events[0]) } +func (s *server) outputsSiacoinHandlerGET(jc jape.Context) { + var outputID types.SiacoinOutputID + if jc.DecodeParam("id", &outputID) != nil { + return + } + + output, err := s.wm.SiacoinElement(outputID) + if jc.Check("couldn't load output", err) != nil { + return + } + jc.Encode(output) +} + +func (s *server) outputsSiafundHandlerGET(jc jape.Context) { + var outputID types.SiafundOutputID + if jc.DecodeParam("id", &outputID) != nil { + return + } + + output, err := s.wm.SiafundElement(outputID) + if jc.Check("couldn't load output", err) != nil { + return + } + jc.Encode(output) +} + // NewServer returns an HTTP handler that serves the walletd API. func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { srv := server{ @@ -774,6 +803,9 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, + "GET /outputs/siacoin/:id": srv.outputsSiacoinHandlerGET, + "GET /outputs/siafund/:id": srv.outputsSiafundHandlerGET, + "GET /events/:id": srv.eventsHandlerGET, }) } diff --git a/wallet/manager.go b/wallet/manager.go index 6a5b428..dcda2ce 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -77,6 +77,9 @@ type ( Events(eventIDs []types.Hash256) ([]Event, error) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) + SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) + SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + SetIndexMode(IndexMode) error LastCommittedIndex() (types.ChainIndex, error) } @@ -283,6 +286,16 @@ func (m *Manager) IndexMode() IndexMode { return m.indexMode } +// SiacoinElement returns the unspent siacoin element with the given id. +func (m *Manager) SiacoinElement(id types.SiacoinOutputID) (types.SiacoinElement, error) { + return m.store.SiacoinElement(id) +} + +// SiafundElement returns the unspent siafund element with the given id. +func (m *Manager) SiafundElement(id types.SiafundOutputID) (types.SiafundElement, error) { + return m.store.SiafundElement(id) +} + // Close closes the wallet manager. func (m *Manager) Close() error { m.tg.Stop() diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 1f6914d..b1bd209 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1268,6 +1268,13 @@ func TestFullIndex(t *testing.T) { if err != nil { t.Fatal(err) } + for _, se := range utxos { + if sce, err := wm.SiacoinElement(types.SiacoinOutputID(se.ID)); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(sce, se) { + t.Fatalf("expected %v, got %v", se, sce) + } + } policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) txn := types.V2Transaction{ @@ -1319,6 +1326,14 @@ func TestFullIndex(t *testing.T) { t.Fatal(err) } + for _, se := range sf { + if sfe, err := wm.SiafundElement(types.SiafundOutputID(se.ID)); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(sfe, se) { + t.Fatalf("expected %v, got %v", se, sfe) + } + } + // send the siafunds to the first address policy = types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk2.PublicKey())) txn = types.V2Transaction{ From 1eb029e8465e813f7835794a61048a78002d4ce2 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 16 Jul 2024 10:23:38 -0700 Subject: [PATCH 237/630] sqlite: increase timeout for flaky test --- persist/sqlite/peers_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index eca3a77..0ef91dc 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -95,14 +95,14 @@ func TestBanPeer(t *testing.T) { } // ban the peer - ps.Ban(peer, time.Second, "test") + ps.Ban(peer, 5*time.Second, "test") if banned, err := ps.Banned(peer); err != nil || !banned { t.Fatal("expected peer to be banned", err) } // wait for the ban to expire - time.Sleep(time.Second) + time.Sleep(5 * time.Second) if banned, err := ps.Banned(peer); err != nil || banned { t.Fatal("expected peer to not be banned", err) From 814e6e0c1b9af22a80cd505b8476e0bb68deb989 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 22 Jul 2024 07:59:01 -0700 Subject: [PATCH 238/630] deps: update core and coreutils --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ec6cc1f..e545068 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.0 - go.sia.tech/coreutils v0.2.0 + go.sia.tech/core v0.4.1 + go.sia.tech/coreutils v0.2.1 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 74b1f03..99222bd 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.0 h1:TlbVuiw1nk7wAybSvuZozRixnI4lpmcK0MVIlpJ9ApA= -go.sia.tech/core v0.4.0/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= -go.sia.tech/coreutils v0.2.0 h1:Tad3SPPyUM0gW/jwCxMFFgOa6YSkcwH0dsUv6muB4NA= -go.sia.tech/coreutils v0.2.0/go.mod h1:WpdAhWmtQ8gyqJfXnHhWLsnWn+j1eDkkWuvFVMDG4IU= +go.sia.tech/core v0.4.1 h1:yawkyvr7mHYKWXa8RsHAPriLtJdvDQzqXgq4/hHqjHQ= +go.sia.tech/core v0.4.1/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= +go.sia.tech/coreutils v0.2.1 h1:XReDUlSt9DM2P2hOEsYtdd8IdHCIyElkXbhQEVm2boY= +go.sia.tech/coreutils v0.2.1/go.mod h1:WvdQ0xUb4QXuMip3rZfZph5TSMGWJ7wNZ6WicEvL+Jc= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 36a0a985c224a7b6db3fe1ab97b8c56507919adc Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 22 Jul 2024 07:59:17 -0700 Subject: [PATCH 239/630] wallet: enable finalization event test --- wallet/wallet_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index b1bd209..09b2f7b 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3505,8 +3505,6 @@ func TestEventTypes(t *testing.T) { }) t.Run("v2 contract resolution - finalization", func(t *testing.T) { - t.Skip("finalization currently errors with commitment hash mismatch") - sces := spendableSiacoinUTXOs() // using the UnlockConditions policy for brevity From f6662da9f74abdbd68861352c83197c4901f6635 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 31 Jul 2024 09:09:03 -0700 Subject: [PATCH 240/630] api: add consensus updates endpoints --- api/api.go | 21 +++++++++++ api/api_test.go | 67 +++++++++++++++++++++++++++++++++++ api/client.go | 92 +++++++++++++++++++++++++++++++++++++++++++------ api/server.go | 71 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 237 insertions(+), 14 deletions(-) diff --git a/api/api.go b/api/api.go index 45a6a02..2e078b2 100644 --- a/api/api.go +++ b/api/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/walletd/wallet" ) @@ -102,3 +103,23 @@ type RescanResponse struct { StartTime time.Time `json:"startTime"` Error *string `json:"error,omitempty"` } + +// An ApplyUpdate is a consensus update that was applied to the best chain. +type ApplyUpdate struct { + Update consensus.ApplyUpdate `json:"update"` + State consensus.State `json:"state"` + Block types.Block `json:"block"` +} + +// A RevertUpdate is a consensus update that was reverted from the best chain. +type RevertUpdate struct { + Update consensus.RevertUpdate `json:"update"` + State consensus.State `json:"state"` + Block types.Block `json:"block"` +} + +// ConsensusUpdatesResponse is the response type for /consensus/updates/:index. +type ConsensusUpdatesResponse struct { + Applied []ApplyUpdate `json:"applied"` + Reverted []RevertUpdate `json:"reverted"` +} diff --git a/api/api_test.go b/api/api_test.go index 5f0a077..c77434a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -15,6 +15,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/jape" @@ -1214,3 +1215,69 @@ func TestP2P(t *testing.T) { t.Fatal(err) } } + +func TestConsensusUpdates(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c, shutdown := runServer(cm, nil, wm) + defer shutdown() + + for i := 0; i < 10; i++ { + b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + waitForBlock(t, cm, ws) + + reverted, applied, err := c.ConsensusUpdates(types.ChainIndex{}, 10) + if err != nil { + t.Fatal(err) + } else if len(reverted) != 0 { + t.Fatal("expected no reverted blocks") + } else if len(applied) != 11 { // genesis + 10 mined blocks (chain manager off-by-one) + t.Fatalf("expected 11 applied blocks, got %v", len(applied)) + } + + for i, cau := range applied { + // using i for height since we're testing the update contents + expected, ok := cm.BestIndex(uint64(i)) + if !ok { + t.Fatalf("failed to get expected index for block %v", i) + } else if cau.State.Index != expected { + t.Fatalf("expected index %v, got %v", expected, cau.State.Index) + } else if cau.State.Network.Name != n.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. + t.Fatalf("expected network to be %q, got %q", n.Name, cau.State.Network.Name) + } + } +} diff --git a/api/client.go b/api/client.go index 6000dc2..3cd1685 100644 --- a/api/client.go +++ b/api/client.go @@ -2,10 +2,12 @@ package api import ( "fmt" + "sync" "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/jape" "go.sia.tech/walletd/wallet" ) @@ -13,7 +15,23 @@ import ( // A Client provides methods for interacting with a walletd API server. type Client struct { c jape.Client - n *consensus.Network // for ConsensusTipState + + mu sync.Mutex // protects n + n *consensus.Network +} + +func (c *Client) getNetwork() (*consensus.Network, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.n == nil { + var err error + c.n, err = c.ConsensusNetwork() + if err != nil { + return nil, err + } + } + return c.n, nil } // State returns information about the current state of the walletd daemon. @@ -35,6 +53,13 @@ func (c *Client) TxpoolTransactions() (txns []types.Transaction, v2txns []types. return resp.Transactions, resp.V2Transactions, err } +// TxpoolParents returns the parents of a transaction that are currently in the +// transaction pool. +func (c *Client) TxpoolParents(txn types.Transaction) (resp []types.Transaction, err error) { + err = c.c.POST("/txpool/parents", txn, &resp) + return +} + // TxpoolFee returns the recommended fee (per weight unit) to ensure a high // probability of inclusion in the next block. func (c *Client) TxpoolFee() (resp types.Currency, err error) { @@ -49,22 +74,67 @@ func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { return } -// ConsensusTip returns the current tip index. -func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { - err = c.c.GET("/consensus/tip", &resp) +// ConsensusIndex returns the consensus index at the specified height. +func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { + err = c.c.GET(fmt.Sprintf("/consensus/index/%d", height), &resp) return } +// ConsensusUpdates returns at most n consensus updates that have occurred since +// the specified index +func (c *Client) ConsensusUpdates(index types.ChainIndex, limit int) ([]chain.RevertUpdate, []chain.ApplyUpdate, error) { + // index.String() is a short-hand representation. We need the full text + indexBuf, err := index.MarshalText() + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal index: %w", err) + } + + var resp ConsensusUpdatesResponse + if err := c.c.GET(fmt.Sprintf("/consensus/updates/%s?limit=%d", indexBuf, limit), &resp); err != nil { + return nil, nil, err + } + + network, err := c.getNetwork() + if err != nil { + return nil, nil, fmt.Errorf("failed to get network metadata: %w", err) + } + + reverted := make([]chain.RevertUpdate, 0, len(resp.Reverted)) + for _, u := range resp.Reverted { + revert := chain.RevertUpdate{ + RevertUpdate: u.Update, + State: u.State, + Block: u.Block, + } + revert.State.Network = network + reverted = append(reverted, revert) + } + + applied := make([]chain.ApplyUpdate, 0, len(resp.Applied)) + for _, u := range resp.Applied { + apply := chain.ApplyUpdate{ + ApplyUpdate: u.Update, + State: u.State, + Block: u.Block, + } + apply.State.Network = network + applied = append(applied, apply) + } + return reverted, applied, nil +} + // ConsensusTipState returns the current tip state. func (c *Client) ConsensusTipState() (resp consensus.State, err error) { - if c.n == nil { - c.n, err = c.ConsensusNetwork() - if err != nil { - return - } + if err = c.c.GET("/consensus/tipstate", &resp); err != nil { + return } - err = c.c.GET("/consensus/tipstate", &resp) - resp.Network = c.n + resp.Network, err = c.getNetwork() + return +} + +// ConsensusTip returns the current tip index. +func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { + err = c.c.GET("/consensus/tip", &resp) return } diff --git a/api/server.go b/api/server.go index 0b236e4..e5f6677 100644 --- a/api/server.go +++ b/api/server.go @@ -15,6 +15,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/build" "go.sia.tech/walletd/wallet" @@ -23,6 +24,8 @@ import ( type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { + UpdatesSince(types.ChainIndex, int) ([]chain.RevertUpdate, []chain.ApplyUpdate, error) + BestIndex(height uint64) (types.ChainIndex, bool) TipState() consensus.State AddBlocks([]types.Block) error @@ -120,6 +123,56 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { jc.Encode(s.cm.TipState()) } +func (s *server) consensusIndexHeightHandler(jc jape.Context) { + var height uint64 + if jc.DecodeParam("height", &height) != nil { + return + } + index, ok := s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + jc.Encode(index) +} + +func (s *server) consensusUpdatesIndexHandler(jc jape.Context) { + var index types.ChainIndex + if jc.DecodeParam("index", &index) != nil { + return + } + + limit := 10 + if jc.DecodeForm("limit", &limit) != nil { + return + } else if limit <= 0 || limit > 100 { + jc.Error(errors.New("limit must be between 0 and 100"), http.StatusBadRequest) + return + } + + reverted, applied, err := s.cm.UpdatesSince(index, limit) + if jc.Check("couldn't get updates", err) != nil { + return + } + + var res ConsensusUpdatesResponse + for _, ru := range reverted { + res.Reverted = append(res.Reverted, RevertUpdate{ + Update: ru.RevertUpdate, + State: ru.State, + Block: ru.Block, + }) + } + for _, au := range applied { + res.Applied = append(res.Applied, ApplyUpdate{ + Update: au.ApplyUpdate, + State: au.State, + Block: au.Block, + }) + } + jc.Encode(res) +} + func (s *server) syncerPeersHandler(jc jape.Context) { var peers []GatewayPeer for _, p := range s.s.Peers() { @@ -173,6 +226,15 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { } } +func (s *server) txpoolParentsHandler(jc jape.Context) { + var txn types.Transaction + if jc.Decode(&txn) != nil { + return + } + + jc.Encode(s.cm.UnconfirmedParents(txn)) +} + func (s *server) txpoolTransactionsHandler(jc jape.Context) { jc.Encode(TxpoolTransactionsResponse{ Transactions: s.cm.PoolTransactions(), @@ -765,14 +827,17 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { return jape.Mux(map[string]jape.Handler{ "GET /state": srv.stateHandler, - "GET /consensus/network": srv.consensusNetworkHandler, - "GET /consensus/tip": srv.consensusTipHandler, - "GET /consensus/tipstate": srv.consensusTipStateHandler, + "GET /consensus/network": srv.consensusNetworkHandler, + "GET /consensus/tip": srv.consensusTipHandler, + "GET /consensus/tipstate": srv.consensusTipStateHandler, + "GET /consensus/updates/:index": srv.consensusUpdatesIndexHandler, + "GET /consensus/index/:height": srv.consensusIndexHeightHandler, "GET /syncer/peers": srv.syncerPeersHandler, "POST /syncer/connect": srv.syncerConnectHandler, "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, + "POST /txpool/parents": srv.txpoolParentsHandler, "GET /txpool/transactions": srv.txpoolTransactionsHandler, "GET /txpool/fee": srv.txpoolFeeHandler, "POST /txpool/broadcast": srv.txpoolBroadcastHandler, From 2a0bc68b0d565095fd8472c80689143a111d6f6a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 7 Aug 2024 15:03:46 -0700 Subject: [PATCH 241/630] cmd: refactor node startup --- cmd/walletd/config.go | 4 +- cmd/walletd/main.go | 60 +++++------- cmd/walletd/node.go | 218 ++++++++++++++++------------------------- cmd/walletd/testnet.go | 59 ----------- cmd/walletd/web.go | 25 ----- config/config.go | 15 ++- 6 files changed, 121 insertions(+), 260 deletions(-) delete mode 100644 cmd/walletd/testnet.go delete mode 100644 cmd/walletd/web.go diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go index adfadfd..3b93294 100644 --- a/cmd/walletd/config.go +++ b/cmd/walletd/config.go @@ -185,9 +185,9 @@ func setAdvancedConfig() { setListenAddress("HTTP Address", &cfg.HTTP.Address) fmt.Println("") - fmt.Println("The gateway address is used to connect to the Sia network.") + fmt.Println("The syncer address is used to connect to the Sia network.") fmt.Println("It should be reachable from other Sia nodes.") - setListenAddress("Gateway Address", &cfg.Consensus.GatewayAddress) + setListenAddress("Syncer Address", &cfg.Syncer.Address) fmt.Println("") fmt.Println("Index mode determines how much of the blockchain to store.") diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index a0596f5..a7b90d2 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "net" "os" "os/signal" "path/filepath" @@ -19,7 +18,6 @@ import ( "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "golang.org/x/term" "gopkg.in/yaml.v3" "lukechampine.com/flagg" ) @@ -60,10 +58,12 @@ var cfg = config.Config{ Address: "localhost:9980", Password: os.Getenv("WALLETD_API_PASSWORD"), }, + Syncer: config.Syncer{ + Address: ":9981", + Bootstrap: true, + }, Consensus: config.Consensus{ - Network: "mainnet", - GatewayAddress: ":9981", - Bootstrap: true, + Network: "mainnet", }, Index: config.Index{ Mode: wallet.IndexModePersonal, @@ -90,16 +90,20 @@ func check(context string, err error) { } } -func getAPIPassword() string { - apiPassword := cfg.HTTP.Password - if apiPassword == "" { - fmt.Print("Enter API password: ") - pw, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - check("Could not read API password:", err) - apiPassword = string(pw) +func mustSetAPIPassword() { + // retry until a valid API password is entered + for { + fmt.Println("Please choose a password to unlock walletd.") + fmt.Println("This password will be required to access the admin UI in your web browser.") + fmt.Println("(The password must be at least 4 characters.)") + cfg.HTTP.Password = readPasswordInput("Enter password") + if len(cfg.HTTP.Password) >= 4 { + break + } + + fmt.Println(wrapANSI("\033[31m", "Password must be at least 4 characters!", "\033[0m")) + fmt.Println("") } - return apiPassword } // tryLoadConfig loads the config file specified by the WALLETD_CONFIG_FILE. If @@ -192,10 +196,10 @@ func main() { rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") - rootCmd.StringVar(&cfg.Consensus.GatewayAddress, "addr", cfg.Consensus.GatewayAddress, "p2p address to listen on") + rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") - rootCmd.BoolVar(&cfg.Consensus.EnableUPNP, "upnp", cfg.Consensus.EnableUPNP, "attempt to forward ports and discover IP with UPnP") - rootCmd.BoolVar(&cfg.Consensus.Bootstrap, "bootstrap", cfg.Consensus.Bootstrap, "attempt to bootstrap the network") + rootCmd.BoolVar(&cfg.Syncer.EnableUPnP, "upnp", cfg.Syncer.EnableUPnP, "attempt to forward ports and discover IP with UPnP") + rootCmd.BoolVar(&cfg.Syncer.Bootstrap, "bootstrap", cfg.Syncer.Bootstrap, "attempt to bootstrap the network") rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (personal, full, none)") rootCmd.IntVar(&cfg.Index.BatchSize, "index.batch", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") @@ -232,11 +236,7 @@ func main() { stdoutFatalError("failed to create directory: " + err.Error()) } - apiPassword := getAPIPassword() - l, err := net.Listen("tcp", cfg.HTTP.Address) - if err != nil { - stdoutFatalError("failed to start HTTP server: " + err.Error()) - } + mustSetAPIPassword() var logCores []zapcore.Core if cfg.Log.StdOut.Enabled { @@ -305,18 +305,9 @@ func main() { log.Fatal("failed to parse index mode", zap.Error(err)) } - n, err := newNode(cfg, log) - if err != nil { - log.Fatal("failed to create node", zap.Error(err)) + if err := runNode(ctx, cfg, log); err != nil { + log.Fatal("failed to run node", zap.Error(err)) } - defer n.Close() - - stop := n.Start() - go startWeb(l, n, apiPassword) - log.Info("walletd started", zap.String("version", build.Version()), zap.String("network", cfg.Consensus.Network), zap.String("commit", build.Commit()), zap.Time("buildDate", build.Time())) - <-ctx.Done() - log.Info("shutting down") - stop() case versionCmd: if len(cmd.Args()) != 0 { cmd.Usage() @@ -357,7 +348,8 @@ func main() { log.Fatal(err) } - c := api.NewClient("http://"+cfg.HTTP.Address+"/api", getAPIPassword()) + mustSetAPIPassword() + c := api.NewClient("http://"+cfg.HTTP.Address+"/api", cfg.HTTP.Password) runCPUMiner(c, minerAddr, minerBlocks) } } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index f12be24..62aaea5 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "net" + "net/http" "path/filepath" "strconv" + "strings" "time" "go.sia.tech/core/consensus" @@ -15,148 +17,88 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" + "go.sia.tech/jape" + "go.sia.tech/walletd/api" + "go.sia.tech/walletd/build" "go.sia.tech/walletd/config" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" + "go.sia.tech/web/walletd" "go.uber.org/zap" "lukechampine.com/upnp" ) -var mainnetBootstrap = []string{ - "108.227.62.195:9981", - "139.162.81.190:9991", - "144.217.7.188:9981", - "147.182.196.252:9981", - "15.235.85.30:9981", - "167.235.234.84:9981", - "173.235.144.230:9981", - "198.98.53.144:7791", - "199.27.255.169:9981", - "2.136.192.200:9981", - "213.159.50.43:9981", - "24.253.116.61:9981", - "46.249.226.103:9981", - "5.165.236.113:9981", - "5.252.226.131:9981", - "54.38.120.222:9981", - "62.210.136.25:9981", - "63.135.62.123:9981", - "65.21.93.245:9981", - "75.165.149.114:9981", - "77.51.200.125:9981", - "81.6.58.121:9981", - "83.194.193.156:9981", - "84.39.246.63:9981", - "87.99.166.34:9981", - "91.214.242.11:9981", - "93.105.88.181:9981", - "93.180.191.86:9981", - "94.130.220.162:9981", -} - -var zenBootstrap = []string{ - "147.135.16.182:9881", - "147.135.39.109:9881", - "51.81.208.10:9881", -} - -var anagamiBootstrap = []string{ - "147.135.16.182:9781", - "98.180.237.163:9981", - "98.180.237.163:11981", - "98.180.237.163:10981", - "94.130.139.59:9801", - "84.86.11.238:9801", - "69.131.14.86:9981", - "68.108.89.92:9981", - "62.30.63.93:9981", - "46.173.150.154:9111", - "195.252.198.117:9981", - "174.174.206.214:9981", - "172.58.232.54:9981", - "172.58.229.31:9981", - "172.56.200.90:9981", - "172.56.162.155:9981", - "163.172.13.180:9981", - "154.47.25.194:9981", - "138.201.19.49:9981", - "100.34.20.44:9981", -} - -type node struct { - chainStore *coreutils.BoltChainDB - cm *chain.Manager - - store *sqlite.Store - s *syncer.Syncer - wm *wallet.Manager - - Start func() (stop func()) -} - -// Close shuts down the node and closes its database. -func (n *node) Close() error { - n.wm.Close() - n.chainStore.Close() - return n.store.Close() +func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + d, err := upnp.Discover(ctx) + if err != nil { + return "", fmt.Errorf("couldn't discover UPnP router: %w", err) + } else if !d.IsForwarded(port, "TCP") { + if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { + log.Debug("couldn't forward port", zap.Error(err)) + } else { + log.Debug("upnp: forwarded p2p port", zap.Uint16("port", port)) + } + } + return d.ExternalIP() } -func newNode(cfg config.Config, log *zap.Logger) (*node, error) { +func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string switch cfg.Consensus.Network { case "mainnet": network, genesisBlock = chain.Mainnet() - bootstrapPeers = mainnetBootstrap + bootstrapPeers = syncer.MainnetBootstrapPeers case "zen": network, genesisBlock = chain.TestnetZen() - bootstrapPeers = zenBootstrap - case "anagami": - network, genesisBlock = TestnetAnagami() - bootstrapPeers = anagamiBootstrap + bootstrapPeers = syncer.ZenBootstrapPeers default: - return nil, errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") + return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) if err != nil { - return nil, fmt.Errorf("failed to open consensus database: %w", err) + return fmt.Errorf("failed to open consensus database: %w", err) } + defer bdb.Close() + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { - return nil, fmt.Errorf("failed to create chain store: %w", err) + return fmt.Errorf("failed to create chain store: %w", err) } cm := chain.NewManager(dbstore, tipState) - l, err := net.Listen("tcp", cfg.Consensus.GatewayAddress) + syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) + if err != nil { + return fmt.Errorf("failed to listen on %q: %w", cfg.Syncer.Address, err) + } + defer syncerListener.Close() + + httpListener, err := net.Listen("tcp", cfg.HTTP.Address) if err != nil { - return nil, err + return fmt.Errorf("failed to listen on %q: %w", cfg.HTTP.Address, err) } - syncerAddr := l.Addr().String() - if cfg.Consensus.EnableUPNP { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if d, err := upnp.Discover(ctx); err != nil { - log.Debug("couldn't discover UPnP router", zap.Error(err)) + defer httpListener.Close() + + syncerAddr := syncerListener.Addr().String() + if cfg.Syncer.EnableUPnP { + _, portStr, _ := net.SplitHostPort(cfg.Syncer.Address) + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return fmt.Errorf("failed to parse syncer port: %w", err) + } + + ip, err := setupUPNP(context.Background(), uint16(port), log) + if err != nil { + log.Warn("failed to set up UPnP", zap.Error(err)) } else { - _, portStr, _ := net.SplitHostPort(cfg.Consensus.GatewayAddress) - port, _ := strconv.Atoi(portStr) - if !d.IsForwarded(uint16(port), "TCP") { - if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { - log.Debug("couldn't forward port", zap.Error(err)) - } else { - log.Debug("upnp: forwarded p2p port", zap.Int("port", port)) - } - } - if ip, err := d.ExternalIP(); err != nil { - log.Debug("couldn't determine external IP", zap.Error(err)) - } else { - log.Debug("external IP is", zap.String("ip", ip)) - syncerAddr = net.JoinHostPort(ip, portStr) - } + syncerAddr = net.JoinHostPort(ip, portStr) } } + // peers will reject us if our hostname is empty or unspecified, so use loopback host, port, _ := net.SplitHostPort(syncerAddr) if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { @@ -165,25 +107,26 @@ func newNode(cfg config.Config, log *zap.Logger) (*node, error) { store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { - return nil, fmt.Errorf("failed to open wallet database: %w", err) + return fmt.Errorf("failed to open wallet database: %w", err) } + defer store.Close() - if cfg.Consensus.Bootstrap { + if cfg.Syncer.Bootstrap { for _, peer := range bootstrapPeers { if err := store.AddPeer(peer); err != nil { - return nil, fmt.Errorf("failed to add bootstrap peer '%s': %w", peer, err) + return fmt.Errorf("failed to add bootstrap peer %q: %w", peer, err) } } - for _, peer := range cfg.Consensus.Peers { + for _, peer := range cfg.Syncer.Peers { if err := store.AddPeer(peer); err != nil { - return nil, fmt.Errorf("failed to add peer '%s': %w", peer, err) + return fmt.Errorf("failed to add peer %q: %w", peer, err) } } } ps, err := sqlite.NewPeerStore(store) if err != nil { - return nil, fmt.Errorf("failed to create peer store: %w", err) + return fmt.Errorf("failed to create peer store: %w", err) } header := gateway.Header{ @@ -192,28 +135,33 @@ func newNode(cfg config.Config, log *zap.Logger) (*node, error) { NetAddress: syncerAddr, } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) + s := syncer.New(syncerListener, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) + defer s.Close() + go s.Run(ctx) + wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(cfg.Index.Mode), wallet.WithSyncBatchSize(cfg.Index.BatchSize)) if err != nil { - return nil, fmt.Errorf("failed to create wallet manager: %w", err) + return fmt.Errorf("failed to create wallet manager: %w", err) } - return &node{ - chainStore: bdb, - cm: cm, - store: store, - s: s, - wm: wm, - Start: func() func() { - ch := make(chan struct{}) - go func() { - s.Run() - close(ch) - }() - return func() { - l.Close() - <-ch - bdb.Close() + + api := jape.BasicAuth(cfg.HTTP.Password)(api.NewServer(cm, s, wm)) + web := walletd.Handler() + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api") { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") + api.ServeHTTP(w, r) + return } - }, - }, nil + web.ServeHTTP(w, r) + }), + ReadTimeout: 10 * time.Second, + } + defer server.Close() + go server.Serve(httpListener) + + log.Info("node started", zap.Stringer("syncer", syncerListener.Addr()), zap.Stringer("http", httpListener.Addr()), zap.String("version", build.Version()), zap.String("commit", build.Commit())) + <-ctx.Done() + log.Info("shutting down") + return nil } diff --git a/cmd/walletd/testnet.go b/cmd/walletd/testnet.go deleted file mode 100644 index dc9c706..0000000 --- a/cmd/walletd/testnet.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import ( - "time" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" -) - -// TestnetAnagami returns the chain parameters and genesis block for the "Anagami" -// testnet chain. -func TestnetAnagami() (*consensus.Network, types.Block) { - n := &consensus.Network{ - Name: "anagami", - - InitialCoinbase: types.Siacoins(300000), - MinimumCoinbase: types.Siacoins(300000), - InitialTarget: types.BlockID{3: 1}, - } - - n.HardforkDevAddr.Height = 1 - n.HardforkDevAddr.OldAddress = types.Address{} - n.HardforkDevAddr.NewAddress = types.Address{} - - n.HardforkTax.Height = 2 - - n.HardforkStorageProof.Height = 3 - - n.HardforkOak.Height = 5 - n.HardforkOak.FixHeight = 8 - n.HardforkOak.GenesisTimestamp = time.Unix(1702300000, 0) // Dec 11, 2023 @ 13:06 GMT - - n.HardforkASIC.Height = 13 - n.HardforkASIC.OakTime = 10 * time.Minute - n.HardforkASIC.OakTarget = n.InitialTarget - - n.HardforkFoundation.Height = 21 - n.HardforkFoundation.PrimaryAddress, _ = types.ParseAddress("addr:5949fdf56a7c18ba27f6526f22fd560526ce02a1bd4fa3104938ab744b69cf63b6b734b8341f") - n.HardforkFoundation.FailsafeAddress = n.HardforkFoundation.PrimaryAddress - - n.HardforkV2.AllowHeight = 2016 // ~2 weeks in - n.HardforkV2.RequireHeight = 2016 + 288 // ~2 days later - - b := types.Block{ - Timestamp: n.HardforkOak.GenesisTimestamp, - Transactions: []types.Transaction{{ - SiacoinOutputs: []types.SiacoinOutput{{ - Address: n.HardforkFoundation.PrimaryAddress, - Value: types.Siacoins(1).Mul64(1e12), - }}, - SiafundOutputs: []types.SiafundOutput{{ - Address: n.HardforkFoundation.PrimaryAddress, - Value: 10000, - }}, - }}, - } - - return n, b -} diff --git a/cmd/walletd/web.go b/cmd/walletd/web.go deleted file mode 100644 index c6de95c..0000000 --- a/cmd/walletd/web.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "net" - "net/http" - "strings" - - "go.sia.tech/jape" - "go.sia.tech/walletd/api" - "go.sia.tech/web/walletd" -) - -func startWeb(l net.Listener, node *node, password string) error { - renter := api.NewServer(node.cm, node.s, node.wm) - api := jape.BasicAuth(password)(renter) - web := walletd.Handler() - return http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api") { - r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") - api.ServeHTTP(w, r) - return - } - web.ServeHTTP(w, r) - })) -} diff --git a/config/config.go b/config/config.go index b8d3f14..7372714 100644 --- a/config/config.go +++ b/config/config.go @@ -9,13 +9,17 @@ type ( Password string `yaml:"password,omitempty"` } + // Syncer contains the configuration for the consensus set syncer. + Syncer struct { + Address string `yaml:"address,omitempty"` + Bootstrap bool `yaml:"bootstrap,omitempty"` + EnableUPnP bool `yaml:"enableUPnP,omitempty"` + Peers []string `yaml:"peers,omitempty"` + } + // Consensus contains the configuration for the consensus set. Consensus struct { - Network string `yaml:"network,omitempty"` - GatewayAddress string `yaml:"gatewayAddress,omitempty"` - Bootstrap bool `yaml:"bootstrap,omitempty"` - Peers []string `yaml:"peers,omitempty"` - EnableUPNP bool `yaml:"enableUPnP,omitempty"` + Network string `yaml:"network,omitempty"` } // Index contains the configuration for the blockchain indexer @@ -56,6 +60,7 @@ type ( HTTP HTTP `yaml:"http,omitempty"` Consensus Consensus `yaml:"consensus,omitempty"` + Syncer Syncer `yaml:"syncer,omitempty"` Log Log `yaml:"log,omitempty"` Index Index `yaml:"index,omitempty"` } From 93cb44cacfa19c694f510d1e467bc3b366e35720 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 7 Aug 2024 15:03:57 -0700 Subject: [PATCH 242/630] deps: update core and coreutils --- api/api_test.go | 6 ++++-- cmd/walletd/main.go | 4 ++++ cmd/walletd/node.go | 1 + go.mod | 6 +++--- go.sum | 12 ++++++------ 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index c77434a..c2c2160 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -940,7 +940,8 @@ func TestP2P(t *testing.T) { UniqueID: gateway.GenerateUniqueID(), NetAddress: l1.Addr().String(), }) - go s1.Run() + go s1.Run(context.Background()) + defer s1.Close() c1, shutdown := runServer(cm1, s1, wm1) defer shutdown() w1, err := c1.AddWallet(api.WalletUpdateRequest{Name: "primary"}) @@ -983,7 +984,8 @@ func TestP2P(t *testing.T) { UniqueID: gateway.GenerateUniqueID(), NetAddress: l2.Addr().String(), }, syncer.WithLogger(zaptest.NewLogger(t))) - go s2.Run() + go s2.Run(context.Background()) + defer s2.Close() c2, shutdown2 := runServer(cm2, s2, wm2) defer shutdown2() diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index a7b90d2..0b7f5ca 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -91,6 +91,10 @@ func check(context string, err error) { } func mustSetAPIPassword() { + if cfg.HTTP.Password != "" { + return + } + // retry until a valid API password is entered for { fmt.Println("Please choose a password to unlock walletd.") diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 62aaea5..dedde8f 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -143,6 +143,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { if err != nil { return fmt.Errorf("failed to create wallet manager: %w", err) } + defer wm.Close() api := jape.BasicAuth(cfg.HTTP.Password)(api.NewServer(cm, s, wm)) web := walletd.Handler() diff --git a/go.mod b/go.mod index e545068..df040fb 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.1 - go.sia.tech/coreutils v0.2.1 + go.sia.tech/core v0.4.2 + go.sia.tech/coreutils v0.2.2 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 @@ -26,6 +26,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.23.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 99222bd..5f6ec93 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.1 h1:yawkyvr7mHYKWXa8RsHAPriLtJdvDQzqXgq4/hHqjHQ= -go.sia.tech/core v0.4.1/go.mod h1:6dN3J2GDX+f8H2p82MJ7V4BFdnmgoHAiovfmBD/F1Hg= -go.sia.tech/coreutils v0.2.1 h1:XReDUlSt9DM2P2hOEsYtdd8IdHCIyElkXbhQEVm2boY= -go.sia.tech/coreutils v0.2.1/go.mod h1:WvdQ0xUb4QXuMip3rZfZph5TSMGWJ7wNZ6WicEvL+Jc= +go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= +go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= +go.sia.tech/coreutils v0.2.2 h1:WWbuk4lEd5SVB/LoBsPG6+heZN2olZT4yOkRZd3pvpg= +go.sia.tech/coreutils v0.2.2/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -37,8 +37,8 @@ golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= From b72f27fbd3105eac4f9efa0e96271a09fc87947b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 9 Aug 2024 12:51:38 -0700 Subject: [PATCH 243/630] wallet: use coreutils wallet types --- go.mod | 4 +- go.sum | 2 - wallet/events.go | 134 ------------------------------------------ wallet/wallet.go | 58 ++++++++++++++---- wallet/wallet_test.go | 48 ++------------- 5 files changed, 54 insertions(+), 192 deletions(-) delete mode 100644 wallet/events.go diff --git a/go.mod b/go.mod index df040fb..8ce388c 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.21.8 toolchain go1.22.3 +replace go.sia.tech/coreutils => ../coreutils + require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.4.2 - go.sia.tech/coreutils v0.2.2 + go.sia.tech/coreutils v0.2.4-0.20240809193500-b680c3b225f0 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 5f6ec93..102b68e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,6 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= -go.sia.tech/coreutils v0.2.2 h1:WWbuk4lEd5SVB/LoBsPG6+heZN2olZT4yOkRZd3pvpg= -go.sia.tech/coreutils v0.2.2/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/wallet/events.go b/wallet/events.go deleted file mode 100644 index a42ab7e..0000000 --- a/wallet/events.go +++ /dev/null @@ -1,134 +0,0 @@ -package wallet - -import ( - "encoding/json" - "fmt" - "time" - - "go.sia.tech/core/types" -) - -// event types indicate the source of an event. Events can -// either be created by sending Siacoins between addresses or they can be -// created by consensus (e.g. a miner payout, a siafund claim, or a contract). -const ( - EventTypeMinerPayout = "miner" - EventTypeFoundationSubsidy = "foundation" - - EventTypeV1Transaction = "v1Transaction" - EventTypeV1ContractResolution = "v1ContractResolution" - - EventTypeV2Transaction = "v2Transaction" - EventTypeV2ContractResolution = "v2ContractResolution" - - EventTypeSiafundClaim = "siafundClaim" -) - -type ( - // EventData provides type safety for the Data field of an Event. - EventData interface { - isEvent() bool - } - - // An Event is something interesting that happened on the Sia blockchain. - Event struct { - ID types.Hash256 `json:"id"` - Index types.ChainIndex `json:"index"` - Timestamp time.Time `json:"timestamp"` - MaturityHeight uint64 `json:"maturityHeight"` - Type string `json:"type"` - Data EventData `json:"data"` - Relevant []types.Address `json:"relevant,omitempty"` - } - - // An EventV1Transaction pairs a v1 transaction with its spent siacoin and - // siafund elements. - EventV1Transaction struct { - Transaction types.Transaction `json:"transaction"` - // v1 siacoin inputs do not describe the value of the spent utxo - SpentSiacoinElements []types.SiacoinElement `json:"spentSiacoinElements"` - // v1 siafund inputs do not describe the value of the spent utxo - SpentSiafundElements []types.SiafundElement `json:"spentSiafundElements"` - } - - // An EventV2Transaction is a v2 transaction. - EventV2Transaction types.V2Transaction - - // An EventPayout represents a payout from a siafund claim, a miner, or the - // foundation subsidy. - EventPayout struct { - SiacoinElement types.SiacoinElement `json:"siacoinElement"` - } - - // An EventV1ContractResolution represents a file contract payout from a v1 - // contract. - EventV1ContractResolution struct { - Parent types.FileContractElement `json:"parent"` - SiacoinElement types.SiacoinElement `json:"siacoinElement"` - Missed bool `json:"missed"` - } - - // An EventV2ContractResolution represents a file contract payout from a v2 - // contract. - EventV2ContractResolution struct { - Resolution types.V2FileContractResolution `json:"resolution"` - SiacoinElement types.SiacoinElement `json:"siacoinElement"` - Missed bool `json:"missed"` - } -) - -func (EventPayout) isEvent() bool { return true } -func (EventV1ContractResolution) isEvent() bool { return true } -func (EventV2ContractResolution) isEvent() bool { return true } -func (EventV1Transaction) isEvent() bool { return true } -func (EventV2Transaction) isEvent() bool { return true } - -// UnmarshalJSON implements the json.Unmarshaler interface. -func (e *Event) UnmarshalJSON(b []byte) error { - var je struct { - ID types.Hash256 `json:"id"` - Index types.ChainIndex `json:"index"` - Timestamp time.Time `json:"timestamp"` - MaturityHeight uint64 `json:"maturityHeight"` - Type string `json:"type"` - Data json.RawMessage `json:"data"` - Relevant []types.Address `json:"relevant,omitempty"` - } - if err := json.Unmarshal(b, &je); err != nil { - return err - } - - e.ID = je.ID - e.Index = je.Index - e.Timestamp = je.Timestamp - e.MaturityHeight = je.MaturityHeight - e.Type = je.Type - e.Relevant = je.Relevant - - var err error - switch je.Type { - case EventTypeMinerPayout, EventTypeFoundationSubsidy, EventTypeSiafundClaim: - var data EventPayout - err = json.Unmarshal(je.Data, &data) - e.Data = data - case EventTypeV1ContractResolution: - var data EventV1ContractResolution - err = json.Unmarshal(je.Data, &data) - e.Data = data - case EventTypeV2ContractResolution: - var data EventV2ContractResolution - err = json.Unmarshal(je.Data, &data) - e.Data = data - case EventTypeV1Transaction: - var data EventV1Transaction - err = json.Unmarshal(je.Data, &data) - e.Data = data - case EventTypeV2Transaction: - var data EventV2Transaction - err = json.Unmarshal(je.Data, &data) - e.Data = data - default: - return fmt.Errorf("unknown event type: %v", je.Type) - } - return err -} diff --git a/wallet/wallet.go b/wallet/wallet.go index 4a8678a..95c5631 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -8,6 +8,42 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/wallet" +) + +// event types indicate the source of an event. Events can +// either be created by sending Siacoins between addresses or they can be +// created by consensus (e.g. a miner payout, a siafund claim, or a contract). +const ( + EventTypeMinerPayout = wallet.EventTypeMinerPayout + EventTypeFoundationSubsidy = wallet.EventTypeFoundationSubsidy + EventTypeSiafundClaim = wallet.EventTypeSiafundClaim + + EventTypeV1Transaction = wallet.EventTypeV1Transaction + EventTypeV1ContractResolution = wallet.EventTypeV1ContractResolution + + EventTypeV2Transaction = wallet.EventTypeV2Transaction + EventTypeV2ContractResolution = wallet.EventTypeV2ContractResolution +) + +type ( + // An EventPayout represents a miner payout, siafund claim, or foundation + // subsidy. + EventPayout = wallet.EventPayout + // An EventV1Transaction pairs a v1 transaction with its spent siacoin and + // siafund elements. + EventV1Transaction = wallet.EventV1Transaction + // An EventV1ContractResolution represents a file contract payout from a v1 + // contract. + EventV1ContractResolution = wallet.EventV1ContractResolution + // EventV2Transaction is a transaction event that includes the transaction + EventV2Transaction = wallet.EventV2Transaction + // An EventV2ContractResolution represents a file contract payout from a v2 + // contract. + EventV2ContractResolution = wallet.EventV2ContractResolution + + EventData = wallet.EventData + Event = wallet.Event ) type ( @@ -93,7 +129,7 @@ func SignTransaction(cs consensus.State, txn *types.Transaction, sigIndex int, k // AppliedEvents extracts a list of relevant events from a chain update. func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) (events []Event) { - addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v EventData, relevant []types.Address) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v wallet.EventData, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] @@ -157,7 +193,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f // handle v1 transactions for _, txn := range b.Transactions { addresses := make(map[types.Address]bool) - e := &EventV1Transaction{ + e := &wallet.EventV1Transaction{ Transaction: txn, SpentSiacoinElements: make([]types.SiacoinElement, 0, len(txn.SiacoinInputs)), SpentSiafundElements: make([]types.SiafundElement, 0, len(txn.SiafundInputs)), @@ -193,7 +229,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sce, ok := sces[sfi.ParentID.ClaimOutputID()] if ok && relevant(sce.SiacoinOutput.Address) { - addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, EventPayout{ + addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } @@ -240,7 +276,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sce, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] if ok && relevant(sfi.ClaimAddress) { - addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, EventPayout{ + addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } @@ -257,7 +293,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f continue } - ev := EventV2Transaction(txn) + ev := wallet.EventV2Transaction(txn) relevant := make([]types.Address, 0, len(addresses)) for addr := range addresses { relevant = append(relevant, addr) @@ -279,7 +315,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } element := sces[types.FileContractID(fce.ID).ValidOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, EventV1ContractResolution{ + addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ Parent: fce, SiacoinElement: element, Missed: false, @@ -293,7 +329,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } element := sces[types.FileContractID(fce.ID).MissedOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, EventV1ContractResolution{ + addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ Parent: fce, SiacoinElement: element, Missed: true, @@ -314,7 +350,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.HostOutput.Address) { element := sces[types.FileContractID(fce.ID).V2HostOutputID()] - addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ + addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, @@ -326,7 +362,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.RenterOutput.Address) { element := sces[types.FileContractID(fce.ID).V2RenterOutputID()] - addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, EventV2ContractResolution{ + addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, @@ -341,7 +377,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { element := sces[cs.Index.ID.MinerOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeMinerPayout, EventPayout{ + addEvent(element.ID, element.MaturityHeight, EventTypeMinerPayout, wallet.EventPayout{ SiacoinElement: element, }, []types.Address{b.MinerPayouts[i].Address}) } @@ -351,7 +387,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(cs.FoundationPrimaryAddress) { element, ok := sces[cs.Index.ID.FoundationOutputID()] if ok { - addEvent(element.ID, element.MaturityHeight, EventTypeFoundationSubsidy, EventPayout{ + addEvent(element.ID, element.MaturityHeight, EventTypeFoundationSubsidy, wallet.EventPayout{ SiacoinElement: element, }, []types.Address{element.SiacoinOutput.Address}) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 09b2f7b..cc5e9cc 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -2980,50 +2980,10 @@ func TestEventTypes(t *testing.T) { t.Fatalf("expected maturity height %v, got %v", maturityHeight, event.MaturityHeight) } - var inflowSum, outflowSum types.Currency - switch ev := event.Data.(type) { - case wallet.EventV1Transaction: - for _, sce := range ev.SpentSiacoinElements { - if sce.SiacoinOutput.Address == addr { - outflowSum = outflowSum.Add(sce.SiacoinOutput.Value) - } - } - for _, sce := range ev.Transaction.SiacoinOutputs { - if sce.Address == addr { - inflowSum = inflowSum.Add(sce.Value) - } - } - case wallet.EventV1ContractResolution: - if ev.SiacoinElement.SiacoinOutput.Address == addr { - inflowSum = ev.SiacoinElement.SiacoinOutput.Value - } - case wallet.EventPayout: - if ev.SiacoinElement.SiacoinOutput.Address == addr { - inflowSum = ev.SiacoinElement.SiacoinOutput.Value - } - case wallet.EventV2ContractResolution: - if ev.SiacoinElement.SiacoinOutput.Address == addr { - inflowSum = ev.SiacoinElement.SiacoinOutput.Value - } - case wallet.EventV2Transaction: - for _, sce := range ev.SiacoinInputs { - if sce.Parent.SiacoinOutput.Address == addr { - outflowSum = outflowSum.Add(sce.Parent.SiacoinOutput.Value) - } - } - for _, sce := range ev.SiacoinOutputs { - if sce.Address == addr { - inflowSum = inflowSum.Add(sce.Value) - } - } - default: - t.Fatalf("unexpected event type %T", ev) - } - - if !inflowSum.Equals(expectedInflow) { - t.Fatalf("expected inflow %v, got %v", expectedInflow, inflowSum) - } else if !outflowSum.Equals(expectedOutflow) { - t.Fatalf("expected outflow %v, got %v", expectedOutflow, outflowSum) + if !event.SiacoinInflow().Equals(expectedInflow) { + t.Fatalf("expected inflow %v, got %v", expectedInflow, event.SiacoinInflow()) + } else if !event.SiacoinOutflow().Equals(expectedOutflow) { + t.Fatalf("expected outflow %v, got %v", expectedOutflow, event.SiacoinOutflow()) } return } From b4935767c2b256ea47bb806fa6cdd03c7c84e9ce Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 9 Aug 2024 12:56:03 -0700 Subject: [PATCH 244/630] wallet: fix lint --- wallet/wallet.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 95c5631..709b1c3 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -42,8 +42,10 @@ type ( // contract. EventV2ContractResolution = wallet.EventV2ContractResolution + // EventData is the data associated with an event. EventData = wallet.EventData - Event = wallet.Event + // An Event is a record of a consensus event that affects the wallet. + Event = wallet.Event ) type ( From d48924947f51612a0488b4b51c1aca9e312ca5b7 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 9 Aug 2024 13:01:41 -0700 Subject: [PATCH 245/630] deps: update coreutils --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8ce388c..341bd7d 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,10 @@ go 1.21.8 toolchain go1.22.3 -replace go.sia.tech/coreutils => ../coreutils - require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.4.2 - go.sia.tech/coreutils v0.2.4-0.20240809193500-b680c3b225f0 + go.sia.tech/coreutils v0.2.4 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 102b68e..ceec7a3 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= +go.sia.tech/coreutils v0.2.4 h1:jEojRSz+O7Rap1zACUbAS+Hzvcdw2+0jx94iBD27eOo= +go.sia.tech/coreutils v0.2.4/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 1114e5bd3066862c21c07f5869de4f03aba5a90a Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 9 Aug 2024 13:31:59 -0700 Subject: [PATCH 246/630] sqlite: fix missing relevant address --- persist/sqlite/addresses.go | 2 +- wallet/wallet_test.go | 87 ++++++++++++++++++++++++------------- 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 497742f..52f4ec4 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -47,7 +47,7 @@ func (s *Store) AddressEvents(address types.Address, offset, limit int) (events if err != nil { return fmt.Errorf("failed to scan event: %w", err) } - + event.Relevant = []types.Address{address} events = append(events, event) } return rows.Err() diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index cc5e9cc..13cd74f 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1459,16 +1459,22 @@ func TestEvents(t *testing.T) { waitForBlock(t, cm, db) // check the payout was received - if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + events, err := wm.AddressEvents(addr, 0, 100) + if err != nil { t.Fatal(err) } else if len(events) != 1 { t.Fatalf("expected 1 events, got %v", len(events)) } else if events[0].Type != wallet.EventTypeMinerPayout { t.Fatalf("expected miner payout event, got %v", events[0].Type) - } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + } + + expected := events[0] + expected.Relevant = nil // clear the relevant field for deep equal + events2, err := wm.Events([]types.Hash256{events[0].ID}) + if err != nil { t.Fatalf("expected to get event: %v", err) - } else if !reflect.DeepEqual(events2[0], events[0]) { - t.Fatalf("expected event %v to match %v", events[0], events2) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2[0]) } assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) @@ -1527,29 +1533,40 @@ func TestEvents(t *testing.T) { assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) // check the events for the transaction - if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + events, err = wm.AddressEvents(addr, 0, 100) + if err != nil { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 events, got %v", len(events)) } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) - } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { t.Fatalf("expected to get event: %v", err) - } else if !reflect.DeepEqual(events2[0], events[0]) { - t.Fatalf("expected event %v to match %v", events[0], events2) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) } // check the events for the second address - if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + events, err = wm.AddressEvents(addr2, 0, 100) + if err != nil { t.Fatal(err) } else if len(events) != 2 { t.Fatalf("expected 2 event, got %v", len(events)) } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) - } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + events2, err = wm.Events([]types.Hash256{events[0].ID}) + if err != nil { t.Fatalf("expected to get event: %v", err) - } else if !reflect.DeepEqual(events2[0], events[0]) { - t.Fatalf("expected event %v to match %v", events[0], events2) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2[0]) } sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) @@ -1586,29 +1603,39 @@ func TestEvents(t *testing.T) { assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, 0) // check the events for the transaction - if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + events, err = wm.AddressEvents(addr2, 0, 100) + if err != nil { t.Fatal(err) } else if len(events) != 4 { t.Fatalf("expected 4 events, got %v", len(events)) } else if events[0].Type != wallet.EventTypeSiafundClaim { t.Fatalf("expected transaction event, got %v", events[0].Type) - } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { t.Fatalf("expected to get event: %v", err) - } else if !reflect.DeepEqual(events2[0], events[0]) { - t.Fatalf("expected event %v to match %v", events[0], events2) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) } // check the events for the first address - if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + events, err = wm.AddressEvents(addr, 0, 100) + if err != nil { t.Fatal(err) } else if len(events) != 3 { t.Fatalf("expected 3 events, got %v", len(events)) } else if events[0].Type != wallet.EventTypeV2Transaction { t.Fatalf("expected transaction event, got %v", events[0].Type) - } else if events2, err := wm.Events([]types.Hash256{events[0].ID}); err != nil { + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { t.Fatalf("expected to get event: %v", err) - } else if !reflect.DeepEqual(events2[0], events[0]) { - t.Fatalf("expected event %v to match %v", events[0], events2) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) } } @@ -2964,7 +2991,7 @@ func TestEventTypes(t *testing.T) { return filtered } - assertEvent := func(id types.Hash256, eventType string, expectedInflow, expectedOutflow types.Currency, maturityHeight uint64) { + assertEvent := func(t *testing.T, id types.Hash256, eventType string, expectedInflow, expectedOutflow types.Currency, maturityHeight uint64) { t.Helper() events, err := wm.AddressEvents(addr, 0, 100) @@ -2993,7 +3020,7 @@ func TestEventTypes(t *testing.T) { // miner payout event mineBlock(1, addr) - assertEvent(types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) + assertEvent(t, types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) // mine until the payout matures mineBlock(int(cm.TipState().MaturityHeight()), types.VoidAddress) @@ -3035,7 +3062,7 @@ func TestEventTypes(t *testing.T) { } // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + assertEvent(t, types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) }) t.Run("v1 contract resolution - missed", func(t *testing.T) { @@ -3093,7 +3120,7 @@ func TestEventTypes(t *testing.T) { // mine until the contract expires to trigger the resolution event blocksRemaining := int(fc.WindowEnd - cm.Tip().Height) mineBlock(blocksRemaining, types.VoidAddress) - assertEvent(types.Hash256(txn.FileContractID(0).MissedOutputID(0)), wallet.EventTypeV1ContractResolution, contractPayout, types.ZeroCurrency, fc.WindowEnd+144) + assertEvent(t, types.Hash256(txn.FileContractID(0).MissedOutputID(0)), wallet.EventTypeV1ContractResolution, contractPayout, types.ZeroCurrency, fc.WindowEnd+144) }) t.Run("v2 transaction", func(t *testing.T) { @@ -3127,7 +3154,7 @@ func TestEventTypes(t *testing.T) { } // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + assertEvent(t, types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) }) t.Run("v2 contract resolution - expired", func(t *testing.T) { @@ -3218,7 +3245,7 @@ func TestEventTypes(t *testing.T) { } // mine a block to confirm the resolution mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) }) t.Run("v2 contract resolution - storage proof", func(t *testing.T) { @@ -3315,7 +3342,7 @@ func TestEventTypes(t *testing.T) { t.Fatal(err) } mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(types.FileContractID(fce.ID).V2HostOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2HostOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) }) t.Run("v2 contract resolution - renewal", func(t *testing.T) { @@ -3461,7 +3488,7 @@ func TestEventTypes(t *testing.T) { t.Fatal(err) } mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) }) t.Run("v2 contract resolution - finalization", func(t *testing.T) { @@ -3561,7 +3588,7 @@ func TestEventTypes(t *testing.T) { t.Fatal(err) } mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) }) t.Run("siafund claim", func(t *testing.T) { @@ -3599,6 +3626,6 @@ func TestEventTypes(t *testing.T) { } // mine a block to confirm the transaction mineBlock(1, types.VoidAddress) - assertEvent(types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) + assertEvent(t, types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) }) } From ff6bc3717192b543c4b391cf779d435e333733bb Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 9 Aug 2024 13:45:51 -0700 Subject: [PATCH 247/630] wallet: clear file contract element proofs --- go.mod | 4 ++-- go.sum | 8 ++++---- wallet/wallet.go | 14 ++++---------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 341bd7d..943cdfc 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.22.3 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.2 - go.sia.tech/coreutils v0.2.4 + go.sia.tech/core v0.4.3 + go.sia.tech/coreutils v0.2.5 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index ceec7a3..18b37fc 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= -go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= -go.sia.tech/coreutils v0.2.4 h1:jEojRSz+O7Rap1zACUbAS+Hzvcdw2+0jx94iBD27eOo= -go.sia.tech/coreutils v0.2.4/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= +go.sia.tech/core v0.4.3 h1:XEX7v6X8eJh4zyOkSHYi6FsyD+N/OEKw/NIigaaWPAU= +go.sia.tech/core v0.4.3/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= +go.sia.tech/coreutils v0.2.5 h1:oMnBGMBRfxhLzTH1ZDBg0Ep0QLE2GE1lND9yfzOzenA= +go.sia.tech/coreutils v0.2.5/go.mod h1:Pg9eE3xL25couNL/vYrtCWP5uXkVvC+SUcMVh1/E7+I= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= diff --git a/wallet/wallet.go b/wallet/wallet.go index 709b1c3..c968e56 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -173,8 +173,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f // collect all elements sces := make(map[types.SiacoinOutputID]types.SiacoinElement) sfes := make(map[types.SiafundOutputID]types.SiafundElement) - fces := make(map[types.FileContractID]types.FileContractElement) - v2fces := make(map[types.FileContractID]types.V2FileContractElement) cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { sce.MerkleProof = nil sces[types.SiacoinOutputID(sce.ID)] = sce @@ -183,14 +181,6 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sfe.MerkleProof = nil sfes[types.SiafundOutputID(sfe.ID)] = sfe }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { - fce.MerkleProof = nil - fces[types.FileContractID(fce.ID)] = fce - }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - fce.MerkleProof = nil - v2fces[types.FileContractID(fce.ID)] = fce - }) // handle v1 transactions for _, txn := range b.Transactions { @@ -309,6 +299,8 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return } + fce.MerkleProof = nil + if valid { for i := range fce.FileContract.ValidProofOutputs { address := fce.FileContract.ValidProofOutputs[i].Address @@ -345,6 +337,8 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return } + fce.MerkleProof = nil + var missed bool if _, ok := res.(*types.V2FileContractExpiration); ok { missed = true From 2a40822b6f90d20437a856264a16a1be4fac48dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:24:26 +0000 Subject: [PATCH 248/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.22.0 to 0.23.0 - [Commits](https://github.com/golang/term/compare/v0.22.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 943cdfc..75b9265 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 - golang.org/x/term v0.22.0 + golang.org/x/term v0.23.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 diff --git a/go.sum b/go.sum index 18b37fc..507a202 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From eb50f7a61117e3739b3528df860df71d164a557d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 13 Aug 2024 11:29:03 +0200 Subject: [PATCH 249/630] .github: adding issue templates --- .github/ISSUE_TEMPLATE/bug_report.yml | 57 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 5 ++ .github/ISSUE_TEMPLATE/feature_request.yml | 42 ++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..32da25c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,57 @@ +name: Bug Report +description: File a bug report +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + Note: Please search to see if an issue already exists for the bug you encountered. + - type: textarea + id: current-behavior + attributes: + label: Current Behavior + description: A concise description of what you're experiencing. + placeholder: Tell us what you see! + validations: + required: true + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + description: A concise description of what you expected to happen. + placeholder: Tell us what you want to see! + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + description: Detailed steps to reproduce the behavior. + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + - type: input + id: version + attributes: + label: Version + description: What version of walletd are you running? If you are running from source, please provide the commit hash. + placeholder: v0.8.0 + validations: + required: true + - type: input + id: os + attributes: + label: What operating system did the problem occur on (e.g. Ubuntu 22.04, macOS 12.0, Windows 11)? + validations: + required: true + - type: textarea + attributes: + label: Anything else? + description: | + Links? References? Anything that will give us more context about the issue you are encountering! + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..1a50396 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Sia Community Discord + url: https://discord.gg/sia + about: Join the Sia community discord for more help with Sia or hostd. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..c6b7f35 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature Request +description: Request a new feature be added to walletd. +labels: ["feature"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + Note: Please search to see if an issue already exists for the feature + you want added. + - type: textarea + id: feature-description + attributes: + label: Description + description: | + A description of the feature you want added + Tip: You can attach images by clicking this area and then dragging files in. + placeholder: Tell us what you want! Be as descriptive as possible. + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of walletd are you running? + placeholder: v0.8.0 + validations: + required: false + - type: input + id: os + attributes: + label: What operating system are you running (e.g. Ubuntu 22.04, macOS, Windows 11)? + validations: + required: false + - type: textarea + attributes: + label: Anything else? + description: | + Links? References? Anything that will give us more context about the feature! + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false \ No newline at end of file From 0401321a4b22cdb79e81b0888b6af26a866b6432 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 11:01:30 -0700 Subject: [PATCH 250/630] deps: update core and coreutils --- go.mod | 12 +++++------- go.sum | 16 ++++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 75b9265..d3b7eb4 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,11 @@ module go.sia.tech/walletd -go 1.21.8 - -toolchain go1.22.3 +go 1.22.5 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.3 - go.sia.tech/coreutils v0.2.5 + go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c + go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 @@ -25,7 +23,7 @@ require ( go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 507a202..c6275c5 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.3 h1:XEX7v6X8eJh4zyOkSHYi6FsyD+N/OEKw/NIigaaWPAU= -go.sia.tech/core v0.4.3/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= -go.sia.tech/coreutils v0.2.5 h1:oMnBGMBRfxhLzTH1ZDBg0Ep0QLE2GE1lND9yfzOzenA= -go.sia.tech/coreutils v0.2.5/go.mod h1:Pg9eE3xL25couNL/vYrtCWP5uXkVvC+SUcMVh1/E7+I= +go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c h1:HJuHf6pBV9GOseVs3Yby3xbYzV8vZWTcsgrO2UGgQW8= +go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= +go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395 h1:MRK96OSDdxOXbHDRWH8LjYgTshUKUBDJyvD1Z/O1y80= +go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395/go.mod h1:TjQITC7A7u3sX22sN54SmcPcn+YmnodEqzNElAA7G/s= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -30,15 +30,15 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= From f1ede0bcb61b7962d2c11f4d8e01626059de005b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 12:13:54 -0700 Subject: [PATCH 251/630] cmd: write errors to stderr --- cmd/walletd/config.go | 31 ++++++++++++++++--------------- cmd/walletd/main.go | 11 ++++++++--- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go index 3b93294..c1ae246 100644 --- a/cmd/walletd/config.go +++ b/cmd/walletd/config.go @@ -19,7 +19,7 @@ func readPasswordInput(context string) string { fmt.Printf("%s: ", context) input, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { - stdoutFatalError("Could not read input: " + err.Error()) + fatalError(fmt.Errorf("could not read password input: %w", err)) } fmt.Println("") return string(input) @@ -30,7 +30,7 @@ func readInput(context string) string { r := bufio.NewReader(os.Stdin) input, err := r.ReadString('\n') if err != nil { - stdoutFatalError("Could not read input: " + err.Error()) + fatalError(fmt.Errorf("could not read input: %w", err)) } return strings.TrimSpace(input) } @@ -84,12 +84,6 @@ func promptYesNo(question string) bool { return strings.EqualFold(answer, "yes") } -// stdoutFatalError prints an error message to stdout and exits with a 1 exit code. -func stdoutFatalError(msg string) { - stdoutError(msg) - os.Exit(1) -} - // stdoutError prints an error message to stdout func stdoutError(msg string) { if cfg.Log.StdOut.EnableANSI { @@ -122,7 +116,7 @@ func setDataDirectory() { dir, err := filepath.Abs(cfg.Directory) if err != nil { - stdoutFatalError("Could not get absolute path of data directory: " + err.Error()) + fatalError(fmt.Errorf("failed to get absolute path of data directory: %w", err)) } fmt.Println("The data directory is where walletd will store its metadata and consensus data.") @@ -200,12 +194,13 @@ func setAdvancedConfig() { fmt.Println("This cannot be changed later without resetting walletd.") fmt.Printf("Currently %q\n", cfg.Index.Mode) mode := readInput(`Enter index mode ("personal" or "full")`) - if strings.EqualFold(mode, "personal") { + switch { + case strings.EqualFold(mode, "personal"): cfg.Index.Mode = wallet.IndexModePersonal - } else if strings.EqualFold(mode, "full") { + case strings.EqualFold(mode, "full"): cfg.Index.Mode = wallet.IndexModeFull - } else { - stdoutFatalError("Invalid index mode: " + mode) + default: + fatalError(fmt.Errorf("invalid index mode: %q", mode)) } fmt.Println("") @@ -242,14 +237,20 @@ func buildConfig() { // write the config file f, err := os.Create(configPath) if err != nil { - stdoutFatalError("failed to create config file: " + err.Error()) + fatalError(fmt.Errorf("failed to create config file: %w", err)) return } defer f.Close() enc := yaml.NewEncoder(f) if err := enc.Encode(cfg); err != nil { - stdoutFatalError("failed to encode config file: " + err.Error()) + fatalError(fmt.Errorf("failed to encode config file: %w", err)) + return + } else if err := f.Sync(); err != nil { + fatalError(fmt.Errorf("failed to sync config file: %w", err)) + return + } else if err := f.Close(); err != nil { + fatalError(fmt.Errorf("failed to close config file: %w", err)) return } } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 0b7f5ca..d5de169 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -110,6 +110,11 @@ func mustSetAPIPassword() { } } +func fatalError(err error) { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) +} + // tryLoadConfig loads the config file specified by the WALLETD_CONFIG_FILE. If // the config file does not exist, it will not be loaded. func tryLoadConfig() { @@ -126,7 +131,7 @@ func tryLoadConfig() { f, err := os.Open(configPath) if err != nil { - stdoutFatalError("failed to open config file: " + err.Error()) + fatalError(fmt.Errorf("failed to open config file: %w", err)) return } defer f.Close() @@ -237,7 +242,7 @@ func main() { defer cancel() if err := os.MkdirAll(cfg.Directory, 0700); err != nil { - stdoutFatalError("failed to create directory: " + err.Error()) + fatalError(fmt.Errorf("failed to create data directory: %w", err)) } mustSetAPIPassword() @@ -284,7 +289,7 @@ func main() { fileWriter, closeFn, err := zap.Open(cfg.Log.File.Path) if err != nil { - stdoutFatalError("failed to open log file: " + err.Error()) + fatalError(fmt.Errorf("failed to open log file: %w", err)) return } defer closeFn() From 390c640073db4a65cf54c0a02e64dad0e7c6285d Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 12:33:35 -0700 Subject: [PATCH 252/630] ci,docker: use Go 1.23 --- .github/workflows/main.yml | 2 +- .github/workflows/publish.yml | 8 ++++---- Dockerfile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcb05b4..d5474f7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.21', '1.22' ] + go-version: [ '1.22', '1.23' ] steps: - name: Configure git run: git config --global core.autocrlf false # required on Windows diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 20d650f..5f7262d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Test uses: ./.github/actions/test docker: @@ -67,7 +67,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Setup run: | sudo apt update @@ -101,7 +101,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Setup env: APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} @@ -169,7 +169,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.23' - name: Setup shell: bash run: | diff --git a/Dockerfile b/Dockerfile index 315ff3b..2318f1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/golang:1.21 AS builder +FROM docker.io/library/golang:1.23 AS builder WORKDIR /walletd From d77642fc5daacb3b42e007cfeb4ffa1ebc59421b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 14:01:57 -0700 Subject: [PATCH 253/630] deps: update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d3b7eb4..58406fe 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.5 require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c - go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395 + go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.22.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index c6275c5..687d833 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c h1:HJuHf6pBV9GOseVs3Yby3xbYzV8vZWTcsgrO2UGgQW8= go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= -go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395 h1:MRK96OSDdxOXbHDRWH8LjYgTshUKUBDJyvD1Z/O1y80= -go.sia.tech/coreutils v0.2.6-0.20240814175830-40722f814395/go.mod h1:TjQITC7A7u3sX22sN54SmcPcn+YmnodEqzNElAA7G/s= +go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b h1:iV7PdyUf7CC6slo4CgY+XuJ6gRS/HtZGjzLVm383rTo= +go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b/go.mod h1:TjQITC7A7u3sX22sN54SmcPcn+YmnodEqzNElAA7G/s= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 0e4c3172de0482e4a8a9f547c91d69b6305d66f3 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 15:24:03 -0700 Subject: [PATCH 254/630] ci: switch to global publish and test workflows --- .github/actions/test/action.yml | 22 --- .github/workflows/main.yml | 22 +-- .github/workflows/publish.yml | 251 +------------------------------- 3 files changed, 9 insertions(+), 286 deletions(-) delete mode 100644 .github/actions/test/action.yml diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index 0b51e44..0000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Test -description: Lints and tests walletd - -runs: - using: composite - steps: - - name: Configure git # required for golangci-lint on Windows - shell: bash - run: git config --global core.autocrlf false - - name: Lint - uses: golangci/golangci-lint-action@v6 - with: - skip-cache: true -# - name: Analyze -# uses: SiaFoundation/action-golang-analysis@HEAD -# with: -# analyzers: | -# go.sia.tech/jape.Analyzer - - name: Test - uses: n8maninger/action-golang-test@v2 - with: - args: "-race;-tags=testing netgo" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d5474f7..f3e416e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,5 @@ -name: Main +name: Lint & Test on: - workflow_dispatch: pull_request: push: branches: @@ -11,21 +10,4 @@ env: jobs: test: - runs-on: ${{ matrix.os }} - permissions: - contents: read - strategy: - matrix: - os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.22', '1.23' ] - steps: - - name: Configure git - run: git config --global core.autocrlf false # required on Windows - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - - name: Test - uses: ./.github/actions/test - - name: Build - run: go build -o bin/ ./cmd/walletd + uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5f7262d..ce4a39c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,249 +12,12 @@ on: concurrency: group: ${{ github.workflow }} + cancel-in-progress: false jobs: - test: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.23' - - name: Test - uses: ./.github/actions/test - docker: - runs-on: ubuntu-latest - needs: [ test ] - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/metadata-action@v5 - name: generate tags - id: meta - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=ref,event=branch - type=sha,prefix= - type=semver,pattern={{version}} - - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-linux: - runs-on: ubuntu-latest - needs: [ test ] - strategy: - matrix: - go-arch: [amd64, arm64] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.23' - - name: Setup - run: | - sudo apt update - go generate ./... - if [ ${{ matrix.go-arch }} == "arm64" ]; then - sudo apt install -y gcc-aarch64-linux-gnu - echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV - fi - - name: Build ${{ matrix.go-arch }} - env: - CGO_ENABLED: 1 - GOOS: linux - GOARCH: ${{ matrix.go-arch }} - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd - cp README.md LICENSE bin/ - zip -qj $ZIP_OUTPUT bin/* - - uses: actions/upload-artifact@v4 - with: - name: walletd_linux_${{ matrix.go-arch }} - path: release/* - build-mac: - runs-on: macos-latest - needs: [ test ] - strategy: - matrix: - go-arch: [amd64, arm64] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.23' - - name: Setup - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - run: | - # extract apple cert - APPLE_CERT_PATH=$RUNNER_TEMP/apple_cert.p12 - KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db - echo -n "$APPLE_CERT_B64" | base64 --decode --output $APPLE_CERT_PATH - - # extract apple key - mkdir -p ~/private_keys - APPLE_API_KEY_PATH=~/private_keys/AuthKey_$APPLE_API_KEY.p8 - echo -n "$APPLE_KEY_B64" | base64 --decode --output $APPLE_API_KEY_PATH - - # create temp keychain - security create-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH - security default-keychain -s $KEYCHAIN_PATH - security set-keychain-settings -lut 21600 $KEYCHAIN_PATH - security unlock-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH - - # import keychain - security import $APPLE_CERT_PATH -P $APPLE_CERT_PASSWORD -A -t cert -f pkcs12 -k $KEYCHAIN_PATH - security find-identity -v $KEYCHAIN_PATH -p codesigning - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $APPLE_KEYCHAIN_PASSWORD $KEYCHAIN_PATH - - # generate - go generate ./... - - # resync system clock https://github.com/actions/runner/issues/2996#issuecomment-1833103110 - sudo sntp -sS time.windows.com - - name: Build ${{ matrix.go-arch }} - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - CGO_ENABLED: 1 - GOOS: darwin - GOARCH: ${{ matrix.go-arch }} - run: | - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - mkdir -p release - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/walletd - ditto -ck bin $ZIP_OUTPUT - xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - uses: actions/upload-artifact@v4 - with: - name: walletd_darwin_${{ matrix.go-arch }} - path: release/* - build-windows: - runs-on: windows-latest - needs: [ test ] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.23' - - name: Setup - shell: bash - run: | - dotnet tool install --global AzureSignTool - go generate ./... - - name: Build amd64 - env: - CGO_ENABLED: 1 - GOOS: windows - GOARCH: amd64 - shell: bash - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd - azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/walletd.exe - cp README.md LICENSE bin/ - 7z a $ZIP_OUTPUT ./bin/* - - uses: actions/upload-artifact@v4 - with: - name: walletd_windows_amd64 - path: release/* - combine-release-assets: - runs-on: ubuntu-latest - needs: [ build-linux, build-mac, build-windows ] - steps: - - name: Merge Artifacts - uses: actions/upload-artifact/merge@v4 - with: - name: walletd - - dispatch-homebrew: # only runs on full releases - if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-') - needs: [ build-mac ] - runs-on: ubuntu-latest - steps: - - name: Extract Tag Name - id: get_tag - run: echo "::set-output name=tag_name::${GITHUB_REF#refs/tags/}" - - - name: Dispatch - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} - repository: siafoundation/homebrew-sia - event-type: release-tagged - client-payload: > - { - "description": "walletd: The Next-Gen Sia Wallet", - "tag": "${{ steps.get_tag.outputs.tag_name }}", - "project": "walletd", - "workflow_id": "${{ github.run_id }}" - } - dispatch-linux: # always runs - needs: [ build-linux ] - runs-on: ubuntu-latest - steps: - - name: Build Dispatch Payload - id: get_payload - uses: actions/github-script@v7 - with: - script: | - const isRelease = context.ref.startsWith('refs/tags/v'), - isBeta = isRelease && context.ref.includes('-beta'), - tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master'; - - let component = 'nightly'; - if (isBeta) { - component = 'beta'; - } else if (isRelease) { - component = 'main'; - } - - return { - description: "walletd: The Next-Gen Sia Wallet", - tag: tag, - project: "walletd", - workflow_id: context.runId, - component: component - }; - - - name: Dispatch - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.PAT_REPOSITORY_DISPATCH }} - repository: siafoundation/linux - event-type: release-tagged - client-payload: ${{ steps.get_payload.outputs.result }} \ No newline at end of file + publish: + uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master + with: + build-args: -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + cgo-enabled: 1 + project: walletd From 20de8f502effd8647e4458fa0d05712c028437f7 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 15:42:47 -0700 Subject: [PATCH 255/630] ci: add temp workflow_dispatch --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3e416e..e0ce504 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: Lint & Test on: + workflow_dispatch: pull_request: push: branches: From f023ee88622eeeccc7879a0d72ef619147aaddb7 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 17:59:13 -0700 Subject: [PATCH 256/630] ci: remove workflow_dispatch --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e0ce504..f3e416e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,5 @@ name: Lint & Test on: - workflow_dispatch: pull_request: push: branches: From 59a45769bd7a0e5c511c65316dba3ef9556ce6aa Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 18:03:36 -0700 Subject: [PATCH 257/630] ci: inherit secrets --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ce4a39c..f4246df 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,7 @@ concurrency: jobs: publish: uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master + secrets: inherit with: build-args: -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' cgo-enabled: 1 From 5945fe86cdfb656642ec3653adfa51894b62f879 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 14 Aug 2024 18:18:54 -0700 Subject: [PATCH 258/630] ci: separate platform build arg inputs --- .github/workflows/publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f4246df..2f1881b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,6 +19,8 @@ jobs: uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master secrets: inherit with: - build-args: -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + linux-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + windows-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + macos-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w' cgo-enabled: 1 project: walletd From 570d7eff6460bdac7976688c2de344658dde97b1 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Thu, 15 Aug 2024 15:28:12 +0000 Subject: [PATCH 259/630] ui: v0.23.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 58406fe..346e89d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b go.sia.tech/jape v0.12.0 - go.sia.tech/web/walletd v0.22.3 + go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.23.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 687d833..cb62b16 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.22.3 h1:I8og0NN2AW1VC2Oi2Kp/e6/Io14PFNiumELju8Hh2dU= -go.sia.tech/web/walletd v0.22.3/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.23.0 h1:5ftJQQwUHG8TYzdzSb+Y1IIPC0jkjNeAuoLUbsv9UTE= +go.sia.tech/web/walletd v0.23.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From ec260a1bd7b7b84948b31a0e989a482fc623699e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 16 Aug 2024 13:31:19 -0700 Subject: [PATCH 260/630] api, cmd: add debug flag and endpoints --- api/api.go | 6 ++ api/api_test.go | 108 ++++++++++++++++++++++++++++-------- api/client.go | 5 ++ api/mine.go | 81 +++++++++++++++++++++++++++ api/server.go | 106 ++++++++++++++++++++++++++++++++--- cmd/walletd/main.go | 4 +- cmd/walletd/node.go | 10 +++- persist/sqlite/consensus.go | 1 - 8 files changed, 286 insertions(+), 35 deletions(-) create mode 100644 api/mine.go diff --git a/api/api.go b/api/api.go index 2e078b2..afe97e2 100644 --- a/api/api.go +++ b/api/api.go @@ -123,3 +123,9 @@ type ConsensusUpdatesResponse struct { Applied []ApplyUpdate `json:"applied"` Reverted []RevertUpdate `json:"reverted"` } + +// DebugMineRequest is the request type for /debug/mine. +type DebugMineRequest struct { + Blocks int `json:"blocks"` + Address types.Address `json:"address"` +} diff --git a/api/api_test.go b/api/api_test.go index c2c2160..48e20a8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -41,17 +41,24 @@ func testNetwork() (*consensus.Network, types.Block) { return n, genesisBlock } -func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Client, func()) { +func runServer(t *testing.T, cm api.ChainManager, s api.Syncer, wm api.WalletManager) *api.Client { + t.Helper() + l, err := net.Listen("tcp", ":0") if err != nil { - panic(err) - } - go func() { - srv := api.NewServer(cm, s, wm) - http.Serve(l, jape.BasicAuth("password")(srv)) - }() - c := api.NewClient("http://"+l.Addr().String(), "password") - return c, func() { l.Close() } + t.Fatal("failed to listen:", err) + } + t.Cleanup(func() { l.Close() }) + + server := &http.Server{ + Handler: jape.BasicAuth("password")(api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)))), + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + } + t.Cleanup(func() { server.Close() }) + + go server.Serve(l) + return api.NewClient("http://"+l.Addr().String(), "password") } func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { @@ -95,8 +102,7 @@ func TestWalletAdd(t *testing.T) { } defer wm.Close() - c, shutdown := runServer(cm, nil, wm) - defer shutdown() + c := runServer(t, cm, nil, wm) checkWalletResponse := func(wr api.WalletUpdateRequest, w wallet.Wallet, isUpdate bool) error { // check wallet @@ -287,8 +293,7 @@ func TestWallet(t *testing.T) { sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) // run server - c, shutdown := runServer(cm, s, wm) - defer shutdown() + c := runServer(t, cm, s, wm) w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) @@ -506,8 +511,7 @@ func TestAddresses(t *testing.T) { defer wm.Close() sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) - c, shutdown := runServer(cm, nil, wm) - defer shutdown() + c := runServer(t, cm, nil, wm) w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) @@ -702,8 +706,7 @@ func TestV2(t *testing.T) { } defer wm.Close() - c, shutdown := runServer(cm, nil, wm) - defer shutdown() + c := runServer(t, cm, nil, wm) primaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) @@ -942,8 +945,7 @@ func TestP2P(t *testing.T) { }) go s1.Run(context.Background()) defer s1.Close() - c1, shutdown := runServer(cm1, s1, wm1) - defer shutdown() + c1 := runServer(t, cm1, s1, wm1) w1, err := c1.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) @@ -986,8 +988,7 @@ func TestP2P(t *testing.T) { }, syncer.WithLogger(zaptest.NewLogger(t))) go s2.Run(context.Background()) defer s2.Close() - c2, shutdown2 := runServer(cm2, s2, wm2) - defer shutdown2() + c2 := runServer(t, cm2, s2, wm2) w2, err := c2.AddWallet(api.WalletUpdateRequest{Name: "secondary"}) if err != nil { @@ -1248,8 +1249,7 @@ func TestConsensusUpdates(t *testing.T) { } defer wm.Close() - c, shutdown := runServer(cm, nil, wm) - defer shutdown() + c := runServer(t, cm, nil, wm) for i := 0; i < 10; i++ { b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Second) @@ -1283,3 +1283,65 @@ func TestConsensusUpdates(t *testing.T) { } } } + +func TestDebugMine(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testNetwork() + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + l, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + ps, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + s := syncer.New(l, cm, ps, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l.Addr().String(), + }) + defer s.Close() + go s.Run(context.Background()) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + jc := jape.Client{ + BaseURL: c.BaseURL(), + Password: "password", + } + + err = jc.POST("/debug/mine", api.DebugMineRequest{ + Blocks: 5, + Address: types.VoidAddress, + }, nil) + if err != nil { + t.Fatal(err) + } + + if cm.Tip().Height != 5 { + t.Fatalf("expected tip height to be 5, got %v", cm.Tip().Height) + } +} diff --git a/api/client.go b/api/client.go index 3cd1685..776617b 100644 --- a/api/client.go +++ b/api/client.go @@ -34,6 +34,11 @@ func (c *Client) getNetwork() (*consensus.Network, error) { return c.n, nil } +// BaseURL returns the URL of the walletd server. +func (c *Client) BaseURL() string { + return c.c.BaseURL +} + // State returns information about the current state of the walletd daemon. func (c *Client) State() (resp StateResponse, err error) { err = c.c.GET("/state", &resp) diff --git a/api/mine.go b/api/mine.go new file mode 100644 index 0000000..9666d0c --- /dev/null +++ b/api/mine.go @@ -0,0 +1,81 @@ +package api + +import ( + "context" + "encoding/binary" + "errors" + + "go.sia.tech/core/types" +) + +// mineBlock constructs a block from the provided address and the transactions +// in the txpool, and attempts to find a nonce for it that meets the PoW target. +func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types.Block, error) { + cs := cm.TipState() + txns := cm.PoolTransactions() + v2Txns := cm.V2PoolTransactions() + + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{ + Value: cs.BlockReward(), + Address: addr, + }}, + } + + if cs.Index.Height >= cs.Network.HardforkV2.AllowHeight { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + } + } + + var weight uint64 + for _, txn := range txns { + if weight += cs.TransactionWeight(txn); weight > cs.MaxBlockWeight() { + break + } + b.Transactions = append(b.Transactions, txn) + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) + } + for _, txn := range v2Txns { + if weight += cs.V2TransactionWeight(txn); weight > cs.MaxBlockWeight() { + break + } + b.V2.Transactions = append(b.V2.Transactions, txn) + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) + } + if b.V2 != nil { + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), addr) + } + + b.Nonce = 0 + buf := make([]byte, 32+8+8+32) + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) + if b.V2 != nil { + copy(buf[:32], "sia/id/block|") + copy(buf[48:], b.V2.Commitment[:]) + } else { + root := b.MerkleRoot() + copy(buf[:32], b.ParentID[:]) + copy(buf[48:], root[:]) + } + factor := cs.NonceFactor() + for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { + select { + case <-ctx.Done(): + return types.Block{}, ctx.Err() + default: + } + + // tip changed, abort mining + if cm.Tip() != cs.Index { + return types.Block{}, errors.New("tip changed") + } + + b.Nonce += factor + binary.LittleEndian.PutUint64(buf[32:], b.Nonce) + } + return b, nil +} diff --git a/api/server.go b/api/server.go index e5f6677..f8300ae 100644 --- a/api/server.go +++ b/api/server.go @@ -4,12 +4,14 @@ import ( "context" "errors" "net/http" + "net/http/pprof" "reflect" "runtime" "sync" "time" "go.sia.tech/jape" + "go.uber.org/zap" "lukechampine.com/frand" "go.sia.tech/core/consensus" @@ -21,11 +23,29 @@ import ( "go.sia.tech/walletd/wallet" ) +// A ServerOption sets an optional parameter for the server. +type ServerOption func(*server) + +// WithLogger sets the logger used by the server. +func WithLogger(log *zap.Logger) ServerOption { + return func(s *server) { + s.log = log + } +} + +// WithDebug enables debug endpoints. +func WithDebug() ServerOption { + return func(s *server) { + s.debugEnabled = true + } +} + type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { UpdatesSince(types.ChainIndex, int) ([]chain.RevertUpdate, []chain.ApplyUpdate, error) + Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) TipState() consensus.State AddBlocks([]types.Block) error @@ -85,11 +105,13 @@ type ( ) type server struct { - startTime time.Time + startTime time.Time + debugEnabled bool - cm ChainManager - s Syncer - wm WalletManager + log *zap.Logger + cm ChainManager + s Syncer + wm WalletManager // for walletsReserveHandler mu sync.Mutex @@ -814,17 +836,78 @@ func (s *server) outputsSiafundHandlerGET(jc jape.Context) { jc.Encode(output) } +func (s *server) debugMineHandler(jc jape.Context) { + var req DebugMineRequest + if jc.Decode(&req) != nil { + return + } + + log := s.log.Named("miner") + ctx := jc.Request.Context() + + for n := req.Blocks; n > 0; { + b, err := mineBlock(ctx, s.cm, req.Address) + if errors.Is(err, context.Canceled) { + return + } else if err != nil { + log.Warn("failed to mine block", zap.Error(err)) + } else if err := s.cm.AddBlocks([]types.Block{b}); err != nil { + log.Warn("failed to add block", zap.Error(err)) + } + + if b.V2 == nil { + s.s.BroadcastHeader(gateway.BlockHeader{ + ParentID: b.ParentID, + Nonce: b.Nonce, + Timestamp: b.Timestamp, + MerkleRoot: b.MerkleRoot(), + }) + } else { + s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + } + + log.Debug("mined block", zap.Stringer("blockID", b.ID())) + n-- + } +} + +func (s *server) pprofHandler(jc jape.Context) { + var handler string + if err := jc.DecodeParam("handler", &handler); err != nil { + return + } + + switch handler { + case "cmdline": + pprof.Cmdline(jc.ResponseWriter, jc.Request) + case "profile": + pprof.Profile(jc.ResponseWriter, jc.Request) + case "symbol": + pprof.Symbol(jc.ResponseWriter, jc.Request) + case "trace": + pprof.Trace(jc.ResponseWriter, jc.Request) + default: + pprof.Index(jc.ResponseWriter, jc.Request) + } + pprof.Index(jc.ResponseWriter, jc.Request) +} + // NewServer returns an HTTP handler that serves the walletd API. -func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { +func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { srv := server{ - startTime: time.Now(), + log: zap.NewNop(), + debugEnabled: false, + startTime: time.Now(), cm: cm, s: s, wm: wm, used: make(map[types.Hash256]bool), } - return jape.Mux(map[string]jape.Handler{ + for _, opt := range opts { + opt(&srv) + } + handlers := map[string]jape.Handler{ "GET /state": srv.stateHandler, "GET /consensus/network": srv.consensusNetworkHandler, @@ -872,5 +955,12 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /outputs/siafund/:id": srv.outputsSiafundHandlerGET, "GET /events/:id": srv.eventsHandlerGET, - }) + } + + if srv.debugEnabled { + handlers["POST /debug/mine"] = srv.debugMineHandler + handlers["GET /debug/pprof/:handler"] = srv.pprofHandler + } + + return jape.Mux(handlers) } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index d5de169..0915146 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -199,9 +199,11 @@ func main() { var minerAddrStr string var minerBlocks int + var enableDebug bool rootCmd := flagg.Root rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) + rootCmd.BoolVar(&enableDebug, "debug", false, "enable debug mode with additional profiling and mining endpoints") rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") @@ -314,7 +316,7 @@ func main() { log.Fatal("failed to parse index mode", zap.Error(err)) } - if err := runNode(ctx, cfg, log); err != nil { + if err := runNode(ctx, cfg, log, enableDebug); err != nil { log.Fatal("failed to run node", zap.Error(err)) } case versionCmd: diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index dedde8f..1004415 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -44,7 +44,7 @@ func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error return d.ExternalIP() } -func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { +func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebug bool) error { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -145,7 +145,13 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } defer wm.Close() - api := jape.BasicAuth(cfg.HTTP.Password)(api.NewServer(cm, s, wm)) + apiOpts := []api.ServerOption{ + api.WithLogger(log.Named("api")), + } + if enableDebug { + apiOpts = append(apiOpts, api.WithDebug()) + } + api := jape.BasicAuth(cfg.HTTP.Password)(api.NewServer(cm, s, wm, apiOpts...)) web := walletd.Handler() server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 048d03b..4518933 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -331,7 +331,6 @@ func scanAddress(s scanner) (ab addressRef, err error) { func applyMatureSiacoinBalance(tx *txn, index types.ChainIndex, log *zap.Logger) error { log = log.With(zap.Uint64("maturityHeight", index.Height)) - log.Debug("applying mature siacoin balance") const query = `SELECT id, address_id, siacoin_value FROM siacoin_elements WHERE maturity_height=$1 AND matured=false AND spent_index_id IS NULL` From 6f9aaeec18121bf53fcea8c91127f32c0ddcac54 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 17 Aug 2024 07:36:05 -0700 Subject: [PATCH 261/630] cmd: add anagami values --- cmd/walletd/node.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 1004415..476808b 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -55,6 +55,9 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu case "zen": network, genesisBlock = chain.TestnetZen() bootstrapPeers = syncer.ZenBootstrapPeers + case "anagami": + network, genesisBlock = chain.TestnetAnagami() + bootstrapPeers = syncer.AnagamiBootstrapPeers default: return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } From 0ba4529eb97a82b4e174bc90f2aab1adcc75c1d1 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Aug 2024 12:39:31 -0700 Subject: [PATCH 262/630] ci: skip test build --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3e416e..d104ec5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,3 +11,5 @@ env: jobs: test: uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master + with: + try-build: false From 17f31a17917d24b8624f51df72863bf13fdebd3f Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Aug 2024 12:40:02 -0700 Subject: [PATCH 263/630] ci: revert test build --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d104ec5..f3e416e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,5 +11,3 @@ env: jobs: test: uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master - with: - try-build: false From 5a040c8a838f2f34ea04ee53045b741c37e36051 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Wed, 21 Aug 2024 12:44:37 -0700 Subject: [PATCH 264/630] deps: update core and coreutils for anagami reset --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 346e89d..6f50f94 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.22.5 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c - go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b + go.sia.tech/core v0.4.4 + go.sia.tech/coreutils v0.3.0 go.sia.tech/jape v0.12.0 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index cb62b16..80ba106 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c h1:HJuHf6pBV9GOseVs3Yby3xbYzV8vZWTcsgrO2UGgQW8= -go.sia.tech/core v0.4.4-0.20240814175157-ebc804c7119c/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= -go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b h1:iV7PdyUf7CC6slo4CgY+XuJ6gRS/HtZGjzLVm383rTo= -go.sia.tech/coreutils v0.2.6-0.20240814205841-6bd57953a01b/go.mod h1:TjQITC7A7u3sX22sN54SmcPcn+YmnodEqzNElAA7G/s= +go.sia.tech/core v0.4.4 h1:DYb0/DxgACstJUGgsRJIVtrsTC0mk6GfA6pTxQwzKV0= +go.sia.tech/core v0.4.4/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= +go.sia.tech/coreutils v0.3.0 h1:TutrhfNe8hq0GxWcibSRIVZQpFpBoKId7pFjxdvDIR8= +go.sia.tech/coreutils v0.3.0/go.mod h1:8DNsiy6Xon5R9M/FnaSzAi2wcATh98EsDV3N6iGq4yI= go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 8616c7847762513bfaba81ebf73153d5c9a5b1f5 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 29 Aug 2024 22:42:16 -0700 Subject: [PATCH 265/630] api,cmd,config: add public endpoints for running as a service --- api/api_test.go | 117 ++++++++++++++++++++++++++++++- api/server.go | 163 +++++++++++++++++++++++++++++--------------- cmd/walletd/main.go | 8 +-- config/config.go | 5 +- 4 files changed, 231 insertions(+), 62 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 48e20a8..1610e69 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -18,6 +18,7 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" + "go.sia.tech/coreutils/testutil" "go.sia.tech/jape" "go.sia.tech/walletd/api" "go.sia.tech/walletd/persist/sqlite" @@ -51,7 +52,7 @@ func runServer(t *testing.T, cm api.ChainManager, s api.Syncer, wm api.WalletMan t.Cleanup(func() { l.Close() }) server := &http.Server{ - Handler: jape.BasicAuth("password")(api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)))), + Handler: api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t))), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -1345,3 +1346,117 @@ func TestDebugMine(t *testing.T) { t.Fatalf("expected tip height to be 5, got %v", cm.Tip().Height) } } + +func TestAPISecurity(t *testing.T) { + n, genesisBlock := testutil.Network() + log := zaptest.NewLogger(t) + + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + ps, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + s := syncer.New(syncerListener, cm, ps, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + defer s.Close() + go s.Run(context.Background()) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + httpListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal("failed to listen:", err) + } + t.Cleanup(func() { httpListener.Close() }) + + server := &http.Server{ + Handler: api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + } + t.Cleanup(func() { server.Close() }) + + go server.Serve(httpListener) + + // create a client with correct credentials + c := api.NewClient("http://"+httpListener.Addr().String(), "test") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // create a client with incorrect credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "wrong") + if _, err := c.ConsensusTip(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + + // replace the handler with a new one that doesn't require auth + server.Handler = api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t))) + + // create a client without credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // create a client with incorrect credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "test") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // replace the handler with one that requires auth and has public endpoints + server.Handler = api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test"), api.WithPublicEndpoints(true)) + + // create a client without credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "") + + // check that a public endpoint is accessible + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // check that a private endpoint is still protected + if _, err := c.Wallets(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + + // create a client with credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "test") + + // check that both public and private endpoints are accessible + if _, err := c.Wallets(); err != nil { + t.Fatal(err) + } else if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } +} diff --git a/api/server.go b/api/server.go index f8300ae..58b09ca 100644 --- a/api/server.go +++ b/api/server.go @@ -40,6 +40,18 @@ func WithDebug() ServerOption { } } +func WithPublicEndpoints(public bool) ServerOption { + return func(s *server) { + s.publicEndpoints = public + } +} + +func WithBasicAuth(password string) ServerOption { + return func(s *server) { + s.password = password + } +} + type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { @@ -105,8 +117,10 @@ type ( ) type server struct { - startTime time.Time - debugEnabled bool + startTime time.Time + debugEnabled bool + publicEndpoints bool + password string log *zap.Logger cm ChainManager @@ -895,9 +909,10 @@ func (s *server) pprofHandler(jc jape.Context) { // NewServer returns an HTTP handler that serves the walletd API. func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { srv := server{ - log: zap.NewNop(), - debugEnabled: false, - startTime: time.Now(), + log: zap.NewNop(), + debugEnabled: false, + publicEndpoints: false, + startTime: time.Now(), cm: cm, s: s, @@ -907,60 +922,98 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption for _, opt := range opts { opt(&srv) } + + // checkAuth checks the request for basic authentication. + checkAuth := func(jc jape.Context) bool { + if srv.password == "" { + // unset password is equivalent to no auth + return true + } + + // verify auth header + _, pass, ok := jc.Request.BasicAuth() + if ok && pass == srv.password { + return true + } + + jc.Error(errors.New("unauthorized"), http.StatusUnauthorized) + return false + } + + // wrapAuthHandler wraps a jape handler with an authentication check. + wrapAuthHandler := func(h jape.Handler) jape.Handler { + return func(jc jape.Context) { + if !checkAuth(jc) { + return + } + h(jc) + } + } + + // wrapPublicAuthHandler wraps a jape handler with an authentication check + // unless publicEndpoints is true. + wrapPublicAuthHandler := func(h jape.Handler) jape.Handler { + return func(jc jape.Context) { + if !srv.publicEndpoints && !checkAuth(jc) { + return + } + h(jc) + } + } + handlers := map[string]jape.Handler{ - "GET /state": srv.stateHandler, - - "GET /consensus/network": srv.consensusNetworkHandler, - "GET /consensus/tip": srv.consensusTipHandler, - "GET /consensus/tipstate": srv.consensusTipStateHandler, - "GET /consensus/updates/:index": srv.consensusUpdatesIndexHandler, - "GET /consensus/index/:height": srv.consensusIndexHeightHandler, - - "GET /syncer/peers": srv.syncerPeersHandler, - "POST /syncer/connect": srv.syncerConnectHandler, - "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, - - "POST /txpool/parents": srv.txpoolParentsHandler, - "GET /txpool/transactions": srv.txpoolTransactionsHandler, - "GET /txpool/fee": srv.txpoolFeeHandler, - "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - - "GET /rescan": srv.rescanHandlerGET, - "POST /rescan": srv.rescanHandlerPOST, - - "GET /wallets": srv.walletsHandler, - "POST /wallets": srv.walletsHandlerPOST, - "POST /wallets/:id": srv.walletsIDHandlerPOST, - "DELETE /wallets/:id": srv.walletsIDHandlerDELETE, - "PUT /wallets/:id/addresses": srv.walletsAddressHandlerPUT, - "DELETE /wallets/:id/addresses/:addr": srv.walletsAddressHandlerDELETE, - "GET /wallets/:id/addresses": srv.walletsAddressesHandlerGET, - "GET /wallets/:id/balance": srv.walletsBalanceHandler, - "GET /wallets/:id/events": srv.walletsEventsHandler, - "GET /wallets/:id/events/unconfirmed": srv.walletsEventsUnconfirmedHandlerGET, - "GET /wallets/:id/outputs/siacoin": srv.walletsOutputsSiacoinHandler, - "GET /wallets/:id/outputs/siafund": srv.walletsOutputsSiafundHandler, - "POST /wallets/:id/reserve": srv.walletsReserveHandler, - "POST /wallets/:id/release": srv.walletsReleaseHandler, - "POST /wallets/:id/fund": srv.walletsFundHandler, - "POST /wallets/:id/fundsf": srv.walletsFundSFHandler, - - "GET /addresses/:addr/balance": srv.addressesAddrBalanceHandler, - "GET /addresses/:addr/events": srv.addressesAddrEventsHandlerGET, - "GET /addresses/:addr/events/unconfirmed": srv.addressesAddrEventsUnconfirmedHandlerGET, - "GET /addresses/:addr/outputs/siacoin": srv.addressesAddrOutputsSCHandler, - "GET /addresses/:addr/outputs/siafund": srv.addressesAddrOutputsSFHandler, - - "GET /outputs/siacoin/:id": srv.outputsSiacoinHandlerGET, - "GET /outputs/siafund/:id": srv.outputsSiafundHandlerGET, - - "GET /events/:id": srv.eventsHandlerGET, + "GET /state": wrapPublicAuthHandler(srv.stateHandler), + + "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), + "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), + "GET /consensus/tipstate": wrapPublicAuthHandler(srv.consensusTipStateHandler), + "GET /consensus/updates/:index": wrapPublicAuthHandler(srv.consensusUpdatesIndexHandler), + "GET /consensus/index/:height": wrapPublicAuthHandler(srv.consensusIndexHeightHandler), + + "POST /syncer/connect": wrapAuthHandler(srv.syncerConnectHandler), + "GET /syncer/peers": wrapPublicAuthHandler(srv.syncerPeersHandler), + "POST /syncer/broadcast/block": wrapPublicAuthHandler(srv.syncerBroadcastBlockHandler), + + "GET /txpool/transactions": wrapPublicAuthHandler(srv.txpoolTransactionsHandler), + "GET /txpool/fee": wrapPublicAuthHandler(srv.txpoolFeeHandler), + "POST /txpool/parents": wrapPublicAuthHandler(srv.txpoolParentsHandler), + "POST /txpool/broadcast": wrapPublicAuthHandler(srv.txpoolBroadcastHandler), + + "GET /addresses/:addr/balance": wrapPublicAuthHandler(srv.addressesAddrBalanceHandler), + "GET /addresses/:addr/events": wrapPublicAuthHandler(srv.addressesAddrEventsHandlerGET), + "GET /addresses/:addr/events/unconfirmed": wrapPublicAuthHandler(srv.addressesAddrEventsUnconfirmedHandlerGET), + "GET /addresses/:addr/outputs/siacoin": wrapPublicAuthHandler(srv.addressesAddrOutputsSCHandler), + "GET /addresses/:addr/outputs/siafund": wrapPublicAuthHandler(srv.addressesAddrOutputsSFHandler), + + "GET /outputs/siacoin/:id": wrapPublicAuthHandler(srv.outputsSiacoinHandlerGET), + "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), + + "GET /events/:id": wrapPublicAuthHandler(srv.eventsHandlerGET), + + "GET /rescan": wrapAuthHandler(srv.rescanHandlerGET), + "POST /rescan": wrapAuthHandler(srv.rescanHandlerPOST), + + "GET /wallets": wrapAuthHandler(srv.walletsHandler), + "POST /wallets": wrapAuthHandler(srv.walletsHandlerPOST), + "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), + "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), + "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), + "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), + "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), + "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), + "GET /wallets/:id/events": wrapAuthHandler(srv.walletsEventsHandler), + "GET /wallets/:id/events/unconfirmed": wrapAuthHandler(srv.walletsEventsUnconfirmedHandlerGET), + "GET /wallets/:id/outputs/siacoin": wrapAuthHandler(srv.walletsOutputsSiacoinHandler), + "GET /wallets/:id/outputs/siafund": wrapAuthHandler(srv.walletsOutputsSiafundHandler), + "POST /wallets/:id/reserve": wrapAuthHandler(srv.walletsReserveHandler), + "POST /wallets/:id/release": wrapAuthHandler(srv.walletsReleaseHandler), + "POST /wallets/:id/fund": wrapAuthHandler(srv.walletsFundHandler), + "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), } if srv.debugEnabled { - handlers["POST /debug/mine"] = srv.debugMineHandler - handlers["GET /debug/pprof/:handler"] = srv.pprofHandler + handlers["POST /debug/mine"] = wrapAuthHandler(srv.debugMineHandler) + handlers["GET /debug/pprof/:handler"] = wrapAuthHandler(srv.pprofHandler) } - return jape.Mux(handlers) } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 0915146..536844c 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -55,8 +55,9 @@ var cfg = config.Config{ Directory: ".", AutoOpenWebUI: true, HTTP: config.HTTP{ - Address: "localhost:9980", - Password: os.Getenv("WALLETD_API_PASSWORD"), + Address: "localhost:9980", + Password: os.Getenv("WALLETD_API_PASSWORD"), + PublicEndpoints: false, }, Syncer: config.Syncer{ Address: ":9981", @@ -122,7 +123,6 @@ func tryLoadConfig() { if str := os.Getenv("WALLETD_CONFIG_FILE"); str != "" { configPath = str } - fmt.Println("loading config from", configPath) // If the config file doesn't exist, don't try to load it. if _, err := os.Stat(configPath); os.IsNotExist(err) { @@ -143,7 +143,6 @@ func tryLoadConfig() { fmt.Println("failed to decode config file:", err) os.Exit(1) } - fmt.Println("config loaded") } // jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for @@ -206,6 +205,7 @@ func main() { rootCmd.BoolVar(&enableDebug, "debug", false, "enable debug mode with additional profiling and mining endpoints") rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") + rootCmd.BoolVar(&cfg.HTTP.PublicEndpoints, "http.public", cfg.HTTP.PublicEndpoints, "disables auth on endpoints that should be publicly accessible when running walletd as a service") rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") diff --git a/config/config.go b/config/config.go index 7372714..dd18249 100644 --- a/config/config.go +++ b/config/config.go @@ -5,8 +5,9 @@ import "go.sia.tech/walletd/wallet" type ( // HTTP contains the configuration for the HTTP server. HTTP struct { - Address string `yaml:"address,omitempty"` - Password string `yaml:"password,omitempty"` + Address string `yaml:"address,omitempty"` + Password string `yaml:"password,omitempty"` + PublicEndpoints bool `yaml:"publicEndpoints,omitempty"` } // Syncer contains the configuration for the consensus set syncer. From 1d09a2a5329628afb49e3478ffb28f647ee90742 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 29 Aug 2024 22:43:53 -0700 Subject: [PATCH 266/630] docs: update readme --- README.md | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 5924dc5..c1483a6 100644 --- a/README.md +++ b/README.md @@ -65,22 +65,36 @@ The priority of configuration settings is as follows: ### Command Line Flags ``` --addr string - p2p address to listen on (default ":9981") --bootstrap - attempt to bootstrap the network (default true) --dir string - directory to store node state in (default "/Users/n8maninger/Downloads/walletd-tmp") --http string - address to serve API on (default "localhost:9980") --index.batch int - max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 64) --index.mode string - address index mode (personal, full, none) (default "full") --network string - network to connect to (default "mainnet") --upnp - attempt to forward ports and discover IP with UPnP +Usage: + walletd [flags] [action] + +Run 'walletd' with no arguments to start the blockchain node and API server. + +Actions: + version print walletd version + seed generate a recovery phrase + mine run CPU miner +Flags: + -addr string + p2p address to listen on (default ":9981") + -bootstrap + attempt to bootstrap the network (default true) + -debug + enable debug mode with additional profiling and mining endpoints + -dir string + directory to store node state in (default "/Users/n8maninger/Downloads/walletd-tmp") + -http string + address to serve API on (default "localhost:9980") + -http.public + disables auth on endpoints that should be publicly accessible when running walletd as a service + -index.batch int + max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 1000) + -index.mode string + address index mode (personal, full, none) (default "full") + -network string + network to connect to (default "mainnet") + -upnp + attempt to forward ports and discover IP with UPnP ``` ### YAML @@ -92,6 +106,7 @@ autoOpenWebUI: true http: address: :9980 password: sia is cool + publicEndpoints: false # when true, auth will be disabled on endpoints that should be publicly accessible when running walletd as a service consensus: network: mainnet gatewayAddress: :9981 From 77bb01bf70a8513b2a43305e4fcfd260da8ece48 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 29 Aug 2024 22:48:17 -0700 Subject: [PATCH 267/630] cmd: move api auth to api package --- cmd/walletd/node.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 476808b..fd3d121 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -17,7 +17,6 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" - "go.sia.tech/jape" "go.sia.tech/walletd/api" "go.sia.tech/walletd/build" "go.sia.tech/walletd/config" @@ -150,11 +149,13 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu apiOpts := []api.ServerOption{ api.WithLogger(log.Named("api")), + api.WithPublicEndpoints(cfg.HTTP.PublicEndpoints), + api.WithBasicAuth(cfg.HTTP.Password), } if enableDebug { apiOpts = append(apiOpts, api.WithDebug()) } - api := jape.BasicAuth(cfg.HTTP.Password)(api.NewServer(cm, s, wm, apiOpts...)) + api := api.NewServer(cm, s, wm, apiOpts...) web := walletd.Handler() server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 73dce134f33870c7ab3c726eacaf2c2fb5b0d0e5 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 29 Aug 2024 22:49:33 -0700 Subject: [PATCH 268/630] api: fix lint --- api/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/server.go b/api/server.go index 58b09ca..661d69c 100644 --- a/api/server.go +++ b/api/server.go @@ -40,12 +40,15 @@ func WithDebug() ServerOption { } } +// WithPublicEndpoints sets whether the server should disable authentication +// on endpoints that are safe for use when running walletd as a service. func WithPublicEndpoints(public bool) ServerOption { return func(s *server) { s.publicEndpoints = public } } +// WithBasicAuth sets the password for basic authentication. func WithBasicAuth(password string) ServerOption { return func(s *server) { s.password = password From 9652eb2fed14c402794c8505a6e90a103bc99875 Mon Sep 17 00:00:00 2001 From: smk762 Date: Sat, 31 Aug 2024 14:46:47 +0800 Subject: [PATCH 269/630] add missing env vars & fix yaml structure --- README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c1483a6..3ea97d6 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,9 @@ The priority of configuration settings is as follows: + `9981` Sia consensus ### Environment Variables -+ `WALLETD_API_PASSWORD` - The password required to access the API ++ `WALLETD_API_PASSWORD` - The password required to access the API. ++ `WALLETD_CONFIG_FILE` - The path to the YAML configuration file. Defaults to `walletd.yml` in the working directory. ++ `WALLETD_LOG_FILE` - The path to the log file. ### Command Line Flags ``` @@ -99,19 +101,21 @@ Flags: ### YAML All configuration settings can be set in a YAML file. The file should be named -`walletd.yaml` in the working directory. All fields are optional. +`walletd.yml` in the working directory. All fields are optional. ```yaml directory: /etc/walletd autoOpenWebUI: true http: - address: :9980 + address: 9980 password: sia is cool publicEndpoints: false # when true, auth will be disabled on endpoints that should be publicly accessible when running walletd as a service consensus: network: mainnet - gatewayAddress: :9981 +syncer: bootstrap: false enableUPnP: false + peers: [] + address: 9981 index: mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) From 8edf2c34a7f22041e66627e419fcc0756c137fa1 Mon Sep 17 00:00:00 2001 From: smk762 Date: Sat, 31 Aug 2024 14:54:34 +0800 Subject: [PATCH 270/630] restore port prefix --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ea97d6..093f2eb 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ All configuration settings can be set in a YAML file. The file should be named directory: /etc/walletd autoOpenWebUI: true http: - address: 9980 + address: :9980 password: sia is cool publicEndpoints: false # when true, auth will be disabled on endpoints that should be publicly accessible when running walletd as a service consensus: @@ -115,7 +115,7 @@ syncer: bootstrap: false enableUPnP: false peers: [] - address: 9981 + address: :9981 index: mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) From d720a4d57aa91ebe6c109a758089f7f95e11b0b4 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 31 Aug 2024 19:57:25 -0700 Subject: [PATCH 271/630] deps: update jape --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6f50f94..0d3fb4e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.22 go.sia.tech/core v0.4.4 go.sia.tech/coreutils v0.3.0 - go.sia.tech/jape v0.12.0 + go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.23.0 diff --git a/go.sum b/go.sum index 80ba106..cd83f41 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ go.sia.tech/core v0.4.4 h1:DYb0/DxgACstJUGgsRJIVtrsTC0mk6GfA6pTxQwzKV0= go.sia.tech/core v0.4.4/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= go.sia.tech/coreutils v0.3.0 h1:TutrhfNe8hq0GxWcibSRIVZQpFpBoKId7pFjxdvDIR8= go.sia.tech/coreutils v0.3.0/go.mod h1:8DNsiy6Xon5R9M/FnaSzAi2wcATh98EsDV3N6iGq4yI= -go.sia.tech/jape v0.12.0 h1:13fBi7c5X8zxTQ05Cd9ZsIfRJgdvGoZqbEzH861z7BU= -go.sia.tech/jape v0.12.0/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= +go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= +go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= From 1eae42fd50dfc66e6c672a6a26149025ae991e82 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 31 Aug 2024 19:58:09 -0700 Subject: [PATCH 272/630] api: return 204 for empty handlers --- api/server.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/api/server.go b/api/server.go index 661d69c..51df847 100644 --- a/api/server.go +++ b/api/server.go @@ -243,7 +243,10 @@ func (s *server) syncerConnectHandler(jc jape.Context) { return } _, err := s.s.Connect(jc.Request.Context(), addr) - jc.Check("couldn't connect to peer", err) + if jc.Check("couldn't connect to peer", err) != nil { + return + } + jc.EmptyResonse() } func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { @@ -263,6 +266,7 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { } else { s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) } + jc.EmptyResonse() } func (s *server) txpoolParentsHandler(jc jape.Context) { @@ -305,6 +309,8 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } s.s.BroadcastV2TransactionSet(index, tbr.V2Transactions) } + + jc.EmptyResonse() } func (s *server) walletsHandler(jc jape.Context) { @@ -367,6 +373,7 @@ func (s *server) walletsIDHandlerDELETE(jc jape.Context) { } else if jc.Check("couldn't remove wallet", err) != nil { return } + jc.EmptyResonse() } func (s *server) rescanHandlerGET(jc jape.Context) { @@ -425,6 +432,8 @@ func (s *server) rescanHandlerPOST(jc jape.Context) { s.scanInfo.Error = &msg } }() + + jc.EmptyResonse() } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { @@ -435,6 +444,7 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { } else if jc.Check("couldn't add address", s.wm.AddAddress(id, addr)) != nil { return } + jc.EmptyResonse() } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { @@ -450,6 +460,7 @@ func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { } else if jc.Check("couldn't remove address", err) != nil { return } + jc.EmptyResonse() } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { @@ -568,6 +579,7 @@ func (s *server) walletsReserveHandler(jc jape.Context) { if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids, wrr.Duration)) != nil { return } + jc.EmptyResonse() } func (s *server) walletsReleaseHandler(jc jape.Context) { @@ -584,6 +596,7 @@ func (s *server) walletsReleaseHandler(jc jape.Context) { for _, id := range wrr.SiafundOutputs { delete(s.used, types.Hash256(id)) } + jc.EmptyResonse() } func (s *server) walletsFundHandler(jc jape.Context) { @@ -886,6 +899,7 @@ func (s *server) debugMineHandler(jc jape.Context) { log.Debug("mined block", zap.Stringer("blockID", b.ID())) n-- } + jc.EmptyResonse() } func (s *server) pprofHandler(jc jape.Context) { From b256f1a2e9bb04998ecc114905c1d5cd56207d07 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sat, 31 Aug 2024 20:02:27 -0700 Subject: [PATCH 273/630] api: add test --- api/api_test.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index 1610e69..d8f3191 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,6 +1,7 @@ package api_test import ( + "bytes" "context" "encoding/hex" "encoding/json" @@ -1460,3 +1461,70 @@ func TestAPISecurity(t *testing.T) { t.Fatal(err) } } + +func TestAPINoContent(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testNetwork() + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + l, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + ps, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + s := syncer.New(l, cm, ps, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l.Addr().String(), + }) + defer s.Close() + go s.Run(context.Background()) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + buf, err := json.Marshal(api.TxpoolBroadcastRequest{ + Transactions: []types.Transaction{}, + V2Transactions: []types.V2Transaction{}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodPost, c.BaseURL()+"/txpool/broadcast", bytes.NewReader(buf)) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("expected status %v, got %v", http.StatusNoContent, resp.StatusCode) + } else if resp.ContentLength != 0 { + t.Fatalf("expected no content, got %v bytes", resp.ContentLength) + } +} From 62bce1757fd6d0e3a11861479b55c70ce4dc0d35 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sun, 1 Sep 2024 15:22:11 -0700 Subject: [PATCH 274/630] deps: update core and coreutils --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0d3fb4e..22a34bd 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.22.5 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.4 - go.sia.tech/coreutils v0.3.0 + go.sia.tech/core v0.4.5 + go.sia.tech/coreutils v0.3.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 @@ -19,7 +19,7 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index cd83f41..3322e38 100644 --- a/go.sum +++ b/go.sum @@ -10,12 +10,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= -go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -go.sia.tech/core v0.4.4 h1:DYb0/DxgACstJUGgsRJIVtrsTC0mk6GfA6pTxQwzKV0= -go.sia.tech/core v0.4.4/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= -go.sia.tech/coreutils v0.3.0 h1:TutrhfNe8hq0GxWcibSRIVZQpFpBoKId7pFjxdvDIR8= -go.sia.tech/coreutils v0.3.0/go.mod h1:8DNsiy6Xon5R9M/FnaSzAi2wcATh98EsDV3N6iGq4yI= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.sia.tech/core v0.4.5 h1:w2D3Mx29UmK1aFd9R7uHFo5JUSTqu3+92NHoRFv3CaU= +go.sia.tech/core v0.4.5/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= +go.sia.tech/coreutils v0.3.1 h1:FLIBM4ryLFvwkZVv8Yyn3KZsUdqX6pX8moS3xfQX5M0= +go.sia.tech/coreutils v0.3.1/go.mod h1:qOBvtTS14Q2lSbY+S3u39AntpttL5kuIv7P8NktKYw0= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 12476680ef708d751a5559ed4afcb1a28bf148a9 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sun, 1 Sep 2024 19:24:29 -0700 Subject: [PATCH 275/630] deps: update --- cmd/walletd/node.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index fd3d121..d419563 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -171,7 +171,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu defer server.Close() go server.Serve(httpListener) - log.Info("node started", zap.Stringer("syncer", syncerListener.Addr()), zap.Stringer("http", httpListener.Addr()), zap.String("version", build.Version()), zap.String("commit", build.Commit())) + log.Info("node started", zap.String("network", network.Name), zap.Stringer("syncer", syncerListener.Addr()), zap.Stringer("http", httpListener.Addr()), zap.String("version", build.Version()), zap.String("commit", build.Commit())) <-ctx.Done() log.Info("shutting down") return nil diff --git a/go.mod b/go.mod index 22a34bd..c04e025 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.22.5 require ( github.com/mattn/go-sqlite3 v1.14.22 - go.sia.tech/core v0.4.5 - go.sia.tech/coreutils v0.3.1 + go.sia.tech/core v0.4.6 + go.sia.tech/coreutils v0.3.2 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 3322e38..da99917 100644 --- a/go.sum +++ b/go.sum @@ -12,10 +12,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.4.5 h1:w2D3Mx29UmK1aFd9R7uHFo5JUSTqu3+92NHoRFv3CaU= -go.sia.tech/core v0.4.5/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= -go.sia.tech/coreutils v0.3.1 h1:FLIBM4ryLFvwkZVv8Yyn3KZsUdqX6pX8moS3xfQX5M0= -go.sia.tech/coreutils v0.3.1/go.mod h1:qOBvtTS14Q2lSbY+S3u39AntpttL5kuIv7P8NktKYw0= +go.sia.tech/core v0.4.6 h1:QLm97a7GWBonfnMEOokqWRAqsWCUPL7kzo6k3Adwx8E= +go.sia.tech/core v0.4.6/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= +go.sia.tech/coreutils v0.3.2 h1:3gJqvs18n1FVZmcrnfIYyzS+rBu06OtIscDDAfUAYQI= +go.sia.tech/coreutils v0.3.2/go.mod h1:woPVmN6GUpIKHdi71Hkb9goIbl7b45TquCsAyEzyxnI= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= From 134a28b063df60a687899ac33aa373bf461480bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:47:24 +0000 Subject: [PATCH 276/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.22 to 1.14.23 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.22...v1.14.23) Updates `golang.org/x/term` from 0.23.0 to 0.24.0 - [Commits](https://github.com/golang/term/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c04e025..bb08fa6 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module go.sia.tech/walletd go 1.22.5 require ( - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.23 go.sia.tech/core v0.4.6 go.sia.tech/coreutils v0.3.2 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.23.0 + golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 @@ -24,6 +24,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/sys v0.24.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index da99917..c100601 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= +github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= @@ -37,10 +37,10 @@ golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 32570732f449b0c2c1f1c335d976506e4bff4f36 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 12 Sep 2024 10:21:17 -0700 Subject: [PATCH 277/630] api: change tpool response to 400 --- api/server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/server.go b/api/server.go index 51df847..224fcd3 100644 --- a/api/server.go +++ b/api/server.go @@ -3,6 +3,7 @@ package api import ( "context" "errors" + "fmt" "net/http" "net/http/pprof" "reflect" @@ -296,7 +297,8 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } if len(tbr.Transactions) != 0 { _, err := s.cm.AddPoolTransactions(tbr.Transactions) - if jc.Check("invalid transaction set", err) != nil { + if err != nil { + jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return } s.s.BroadcastTransactionSet(tbr.Transactions) @@ -304,7 +306,8 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if len(tbr.V2Transactions) != 0 { index := s.cm.TipState().Index _, err := s.cm.AddV2PoolTransactions(index, tbr.V2Transactions) - if jc.Check("invalid v2 transaction set", err) != nil { + if err != nil { + jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return } s.s.BroadcastV2TransactionSet(index, tbr.V2Transactions) From c669216bcfb11e4ab0d219e120aca79920f82d29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:16:39 +0000 Subject: [PATCH 278/630] build(deps): bump the all-dependencies group across 1 directory with 2 updates Bumps the all-dependencies group with 2 updates in the / directory: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.4.6 to 0.4.7 - [Commits](https://github.com/SiaFoundation/core/compare/v0.4.6...v0.4.7) Updates `go.sia.tech/coreutils` from 0.3.2 to 0.4.0 - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.3.2...v0.4.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 9 +++++---- go.sum | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index bb08fa6..8127cd9 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,12 @@ module go.sia.tech/walletd go 1.22.5 +toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.23 - go.sia.tech/core v0.4.6 - go.sia.tech/coreutils v0.3.2 + go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf + go.sia.tech/coreutils v0.4.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 @@ -20,10 +21,10 @@ require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect go.etcd.io/bbolt v1.3.11 // indirect - go.sia.tech/mux v1.2.0 // indirect + go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index c100601..9e6e239 100644 --- a/go.sum +++ b/go.sum @@ -12,14 +12,14 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.4.6 h1:QLm97a7GWBonfnMEOokqWRAqsWCUPL7kzo6k3Adwx8E= -go.sia.tech/core v0.4.6/go.mod h1:Zuq0Tn2aIXJyO0bjGu8cMeVWe+vwQnUfZhG1LCmjD5c= -go.sia.tech/coreutils v0.3.2 h1:3gJqvs18n1FVZmcrnfIYyzS+rBu06OtIscDDAfUAYQI= -go.sia.tech/coreutils v0.3.2/go.mod h1:woPVmN6GUpIKHdi71Hkb9goIbl7b45TquCsAyEzyxnI= +go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf h1:x/lM7Y8Rlo12rcpPXapLvSVNyrHZEKO6j4eLNccPMKw= +go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M= +go.sia.tech/coreutils v0.4.0 h1:GkxJ2B7upm3/yhIOIku5oafbL/snd7tdmkj0NjrugnI= +go.sia.tech/coreutils v0.4.0/go.mod h1:dmpPtY/XVl7o0Pcyx2XpV6ujduJvNIr3ydvi3xdnXuI= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= -go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= -go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= +go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= +go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.23.0 h1:5ftJQQwUHG8TYzdzSb+Y1IIPC0jkjNeAuoLUbsv9UTE= @@ -30,8 +30,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= From 66d48b0a33e2d4baab3b7afcabf6b25a5c4624ab Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 3 Oct 2024 09:20:07 -0400 Subject: [PATCH 279/630] deps: tidy --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8127cd9..ba08fc7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module go.sia.tech/walletd -go 1.22.5 +go 1.23.0 + toolchain go1.23.2 require ( From 847affc9dc646d8df9126c9c5a3d719816f08325 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 16:49:47 +0000 Subject: [PATCH 280/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.23 to 1.14.24 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.23...v1.14.24) Updates `golang.org/x/term` from 0.24.0 to 0.25.0 - [Commits](https://github.com/golang/term/compare/v0.24.0...v0.25.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index ba08fc7..6817015 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.23.0 toolchain go1.23.2 require ( - github.com/mattn/go-sqlite3 v1.14.23 + github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf go.sia.tech/coreutils v0.4.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.24.0 + golang.org/x/term v0.25.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.4.2 @@ -26,6 +26,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.27.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 9e6e239..5618991 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/mattn/go-sqlite3 v1.14.23 h1:gbShiuAP1W5j9UOksQ06aiiqPMxYecovVGwmTxWtuw0= -github.com/mattn/go-sqlite3 v1.14.23/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= @@ -37,10 +37,10 @@ golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 04d3ab2f27ee095f9db359defbf78fe07e4825d5 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 8 Oct 2024 12:06:47 -0700 Subject: [PATCH 281/630] sqlite: fix duplicate events --- persist/sqlite/wallet.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 95564b4..5ad3e6e 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -671,14 +671,30 @@ func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet. return nil, nil, nil } - const eventsQuery = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data + const eventsQuery = `WITH event_ids AS ( + SELECT + ev.id FROM events ev - INNER JOIN event_addresses ea ON (ev.id = ea.event_id) - INNER JOIN wallet_addresses wa ON (ea.address_id = wa.address_id) - INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) - WHERE wa.wallet_id=$1 + INNER JOIN event_addresses ea ON ev.id = ea.event_id + INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id + WHERE wa.wallet_id = $1 + GROUP BY ev.id ORDER BY ev.maturity_height DESC, ev.id DESC - LIMIT $2 OFFSET $3` + LIMIT $2 OFFSET $3 +) +SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + ev.event_type, + ev.event_data +FROM events ev +INNER JOIN event_ids ei ON ev.id = ei.id +INNER JOIN chain_indices ci ON ev.chain_index_id = ci.id +ORDER BY ev.maturity_height DESC, ev.id DESC;` rows, err := tx.Query(eventsQuery, id, limit, offset) if err != nil { From a145dd24c68a25d390425f31bd88249e95b8b306 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:41:04 +0000 Subject: [PATCH 282/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [lukechampine.com/frand](https://github.com/lukechampine/frand). Updates `go.sia.tech/coreutils` from 0.4.0 to 0.4.1 - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.4.0...v0.4.1) Updates `lukechampine.com/frand` from 1.4.2 to 1.5.1 - [Commits](https://github.com/lukechampine/frand/compare/v1.4.2...v1.5.1) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: lukechampine.com/frand dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 ++---- go.sum | 11 ++++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 6817015..acc64aa 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,23 @@ module go.sia.tech/walletd go 1.23.0 - toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf - go.sia.tech/coreutils v0.4.0 + go.sia.tech/coreutils v0.4.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.25.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 - lukechampine.com/frand v1.4.2 + lukechampine.com/frand v1.5.1 lukechampine.com/upnp v0.3.0 ) require ( - github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect go.etcd.io/bbolt v1.3.11 // indirect go.sia.tech/mux v1.3.0 // indirect diff --git a/go.sum b/go.sum index 5618991..fde8aea 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -14,8 +12,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf h1:x/lM7Y8Rlo12rcpPXapLvSVNyrHZEKO6j4eLNccPMKw= go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M= -go.sia.tech/coreutils v0.4.0 h1:GkxJ2B7upm3/yhIOIku5oafbL/snd7tdmkj0NjrugnI= -go.sia.tech/coreutils v0.4.0/go.mod h1:dmpPtY/XVl7o0Pcyx2XpV6ujduJvNIr3ydvi3xdnXuI= +go.sia.tech/coreutils v0.4.1 h1:ExQ9g6EtnFe70ptNBG+OtZyFU3aBoEzE/06rtbN6f4c= +go.sia.tech/coreutils v0.4.1/go.mod h1:v60kPqZERsb1ZS0PVe4S8hr2ArNEwTdp7XTzErXnV2U= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= @@ -36,7 +34,6 @@ golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= @@ -49,7 +46,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= lukechampine.com/flagg v1.1.1/go.mod h1:a9ZuZu5LSPXELWSJrabRD00ort+lDXSOQu34xWgEoDI= -lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= -lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= +lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w= +lukechampine.com/frand v1.5.1/go.mod h1:4VstaWc2plN4Mjr10chUD46RAVGWhpkZ5Nja8+Azp0Q= lukechampine.com/upnp v0.3.0 h1:UVCD6eD6fmJmwak6DVE3vGN+L46Fk8edTcC6XYCb6C4= lukechampine.com/upnp v0.3.0/go.mod h1:sOuF+fGSDKjpUm6QI0mfb82ScRrhj8bsqsD78O5nK1k= From aba0697567ff0027bfc0cb10a95a0e6776ffc967 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 21 Oct 2024 13:07:40 +0200 Subject: [PATCH 283/630] go mod tidy --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index acc64aa..6dc416e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module go.sia.tech/walletd -go 1.23.0 +go 1.23.1 + toolchain go1.23.2 require ( From 397c950a0e2d75543b354d6323038400f31580dc Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 21 Oct 2024 09:43:38 -0700 Subject: [PATCH 284/630] Support confirmation field in events (#185) * sqlite: support confirmations field in events * sqlite: reset confirmations during rescan --- go.mod | 4 ++-- go.sum | 8 ++++---- persist/sqlite/addresses.go | 33 +++++++++++++++++++++++++-------- persist/sqlite/consensus.go | 6 +++--- persist/sqlite/events.go | 31 ++++++++++++++++++++++++------- persist/sqlite/init.go | 2 +- persist/sqlite/init.sql | 3 ++- persist/sqlite/migrations.go | 34 ++++++++++++++++++++++++++++++++++ persist/sqlite/wallet.go | 11 ++++++++++- 9 files changed, 105 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 6dc416e..88f79a6 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf - go.sia.tech/coreutils v0.4.1 + go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.0 go.uber.org/zap v1.27.0 @@ -24,7 +24,7 @@ require ( go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index fde8aea..d564812 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf h1:x/lM7Y8Rlo12rcpPXapLvSVNyrHZEKO6j4eLNccPMKw= go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M= -go.sia.tech/coreutils v0.4.1 h1:ExQ9g6EtnFe70ptNBG+OtZyFU3aBoEzE/06rtbN6f4c= -go.sia.tech/coreutils v0.4.1/go.mod h1:v60kPqZERsb1ZS0PVe4S8hr2ArNEwTdp7XTzErXnV2U= +go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93 h1:38XTWxsDR4Q8ILQHFFSoB906jRrcb1UBaWg5d8MBt9M= +go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93/go.mod h1:JIaR+zdGZsqPLBM5mVsnwWJ7hBsES+SAEDQg5EFBitM= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= @@ -28,8 +28,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 52f4ec4..64e3d74 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -27,14 +27,31 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e // AddressEvents returns the events of a single address. func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts - INNER JOIN event_addresses ea ON (ev.id = ea.event_id) - INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) - INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) - WHERE sa.sia_address = $1 - ORDER BY ev.maturity_height DESC, ev.id DESC - LIMIT $2 OFFSET $3` + const query = ` +WITH last_chain_index AS ( + SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 +) +SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + CASE + WHEN last_chain_index.height < ci.height THEN 0 + ELSE last_chain_index.height - ci.height + END AS confirmations, + ev.event_type, + ev.event_data +FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts +INNER JOIN event_addresses ea ON (ev.id = ea.event_id) +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) +CROSS JOIN last_chain_index +WHERE sa.sia_address = $1 +ORDER BY ev.maturity_height DESC, ev.id DESC +LIMIT $2 OFFSET $3` rows, err := tx.Query(query, encode(address), limit, offset) if err != nil { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 4518933..fa6ea33 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -280,13 +280,13 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. // LastCommittedIndex returns the last chain index that was committed. func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { - err = s.db.QueryRow(`SELECT last_indexed_tip FROM global_settings`).Scan(decode(&index)) + err = s.db.QueryRow(`SELECT last_indexed_height, last_indexed_id FROM global_settings`).Scan(&index.Height, decode(&index.ID)) return } // ResetLastIndex resets the last indexed tip to trigger a full rescan. func (s *Store) ResetLastIndex() error { - _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_tip=$1`, encode(types.ChainIndex{})) + _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1`, encode(types.BlockID{})) return err } @@ -1324,7 +1324,7 @@ func pruneSpentSiafundElements(tx *txn, height uint64) (removed int64, err error } func setGlobalState(tx *txn, index types.ChainIndex, numLeaves uint64) error { - _, err := tx.Exec(`UPDATE global_settings SET last_indexed_tip=$1, element_num_leaves=$2`, encode(index), numLeaves) + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_height=$1, last_indexed_id=$2, element_num_leaves=$3`, index.Height, encode(index.ID), numLeaves) return err } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index de7dc2d..69c62da 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -17,12 +17,29 @@ func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err err // sqlite doesn't have easy support for IN clauses, use a statement since // the number of event IDs is likely to be small instead of dynamically // building the query - const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, ev.event_type, ev.event_data - FROM events ev - INNER JOIN event_addresses ea ON (ev.id = ea.event_id) - INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) - INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) - WHERE ev.event_id = $1` + const query = ` +WITH last_chain_index AS ( + SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 +) +SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + CASE + WHEN last_chain_index.height < ci.height THEN 0 + ELSE last_chain_index.height - ci.height + END AS confirmations, + ev.event_type, + ev.event_data +FROM events ev +INNER JOIN event_addresses ea ON (ev.id = ea.event_id) +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) +CROSS JOIN last_chain_index +WHERE ev.event_id = $1` stmt, err := tx.Prepare(query) if err != nil { @@ -48,7 +65,7 @@ func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err err func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { var eventBuf []byte - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Confirmations, &ev.Type, &eventBuf) if err != nil { return } diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index 9579e04..2949588 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -18,7 +18,7 @@ import ( var initDatabase string func initializeSettings(tx *txn, target int64) error { - _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_tip, element_num_leaves) VALUES (0, ?, ?, ?)`, target, encode(types.ChainIndex{}), 0) + _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_height, last_indexed_id, element_num_leaves) VALUES (0, ?, 0, ?, 0)`, target, encode(types.BlockID{})) return err } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 3517b44..37eba72 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -110,6 +110,7 @@ CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row db_version INTEGER NOT NULL, -- used for migrations index_mode INTEGER, -- the mode of the data store - last_indexed_tip BLOB NOT NULL, -- the last chain index that was processed + last_indexed_height INTEGER NOT NULL, -- the height of the last chain index that was processed + last_indexed_id BLOB NOT NULL, -- the block ID of the last chain index that was processed element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree ); diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 79544ec..8429fe4 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -1,9 +1,42 @@ package sqlite import ( + "fmt" + + "go.sia.tech/core/types" "go.uber.org/zap" ) +// migrateVersion4 splits the height and ID of the last indexed tip into two +// separate columns for easier querying. +func migrateVersion4(tx *txn, _ *zap.Logger) error { + var dbVersion int + var indexMode int + var elementNumLeaves uint64 + var index types.ChainIndex + err := tx.QueryRow(`SELECT db_version, index_mode, element_num_leaves, last_indexed_tip FROM global_settings`).Scan(&dbVersion, &indexMode, &elementNumLeaves, decode(&index)) + if err != nil { + return fmt.Errorf("failed to get last indexed tip: %w", err) + } else if _, err := tx.Exec(`DROP TABLE global_settings`); err != nil { + return fmt.Errorf("failed to drop global_settings: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + index_mode INTEGER, -- the mode of the data store + last_indexed_height INTEGER NOT NULL, -- the height of the last chain index that was processed + last_indexed_id BLOB NOT NULL, -- the block ID of the last chain index that was processed + element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree +);`) + if err != nil { + return fmt.Errorf("failed to create global_settings: %w", err) + } + + _, err = tx.Exec(`INSERT INTO global_settings (id, db_version, index_mode, last_indexed_height, last_indexed_id, element_num_leaves) VALUES (0, ?, ?, ?, ?, ?)`, dbVersion, indexMode, index.Height, encode(index.ID), elementNumLeaves) + return err +} + // migrateVersion3 adds additional indices to event_addresses and wallet_addresses // to improve query performance. func migrateVersion3(tx *txn, _ *zap.Logger) error { @@ -57,4 +90,5 @@ CREATE INDEX IF NOT EXISTS syncer_bans_expiration_index_idx ON syncer_bans (expi var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion2, migrateVersion3, + migrateVersion4, } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 5ad3e6e..05f8f76 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -671,7 +671,11 @@ func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet. return nil, nil, nil } - const eventsQuery = `WITH event_ids AS ( + const eventsQuery = ` +WITH last_chain_index AS ( + SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 +), +event_ids AS ( SELECT ev.id FROM events ev @@ -689,11 +693,16 @@ SELECT ev.date_created, ci.height, ci.block_id, + CASE + WHEN last_chain_index.height < ci.height THEN 0 + ELSE last_chain_index.height - ci.height + END AS confirmations, ev.event_type, ev.event_data FROM events ev INNER JOIN event_ids ei ON ev.id = ei.id INNER JOIN chain_indices ci ON ev.chain_index_id = ci.id +CROSS JOIN last_chain_index ORDER BY ev.maturity_height DESC, ev.id DESC;` rows, err := tx.Query(eventsQuery, id, limit, offset) From 18264a5c22311040fff65a019ce6d5ccd364e8c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:45:12 +0000 Subject: [PATCH 285/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.23.0 to 0.23.1 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.23.0...walletd@0.23.1) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 88f79a6..cbdcacd 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.23.0 + go.sia.tech/web/walletd v0.23.1 go.uber.org/zap v1.27.0 golang.org/x/term v0.25.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index d564812..6e718e4 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.23.0 h1:5ftJQQwUHG8TYzdzSb+Y1IIPC0jkjNeAuoLUbsv9UTE= -go.sia.tech/web/walletd v0.23.0/go.mod h1:ZytUl1hnaZuxshPk8VE1K7Bcsy3IfmNmet3KVVEEE1E= +go.sia.tech/web/walletd v0.23.1 h1:z0S3tHLpxnqedkXbsZIa+VJTJmZpvfabpRurqmoKL4w= +go.sia.tech/web/walletd v0.23.1/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 5b5258b6d8e65f91c1a668b08c7cf307ef22f74a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:59:50 +0000 Subject: [PATCH 286/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.4.8-0.20240928202806-0e77790bd8bf to 0.5.0 - [Commits](https://github.com/SiaFoundation/core/commits/v0.5.0) Updates `go.sia.tech/coreutils` from 0.4.2-0.20241017012544-0b4946403c93 to 0.5.0 - [Commits](https://github.com/SiaFoundation/coreutils/commits/v0.5.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cbdcacd..71822d4 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf - go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93 + go.sia.tech/core v0.5.0 + go.sia.tech/coreutils v0.5.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 6e718e4..9992d92 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf h1:x/lM7Y8Rlo12rcpPXapLvSVNyrHZEKO6j4eLNccPMKw= -go.sia.tech/core v0.4.8-0.20240928202806-0e77790bd8bf/go.mod h1:j2Ke8ihV8or7d2VDrFZWcCkwSVHO0DNMQJAGs9Qop2M= -go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93 h1:38XTWxsDR4Q8ILQHFFSoB906jRrcb1UBaWg5d8MBt9M= -go.sia.tech/coreutils v0.4.2-0.20241017012544-0b4946403c93/go.mod h1:JIaR+zdGZsqPLBM5mVsnwWJ7hBsES+SAEDQg5EFBitM= +go.sia.tech/core v0.5.0 h1:feLC7DSCF+PhU157s/94106hFKyiGrGQ9HC3/dF/l7E= +go.sia.tech/core v0.5.0/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g= +go.sia.tech/coreutils v0.5.0 h1:/xKxdw83iZy0jjLzI2NGHyG4azyjK5DJscxpkr6nIGQ= +go.sia.tech/coreutils v0.5.0/go.mod h1:VYM4FcmlhVrpDGvglLHjRW+gitoaxPNLvp5mL2quilo= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= From 469c3764a82d2e74273c1cb525bfd7aafeff3e64 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 29 Oct 2024 09:27:01 +0100 Subject: [PATCH 287/630] wallet: fix build --- wallet/wallet_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 13cd74f..4fd7d5e 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3569,9 +3569,7 @@ func TestEventTypes(t *testing.T) { fc = fce.V2FileContract fc.RevisionNumber = types.MaxRevisionNumber finalizationSigHash := cm.TipState().ContractSigHash(fc) - fc.RenterSignature = pk.SignHash(finalizationSigHash) - fc.HostSignature = pk.SignHash(finalizationSigHash) - finalization := types.V2FileContractFinalization(fc) + finalization := types.V2FileContractFinalization(pk.SignHash(finalizationSigHash)) // create the resolution transaction finalizationTxn := types.V2Transaction{ From 1c7965329a9e1db5c917e9f10975c1db01cf489e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 29 Oct 2024 21:41:23 -0700 Subject: [PATCH 288/630] all: update core and coreutils wallet: fix lint --- api/api_test.go | 8 +- go.mod | 4 +- go.sum | 8 +- persist/sqlite/addresses.go | 16 +-- persist/sqlite/consensus.go | 201 +++++++++++++++++-------------- persist/sqlite/consensus_test.go | 8 +- persist/sqlite/events.go | 50 ++++---- persist/sqlite/init.sql | 4 +- persist/sqlite/migrations.go | 71 +++++++++++ persist/sqlite/utxo.go | 8 +- persist/sqlite/wallet.go | 20 +-- wallet/update.go | 40 ++---- wallet/wallet.go | 24 ++-- wallet/wallet_test.go | 28 ++--- 14 files changed, 278 insertions(+), 212 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index d8f3191..fbf5039 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -801,11 +801,11 @@ func TestV2(t *testing.T) { Value: sce.SiacoinOutput.Value, }}, Signatures: []types.TransactionSignature{{ - ParentID: sce.ID, + ParentID: types.Hash256(sce.ID), CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } - sig := key.SignHash(cm.TipState().WholeSigHash(txn, sce.ID, 0, 0, nil)) + sig := key.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(sce.ID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] if err := addBlock([]types.Transaction{txn}, nil); err != nil { return err @@ -1104,7 +1104,7 @@ func TestP2P(t *testing.T) { Value: sce.SiacoinOutput.Value, }}, Signatures: []types.TransactionSignature{{ - ParentID: sce.ID, + ParentID: types.Hash256(sce.ID), CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } @@ -1112,7 +1112,7 @@ func TestP2P(t *testing.T) { if err != nil { return err } - sig := key.SignHash(cs.WholeSigHash(txn, sce.ID, 0, 0, nil)) + sig := key.SignHash(cs.WholeSigHash(txn, types.Hash256(sce.ID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { return err diff --git a/go.mod b/go.mod index 71822d4..919ea9f 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.5.0 - go.sia.tech/coreutils v0.5.0 + go.sia.tech/core v0.6.1 + go.sia.tech/coreutils v0.6.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 9992d92..01d1b5b 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.5.0 h1:feLC7DSCF+PhU157s/94106hFKyiGrGQ9HC3/dF/l7E= -go.sia.tech/core v0.5.0/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g= -go.sia.tech/coreutils v0.5.0 h1:/xKxdw83iZy0jjLzI2NGHyG4azyjK5DJscxpkr6nIGQ= -go.sia.tech/coreutils v0.5.0/go.mod h1:VYM4FcmlhVrpDGvglLHjRW+gitoaxPNLvp5mL2quilo= +go.sia.tech/core v0.6.1 h1:eaExM2E2eNr43su2XDkY5J24E3F54YGS7hcC3WtVjVk= +go.sia.tech/core v0.6.1/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g= +go.sia.tech/coreutils v0.6.0 h1:r0IZt+aVdGG2uIHl7OtaWRYdVx4NQ7ezRoSGa0Ej8GY= +go.sia.tech/coreutils v0.6.0/go.mod h1:XlsnogeYU/Tdjzp/HUNAj5T7tZCdmeBHIBjymbPC+uQ= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 64e3d74..526e08e 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -103,14 +103,14 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIn if s.indexMode == wallet.IndexModeFull { indices := make([]uint64, len(siacoins)) for i, se := range siacoins { - indices[i] = se.LeafIndex + indices[i] = se.StateElement.LeafIndex } proofs, err := fillElementProofs(tx, indices) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { - siacoins[i].MerkleProof = proof + siacoins[i].StateElement.MerkleProof = proof } } return nil @@ -148,14 +148,14 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) if s.indexMode == wallet.IndexModeFull { indices := make([]uint64, len(siafunds)) for i, se := range siafunds { - indices[i] = se.LeafIndex + indices[i] = se.StateElement.LeafIndex } proofs, err := fillElementProofs(tx, indices) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { - siafunds[i].MerkleProof = proof + siafunds[i].StateElement.MerkleProof = proof } } return nil @@ -244,13 +244,13 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 for i, output := range txn.SiacoinOutputs { sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(i), StateElement: types.StateElement{ - ID: types.Hash256(txn.SiacoinOutputID(i)), LeafIndex: types.UnassignedLeafIndex, }, SiacoinOutput: output, } - siacoinElementCache[types.SiacoinOutputID(sce.StateElement.ID)] = sce + siacoinElementCache[sce.ID] = sce relevant = true } @@ -268,13 +268,13 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 for i, output := range txn.SiafundOutputs { sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(i), StateElement: types.StateElement{ - ID: types.Hash256(txn.SiafundOutputID(i)), LeafIndex: types.UnassignedLeafIndex, }, SiafundOutput: output, } - siafundElementCache[types.SiafundOutputID(sfe.StateElement.ID)] = sfe + siafundElementCache[sfe.ID] = sfe relevant = true } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index fa6ea33..2129eab 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -3,7 +3,6 @@ package sqlite import ( "bytes" "database/sql" - "encoding/json" "errors" "fmt" @@ -26,96 +25,36 @@ type addressRef struct { Balance wallet.Balance } -func (ut *updateTx) SiacoinStateElements() ([]types.StateElement, error) { - if ut.indexMode == wallet.IndexModeFull { - panic("SiacoinStateElements called in full index mode") - } - - const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` - rows, err := ut.tx.Query(query) - if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) - } - defer rows.Close() - - var elements []types.StateElement - for rows.Next() { - se, err := scanStateElement(rows) - if err != nil { - return nil, fmt.Errorf("failed to scan state element: %w", err) - } - elements = append(elements, se) - } - return elements, rows.Err() +type stateElement struct { + ID types.Hash256 + types.StateElement } -func (ut *updateTx) UpdateSiacoinStateElements(elements []types.StateElement) error { +func (ut *updateTx) UpdateStateElementProofs(update wallet.ProofUpdater) error { if ut.indexMode == wallet.IndexModeFull { - panic("UpdateSiacoinStateElements called in full index mode") + panic("UpdateStateElementProofs called in full index mode") } - log := ut.tx.log.Named("UpdateSiacoinStateElements") - log.Debug("updating siacoin state elements", zap.Int("count", len(elements))) - - const query = `UPDATE siacoin_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` - stmt, err := ut.tx.Prepare(query) + se, err := getSiacoinStateElements(ut.tx) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return fmt.Errorf("failed to get siacoin state elements: %w", err) } - defer stmt.Close() - - for _, se := range elements { - var dummy types.Hash256 - err := stmt.QueryRow(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) - if err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } - log.Debug("updated element proof", zap.Stringer("id", se.ID), zap.Uint64("leafIndex", se.LeafIndex)) + for i := range se { + update.UpdateElementProof(&se[i].StateElement) } - return nil -} - -func (ut *updateTx) SiafundStateElements() ([]types.StateElement, error) { - if ut.indexMode == wallet.IndexModeFull { - panic("SiafundStateElements called in full index mode") + if err := updateSiacoinStateElements(ut.tx, se); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) } - const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` - rows, err := ut.tx.Query(query) + sfe, err := getSiafundStateElements(ut.tx) if err != nil { - return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + return fmt.Errorf("failed to get siafund state elements: %w", err) } - defer rows.Close() - - var elements []types.StateElement - for rows.Next() { - se, err := scanStateElement(rows) - if err != nil { - return nil, fmt.Errorf("failed to scan state element: %w", err) - } - elements = append(elements, se) - } - return elements, rows.Err() -} - -func (ut *updateTx) UpdateSiafundStateElements(elements []types.StateElement) error { - if ut.indexMode == wallet.IndexModeFull { - panic("UpdateSiafundStateElements called in full index mode") - } - - const query = `UPDATE siafund_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3 RETURNING id` - stmt, err := ut.tx.Prepare(query) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + for i := range sfe { + update.UpdateElementProof(&sfe[i].StateElement) } - defer stmt.Close() - - for _, se := range elements { - var dummy types.Hash256 - err := stmt.QueryRow(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)).Scan(decode(&dummy)) - if err != nil { - return fmt.Errorf("failed to execute statement: %w", err) - } + if err := updateSiafundStateElements(ut.tx, sfe); err != nil { + return fmt.Errorf("failed to update siafund state elements: %w", err) } return nil } @@ -319,9 +258,90 @@ func (s *Store) SetIndexMode(mode wallet.IndexMode) error { }) } -func scanStateElement(s scanner) (se types.StateElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)) - return +func getSiacoinStateElements(tx *txn) ([]stateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` + rows, err := tx.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var elements []stateElement + for rows.Next() { + var se stateElement + if err := rows.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + elements = append(elements, se) + } + return elements, rows.Err() +} + +func getSiafundStateElements(tx *txn) ([]stateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` + rows, err := tx.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() + + var elements []stateElement + for rows.Next() { + var se stateElement + if err := rows.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + elements = append(elements, se) + } + return elements, rows.Err() +} + +func updateSiafundStateElements(tx *txn, elements []stateElement) error { + if len(elements) == 0 { + return nil + } + const query = `UPDATE siafund_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3` + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, se := range elements { + res, err := stmt.Exec(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func updateSiacoinStateElements(tx *txn, elements []stateElement) error { + if len(elements) == 0 { + return nil + } + const query = `UPDATE siacoin_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3` + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, se := range elements { + res, err := stmt.Exec(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil } func scanAddress(s scanner) (ab addressRef, err error) { @@ -530,10 +550,10 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, // in full index mode, Merkle proofs are stored in the state tree table // rather than per element. if indexMode == wallet.IndexModeFull { - se.MerkleProof = nil + se.StateElement.MerkleProof = nil } - _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.MerkleProof), se.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -804,10 +824,10 @@ func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, // in full index mode, Merkle proofs are stored in the state tree table // rather than per element. if indexMode == wallet.IndexModeFull { - se.MerkleProof = nil + se.StateElement.MerkleProof = nil } - _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encode(se.MerkleProof), se.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } else if exists { @@ -1052,15 +1072,18 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { defer relevantAddrStmt.Close() var buf bytes.Buffer - enc := json.NewEncoder(&buf) + enc := types.NewEncoder(&buf) for _, event := range events { buf.Reset() - if err := enc.Encode(event.Data); err != nil { - return fmt.Errorf("failed to encode event: %w", err) + ev, ok := event.Data.(types.EncoderTo) + if !ok { + panic("event data does not implement types.EncoderTo") // developer error } + ev.EncodeTo(enc) + enc.Flush() var eventID int64 - err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, buf.String(), indexID).Scan(&eventID) + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, buf.Bytes(), indexID).Scan(&eventID) if errors.Is(err, sql.ErrNoRows) { continue // skip if the event already exists } else if err != nil { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index cc77c4f..107bace 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -152,10 +152,10 @@ func TestPruneSiacoins(t *testing.T) { }, } - sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures = append(txn.Signatures, types.TransactionSignature{ - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), CoveredFields: types.CoveredFields{WholeTransaction: true}, PublicKeyIndex: 0, Timelock: 0, @@ -277,10 +277,10 @@ func TestPruneSiafunds(t *testing.T) { }, } - sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures = append(txn.Signatures, types.TransactionSignature{ - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), CoveredFields: types.CoveredFields{WholeTransaction: true}, PublicKeyIndex: 0, Timelock: 0, diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index 69c62da..1ecd94f 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -2,7 +2,6 @@ package sqlite import ( "database/sql" - "encoding/json" "errors" "fmt" @@ -62,47 +61,44 @@ WHERE ev.event_id = $1` return } +func decodeEventData[T wallet.EventPayout | + wallet.EventV1Transaction | + wallet.EventV2Transaction | + wallet.EventV1ContractResolution | + wallet.EventV2ContractResolution, TP interface { + *T + types.DecoderFrom +}](dec *types.Decoder) T { + v := new(T) + TP(v).DecodeFrom(dec) + return *v +} + func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { var eventBuf []byte - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Confirmations, &ev.Type, &eventBuf) if err != nil { return } + dec := types.NewBufDecoder(eventBuf) switch ev.Type { case wallet.EventTypeV1Transaction: - var tx wallet.EventV1Transaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - ev.Data = tx + ev.Data = decodeEventData[wallet.EventV1Transaction](dec) case wallet.EventTypeV2Transaction: - var tx wallet.EventV2Transaction - if err = json.Unmarshal(eventBuf, &tx); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal transaction event: %w", err) - } - ev.Data = tx + ev.Data = decodeEventData[wallet.EventV2Transaction](dec) case wallet.EventTypeV1ContractResolution: - var r wallet.EventV1ContractResolution - if err = json.Unmarshal(eventBuf, &r); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal missed file contract event: %w", err) - } - ev.Data = r + ev.Data = decodeEventData[wallet.EventV1ContractResolution](dec) case wallet.EventTypeV2ContractResolution: - var r wallet.EventV2ContractResolution - if err = json.Unmarshal(eventBuf, &r); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal file contract event: %w", err) - } - ev.Data = r + ev.Data = decodeEventData[wallet.EventV2ContractResolution](dec) case wallet.EventTypeSiafundClaim, wallet.EventTypeMinerPayout, wallet.EventTypeFoundationSubsidy: - var p wallet.EventPayout - if err = json.Unmarshal(eventBuf, &p); err != nil { - return wallet.Event{}, 0, fmt.Errorf("failed to unmarshal event %q (%q): %w", ev.ID, ev.Type, err) - } - ev.Data = p + ev.Data = decodeEventData[wallet.EventPayout](dec) default: return wallet.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) } + if err := dec.Err(); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to decode event data: %w", err) + } + return } diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 37eba72..cf6a94b 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -17,7 +17,7 @@ CREATE TABLE siacoin_elements ( id BLOB PRIMARY KEY, siacoin_value BLOB NOT NULL, merkle_proof BLOB NOT NULL, - leaf_index INTEGER NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ address_id INTEGER NOT NULL REFERENCES sia_addresses (id), matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ @@ -34,7 +34,7 @@ CREATE TABLE siafund_elements ( id BLOB PRIMARY KEY, claim_start BLOB NOT NULL, merkle_proof BLOB NOT NULL, - leaf_index INTEGER NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, siafund_value INTEGER NOT NULL, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 8429fe4..bce12b8 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,6 +7,77 @@ import ( "go.uber.org/zap" ) +// migrateVersion5 resets the database to trigger a full resync to switch +// events from JSON to Sia encoding +func migrateVersion5(tx *txn, _ *zap.Logger) error { + if _, err := tx.Exec(`DELETE FROM siacoin_elements;`); err != nil { + return fmt.Errorf("failed to delete siacoin_elements: %w", err) + } else if _, err := tx.Exec(`DELETE FROM siafund_elements;`); err != nil { + return fmt.Errorf("failed to delete siafund_elements: %w", err) + } else if _, err := tx.Exec(`DELETE FROM state_tree;`); err != nil { + return fmt.Errorf("failed to delete state_tree: %w", err) + } else if _, err := tx.Exec(`DELETE FROM event_addresses;`); err != nil { + return fmt.Errorf("failed to delete event_addresses: %w", err) + } else if _, err := tx.Exec(`DELETE FROM events;`); err != nil { + return fmt.Errorf("failed to delete events: %w", err) + } else if _, err := tx.Exec(`DELETE FROM chain_indices;`); err != nil { + return fmt.Errorf("failed to delete chain_indices: %w", err) + } else if _, err := tx.Exec(`DROP TABLE siacoin_elements;`); err != nil { + return fmt.Errorf("failed to drop siacoin_elements: %w", err) + } else if _, err := tx.Exec(`DROP TABLE siafund_elements;`); err != nil { + return fmt.Errorf("failed to drop siafund_elements: %w", err) + } + + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1, element_num_leaves=0`, encode(types.ChainIndex{})) + if err != nil { + return fmt.Errorf("failed to reset global_settings: %w", err) + } + + _, err = tx.Exec(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$1, siafund_balance=0;`, encode(types.ZeroCurrency)) + if err != nil { + return fmt.Errorf("failed to reset sia_addresses: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id);`) + if err != nil { + return fmt.Errorf("failed to create siacoin_elements: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id);`) + if err != nil { + return fmt.Errorf("failed to create siafund_elements: %w", err) + } + return nil +} + // migrateVersion4 splits the height and ID of the last indexed tip into two // separate columns for easier querying. func migrateVersion4(tx *txn, _ *zap.Logger) error { diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go index 5bcf9bf..881d700 100644 --- a/persist/sqlite/utxo.go +++ b/persist/sqlite/utxo.go @@ -24,13 +24,13 @@ WHERE se.id=$1 AND spent_index_id IS NULL` // retrieve the merkle proofs for the siacoin element if s.indexMode == wallet.IndexModeFull { - proof, err := fillElementProofs(tx, []uint64{ele.LeafIndex}) + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } else if len(proof) != 1 { panic("expected exactly one proof") // should never happen } - ele.MerkleProof = proof[0] + ele.StateElement.MerkleProof = proof[0] } return nil }) @@ -55,13 +55,13 @@ WHERE se.id=$1 AND spent_index_id IS NULL` // retrieve the merkle proofs for the siafund element if s.indexMode == wallet.IndexModeFull { - proof, err := fillElementProofs(tx, []uint64{ele.LeafIndex}) + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } else if len(proof) != 1 { panic("expected exactly one proof") // should never happen } - ele.MerkleProof = proof[0] + ele.StateElement.MerkleProof = proof[0] } return nil }) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 05f8f76..45f572f 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -258,14 +258,14 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, index types.ChainIndex, offse if s.indexMode == wallet.IndexModeFull { indices := make([]uint64, len(siacoins)) for i, se := range siacoins { - indices[i] = se.LeafIndex + indices[i] = se.StateElement.LeafIndex } proofs, err := fillElementProofs(tx, indices) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { - siacoins[i].MerkleProof = proof + siacoins[i].StateElement.MerkleProof = proof } } return nil @@ -307,14 +307,14 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds if s.indexMode == wallet.IndexModeFull { indices := make([]uint64, len(siafunds)) for i, se := range siafunds { - indices[i] = se.LeafIndex + indices[i] = se.StateElement.LeafIndex } proofs, err := fillElementProofs(tx, indices) if err != nil { return fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { - siafunds[i].MerkleProof = proof + siafunds[i].StateElement.MerkleProof = proof } } return nil @@ -489,13 +489,13 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti } sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(i), StateElement: types.StateElement{ - ID: types.Hash256(txn.SiacoinOutputID(i)), LeafIndex: types.UnassignedLeafIndex, }, SiacoinOutput: output, } - siacoinElementCache[types.SiacoinOutputID(sce.StateElement.ID)] = sce + siacoinElementCache[sce.ID] = sce } for _, input := range txn.SiafundInputs { @@ -528,13 +528,13 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti } sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(i), StateElement: types.StateElement{ - ID: types.Hash256(txn.SiafundOutputID(i)), LeafIndex: types.UnassignedLeafIndex, }, SiafundOutput: output, } - siafundElementCache[types.SiafundOutputID(sfe.StateElement.ID)] = sfe + siafundElementCache[sfe.ID] = sfe } if len(relevant) == 0 { @@ -593,12 +593,12 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti } func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { - err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.MerkleProof), &se.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.StateElement.MerkleProof), &se.StateElement.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) return } func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { - err = s.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + err = s.Scan(decode(&se.ID), &se.StateElement.LeafIndex, decode(&se.StateElement.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) return } diff --git a/wallet/update.go b/wallet/update.go index 4f458dd..9392c7a 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -12,10 +12,15 @@ type ( // A stateTreeUpdater is an interface for applying and reverting // Merkle tree updates. stateTreeUpdater interface { - UpdateElementProof(e *types.StateElement) + UpdateElementProof(*types.StateElement) ForEachTreeNode(fn func(row uint64, col uint64, h types.Hash256)) } + // A ProofUpdater is an interface for updating Merkle proofs. + ProofUpdater interface { + UpdateElementProof(*types.StateElement) + } + // AddressBalance pairs an address with its balance. AddressBalance struct { Address types.Address `json:"address"` @@ -53,12 +58,7 @@ type ( // An UpdateTx atomically updates the state of a store. UpdateTx interface { - SiacoinStateElements() ([]types.StateElement, error) - UpdateSiacoinStateElements([]types.StateElement) error - - SiafundStateElements() ([]types.StateElement, error) - UpdateSiafundStateElements([]types.StateElement) error - + UpdateStateElementProofs(ProofUpdater) error UpdateStateTree([]TreeNodeUpdate) error AddressRelevant(types.Address) (bool, error) @@ -82,31 +82,7 @@ func updateStateElements(tx UpdateTx, update stateTreeUpdater, indexMode IndexMo }) return tx.UpdateStateTree(updates) } else { - // fetch all siacoin and siafund state elements - siacoinStateElements, err := tx.SiacoinStateElements() - if err != nil { - return fmt.Errorf("failed to get siacoin state elements: %w", err) - } - - // update siacoin element proofs - for i := range siacoinStateElements { - update.UpdateElementProof(&siacoinStateElements[i]) - } - - if err := tx.UpdateSiacoinStateElements(siacoinStateElements); err != nil { - return fmt.Errorf("failed to update siacoin state elements: %w", err) - } - - siafundStateElements, err := tx.SiafundStateElements() - if err != nil { - return fmt.Errorf("failed to get siafund state elements: %w", err) - } - - // update siafund element proofs - for i := range siafundStateElements { - update.UpdateElementProof(&siafundStateElements[i]) - } - return tx.UpdateSiafundStateElements(siafundStateElements) + return tx.UpdateStateElementProofs(update) } } diff --git a/wallet/wallet.go b/wallet/wallet.go index c968e56..93bc107 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -174,11 +174,11 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sces := make(map[types.SiacoinOutputID]types.SiacoinElement) sfes := make(map[types.SiafundOutputID]types.SiafundElement) cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { - sce.MerkleProof = nil + sce.StateElement.MerkleProof = nil sces[types.SiacoinOutputID(sce.ID)] = sce }) cu.ForEachSiafundElement(func(sfe types.SiafundElement, _, _ bool) { - sfe.MerkleProof = nil + sfe.StateElement.MerkleProof = nil sfes[types.SiafundOutputID(sfe.ID)] = sfe }) @@ -221,7 +221,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sce, ok := sces[sfi.ParentID.ClaimOutputID()] if ok && relevant(sce.SiacoinOutput.Address) { - addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ + addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } @@ -268,7 +268,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f sce, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] if ok && relevant(sfi.ClaimAddress) { - addEvent(sce.ID, sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ + addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) } @@ -299,7 +299,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return } - fce.MerkleProof = nil + fce.StateElement.MerkleProof = nil if valid { for i := range fce.FileContract.ValidProofOutputs { @@ -309,7 +309,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } element := sces[types.FileContractID(fce.ID).ValidOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ Parent: fce, SiacoinElement: element, Missed: false, @@ -323,7 +323,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } element := sces[types.FileContractID(fce.ID).MissedOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ Parent: fce, SiacoinElement: element, Missed: true, @@ -337,7 +337,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return } - fce.MerkleProof = nil + fce.StateElement.MerkleProof = nil var missed bool if _, ok := res.(*types.V2FileContractExpiration); ok { @@ -346,7 +346,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.HostOutput.Address) { element := sces[types.FileContractID(fce.ID).V2HostOutputID()] - addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, @@ -358,7 +358,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(fce.V2FileContract.RenterOutput.Address) { element := sces[types.FileContractID(fce.ID).V2RenterOutputID()] - addEvent(element.ID, element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ Resolution: types.V2FileContractResolution{ Parent: fce, Resolution: res, @@ -373,7 +373,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { element := sces[cs.Index.ID.MinerOutputID(i)] - addEvent(element.ID, element.MaturityHeight, EventTypeMinerPayout, wallet.EventPayout{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeMinerPayout, wallet.EventPayout{ SiacoinElement: element, }, []types.Address{b.MinerPayouts[i].Address}) } @@ -383,7 +383,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f if relevant(cs.FoundationPrimaryAddress) { element, ok := sces[cs.Index.ID.FoundationOutputID()] if ok { - addEvent(element.ID, element.MaturityHeight, EventTypeFoundationSubsidy, wallet.EventPayout{ + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeFoundationSubsidy, wallet.EventPayout{ SiacoinElement: element, }, []types.Address{element.SiacoinOutput.Address}) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 4fd7d5e..dab8c1f 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -409,13 +409,13 @@ func TestEphemeralBalance(t *testing.T) { }, Signatures: []types.TransactionSignature{ { - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } - parentSigHash := cm.TipState().WholeSigHash(parentTxn, utxos[0].ID, 0, 0, nil) + parentSigHash := cm.TipState().WholeSigHash(parentTxn, types.Hash256(utxos[0].ID), 0, 0, nil) parentSig := pk.SignHash(parentSigHash) parentTxn.Signatures[0].Signature = parentSig[:] @@ -1071,13 +1071,13 @@ func TestOrphans(t *testing.T) { }, Signatures: []types.TransactionSignature{ { - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } - sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures[0].Signature = sig[:] @@ -1727,13 +1727,13 @@ func TestWalletUnconfirmedEvents(t *testing.T) { }, Signatures: []types.TransactionSignature{ { - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } - sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures[0].Signature = sig[:] @@ -1824,7 +1824,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } txnData = events[1].Data.(wallet.EventV1Transaction) - if txnData.SpentSiacoinElements[0].ID != types.Hash256(ephemeralOutputID) { + if txnData.SpentSiacoinElements[0].ID != ephemeralOutputID { t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) @@ -1935,13 +1935,13 @@ func TestAddressUnconfirmedEvents(t *testing.T) { }, Signatures: []types.TransactionSignature{ { - ParentID: utxos[0].ID, + ParentID: types.Hash256(utxos[0].ID), PublicKeyIndex: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } - sigHash := cm.TipState().WholeSigHash(txn, utxos[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures[0].Signature = sig[:] @@ -2041,7 +2041,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } txnData = events[1].Data.(wallet.EventV1Transaction) - if txnData.SpentSiacoinElements[0].ID != types.Hash256(ephemeralOutputID) { + if txnData.SpentSiacoinElements[0].ID != ephemeralOutputID { t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) @@ -3043,7 +3043,7 @@ func TestEventTypes(t *testing.T) { }, Signatures: []types.TransactionSignature{ { - ParentID: sce[0].ID, + ParentID: types.Hash256(sce[0].ID), PublicKeyIndex: 0, Timelock: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, @@ -3052,7 +3052,7 @@ func TestEventTypes(t *testing.T) { } // sign the transaction - sigHash := cm.TipState().WholeSigHash(txn, sce[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(sce[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures[0].Signature = sig[:] @@ -3097,14 +3097,14 @@ func TestEventTypes(t *testing.T) { FileContracts: []types.FileContract{fc}, Signatures: []types.TransactionSignature{ { - ParentID: sce[0].ID, + ParentID: types.Hash256(sce[0].ID), PublicKeyIndex: 0, Timelock: 0, CoveredFields: types.CoveredFields{WholeTransaction: true}, }, }, } - sigHash := cm.TipState().WholeSigHash(txn, sce[0].ID, 0, 0, nil) + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(sce[0].ID), 0, 0, nil) sig := pk.SignHash(sigHash) txn.Signatures[0].Signature = sig[:] From f8f86eeca31baf56a62e1ae7c16c36c4b85981bd Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 31 Oct 2024 07:19:04 -0700 Subject: [PATCH 289/630] sqlite: run migration --- persist/sqlite/migrations.go | 1 + 1 file changed, 1 insertion(+) diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index bce12b8..c00bbd9 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -162,4 +162,5 @@ var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion2, migrateVersion3, migrateVersion4, + migrateVersion5, } From 465e9da22d826b27e9f32c1987d619f4d593f03b Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 31 Oct 2024 15:59:38 -0700 Subject: [PATCH 290/630] api: Addr -> Address --- api/api.go | 2 +- api/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/api.go b/api/api.go index afe97e2..0014bd4 100644 --- a/api/api.go +++ b/api/api.go @@ -22,7 +22,7 @@ type StateResponse struct { // A GatewayPeer is a currently-connected peer. type GatewayPeer struct { - Addr string `json:"addr"` + Address string `json:"address"` Inbound bool `json:"inbound"` Version string `json:"version"` diff --git a/api/server.go b/api/server.go index 224fcd3..7549ce5 100644 --- a/api/server.go +++ b/api/server.go @@ -218,7 +218,7 @@ func (s *server) syncerPeersHandler(jc jape.Context) { for _, p := range s.s.Peers() { // create peer response with known fields peer := GatewayPeer{ - Addr: p.Addr(), + Address: p.Addr(), Inbound: p.Inbound, Version: p.Version(), } From 9d337922a666891fb0ca3736a161592d24e4da09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:31:12 +0000 Subject: [PATCH 291/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.23.1 to 0.23.2 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.23.1...walletd@0.23.2) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 919ea9f..9b71f3b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.6.1 go.sia.tech/coreutils v0.6.0 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.23.1 + go.sia.tech/web/walletd v0.23.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.25.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 01d1b5b..b40c838 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.23.1 h1:z0S3tHLpxnqedkXbsZIa+VJTJmZpvfabpRurqmoKL4w= -go.sia.tech/web/walletd v0.23.1/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.23.2 h1:GiLQ00doYMWw7L3Jiky0yeygL8lJcpnDRS/3+a+aHEI= +go.sia.tech/web/walletd v0.23.2/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From aaf765e1ebf9e4b5991e7feeb069d454b3ca0674 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 5 Nov 2024 10:01:48 +0100 Subject: [PATCH 292/630] publish.yml: add project-desc and version-tag --- .github/workflows/publish.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2f1881b..741e4ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,8 +7,8 @@ on: branches: - master tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-**' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-**" concurrency: group: ${{ github.workflow }} @@ -24,3 +24,5 @@ jobs: macos-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w' cgo-enabled: 1 project: walletd + project-desc: "walletd: The new Sia wallet" + version-tag: ${{ github.ref_name }} From 8f9e21b2bbf49f3c054bb18471faf595ccdfea46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:32:27 +0000 Subject: [PATCH 293/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.25.0 to 0.26.0 - [Commits](https://github.com/golang/term/compare/v0.25.0...v0.26.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9b71f3b..81a0599 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.23.2 go.uber.org/zap v1.27.0 - golang.org/x/term v0.25.0 + golang.org/x/term v0.26.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -25,6 +25,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.27.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index b40c838..58d55b8 100644 --- a/go.sum +++ b/go.sum @@ -34,10 +34,10 @@ golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From bb74f5bf1e63fda6a056ba3f08aeac69c20a02cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:31:31 +0000 Subject: [PATCH 294/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/core` from 0.6.1 to 0.6.2 - [Commits](https://github.com/SiaFoundation/core/compare/v0.6.1...v0.6.2) Updates `go.sia.tech/web/walletd` from 0.23.2 to 0.24.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.23.2...hostd@0.24.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 81a0599..96d9c3f 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.6.1 + go.sia.tech/core v0.6.2 go.sia.tech/coreutils v0.6.0 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.23.2 + go.sia.tech/web/walletd v0.24.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.26.0 gopkg.in/yaml.v3 v3.0.1 @@ -24,7 +24,7 @@ require ( go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.29.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 58d55b8..6882579 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.6.1 h1:eaExM2E2eNr43su2XDkY5J24E3F54YGS7hcC3WtVjVk= -go.sia.tech/core v0.6.1/go.mod h1:P3C1BWa/7J4XgdzWuaYHBvLo2RzZ0UBaJM4TG1GWB2g= +go.sia.tech/core v0.6.2 h1:8NEjxyD93A+EhZopsBy/LvuHH+zUSjRNKnf9rXgtIwU= +go.sia.tech/core v0.6.2/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= go.sia.tech/coreutils v0.6.0 h1:r0IZt+aVdGG2uIHl7OtaWRYdVx4NQ7ezRoSGa0Ej8GY= go.sia.tech/coreutils v0.6.0/go.mod h1:XlsnogeYU/Tdjzp/HUNAj5T7tZCdmeBHIBjymbPC+uQ= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= @@ -20,16 +20,16 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.23.2 h1:GiLQ00doYMWw7L3Jiky0yeygL8lJcpnDRS/3+a+aHEI= -go.sia.tech/web/walletd v0.23.2/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.24.0 h1:8l27tssquF/ONiXCd0m8707HIiohszGgjqJRHcGCTcE= +go.sia.tech/web/walletd v0.24.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= From b9843b4e34c0d70a90036187a69ce6d66e93f805 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 20 Nov 2024 10:29:13 -0800 Subject: [PATCH 295/630] chore(deps): update coreutils and core --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 96d9c3f..3d33afb 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.6.2 - go.sia.tech/coreutils v0.6.0 + go.sia.tech/coreutils v0.7.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.24.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 6882579..6fbf308 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.6.2 h1:8NEjxyD93A+EhZopsBy/LvuHH+zUSjRNKnf9rXgtIwU= go.sia.tech/core v0.6.2/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= -go.sia.tech/coreutils v0.6.0 h1:r0IZt+aVdGG2uIHl7OtaWRYdVx4NQ7ezRoSGa0Ej8GY= -go.sia.tech/coreutils v0.6.0/go.mod h1:XlsnogeYU/Tdjzp/HUNAj5T7tZCdmeBHIBjymbPC+uQ= +go.sia.tech/coreutils v0.7.0 h1:YpgOUD4vrpDz0KC7FJz+UCOaKaqV5EkX3gMUUmJoz5s= +go.sia.tech/coreutils v0.7.0/go.mod h1:eMoqzqO4opKQ6n9tTTxHQmccfNlj+8RFOYuDSL/Qd4g= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= From 6b9e903bfae17b5c7fb1f8938ca5676443dc571e Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 20 Nov 2024 10:32:34 -0800 Subject: [PATCH 296/630] chore(changelog): automate changelog --- .github/workflows/prepare-release.yml | 26 +++++++++++++ CHANGELOG.md | 12 ++++++ knope.toml | 55 +++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 .github/workflows/prepare-release.yml create mode 100644 CHANGELOG.md create mode 100644 knope.toml diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000..7b6ca01 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,26 @@ +on: + push: + branches: [master] + +permissions: + contents: write + pull-requests: write + +name: Create Release PR +jobs: + prepare-release: + if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope prepare-release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6dddb0e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +## 0.8.0 + +This is the first stable release for the walletd app -- the new reference wallet for users and exchanges + +### Breaking changes + +- SiaFund support +- Ledger hardware wallet support +- Multi-wallet support +- Full index mode for exchanges and wallet integrators +- Redesigned events list +- Redesigned transaction flow diff --git a/knope.toml b/knope.toml new file mode 100644 index 0000000..2cbe180 --- /dev/null +++ b/knope.toml @@ -0,0 +1,55 @@ +[package] +changelog = "CHANGELOG.md" +versioned_files = ["go.mod"] +ignore_go_major_versioning = true + +[[workflows]] +name = "document-change" + +[[workflows.steps]] +type = "CreateChangeFile" + +[[workflows]] +name = "prepare-release" + +[[workflows.steps]] +type = "Command" +command = "git switch -c release" + +[[workflows.steps]] +type = "PrepareRelease" + +[[workflows.steps]] +type = "Command" +command = "git commit -m \"chore: prepare release $version\"" +variables = { "$version" = "Version" } + +[[workflows.steps]] +type = "Command" +command = "git push --force --set-upstream origin release" + +[workflows.steps.variables] +"$version" = "Version" + +[[workflows.steps]] +type = "CreatePullRequest" +base = "master" + +[workflows.steps.title] +template = "chore: prepare release $version" +variables = { "$version" = "Version" } + +[workflows.steps.body] +template = "This PR was created automatically. Merging it will finalize the changelog for $version\n\n$changelog" +variables = { "$changelog" = "ChangelogEntry", "$version" = "Version" } + +# Do not enable releases, just changelogs for now. +# [[workflows]] +# name = "release" +# +# [[workflows.steps]] +# type = "Release" + +[github] +owner = "SiaFoundation" +repo = "walletd" From 40412d59b6a81d2e547f59de393e4ff5398e1c79 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Dec 2024 18:03:02 -0800 Subject: [PATCH 297/630] chore: update core --- api/mine.go | 15 +---- api/server.go | 16 +---- go.mod | 4 +- go.sum | 8 +-- wallet/wallet.go | 2 +- wallet/wallet_test.go | 133 +++--------------------------------------- 6 files changed, 20 insertions(+), 158 deletions(-) diff --git a/api/mine.go b/api/mine.go index 9666d0c..d2b6bd1 100644 --- a/api/mine.go +++ b/api/mine.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/binary" "errors" "go.sia.tech/core/types" @@ -50,19 +49,8 @@ func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types. } b.Nonce = 0 - buf := make([]byte, 32+8+8+32) - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) - binary.LittleEndian.PutUint64(buf[40:], uint64(b.Timestamp.Unix())) - if b.V2 != nil { - copy(buf[:32], "sia/id/block|") - copy(buf[48:], b.V2.Commitment[:]) - } else { - root := b.MerkleRoot() - copy(buf[:32], b.ParentID[:]) - copy(buf[48:], root[:]) - } factor := cs.NonceFactor() - for types.BlockID(types.HashBytes(buf)).CmpWork(cs.ChildTarget) < 0 { + for b.ID().CmpWork(cs.ChildTarget) < 0 { select { case <-ctx.Done(): return types.Block{}, ctx.Err() @@ -75,7 +63,6 @@ func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types. } b.Nonce += factor - binary.LittleEndian.PutUint64(buf[32:], b.Nonce) } return b, nil } diff --git a/api/server.go b/api/server.go index 7549ce5..4da2206 100644 --- a/api/server.go +++ b/api/server.go @@ -79,7 +79,7 @@ type ( Peers() []*syncer.Peer PeerInfo(addr string) (syncer.PeerInfo, error) Connect(ctx context.Context, addr string) (*syncer.Peer, error) - BroadcastHeader(bh gateway.BlockHeader) + BroadcastHeader(types.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) BroadcastV2BlockOutline(bo gateway.V2BlockOutline) @@ -258,12 +258,7 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { return } if b.V2 == nil { - s.s.BroadcastHeader(gateway.BlockHeader{ - ParentID: b.ParentID, - Nonce: b.Nonce, - Timestamp: b.Timestamp, - MerkleRoot: b.MerkleRoot(), - }) + s.s.BroadcastHeader(b.Header()) } else { s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) } @@ -889,12 +884,7 @@ func (s *server) debugMineHandler(jc jape.Context) { } if b.V2 == nil { - s.s.BroadcastHeader(gateway.BlockHeader{ - ParentID: b.ParentID, - Nonce: b.Nonce, - Timestamp: b.Timestamp, - MerkleRoot: b.MerkleRoot(), - }) + s.s.BroadcastHeader(b.Header()) } else { s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) } diff --git a/go.mod b/go.mod index 3d33afb..a64e890 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.6.2 - go.sia.tech/coreutils v0.7.0 + go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6 + go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.24.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 6fbf308..a9e0008 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.6.2 h1:8NEjxyD93A+EhZopsBy/LvuHH+zUSjRNKnf9rXgtIwU= -go.sia.tech/core v0.6.2/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= -go.sia.tech/coreutils v0.7.0 h1:YpgOUD4vrpDz0KC7FJz+UCOaKaqV5EkX3gMUUmJoz5s= -go.sia.tech/coreutils v0.7.0/go.mod h1:eMoqzqO4opKQ6n9tTTxHQmccfNlj+8RFOYuDSL/Qd4g= +go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6 h1:52hztNcOJ+eql7dHMBl+g9VL4Lxr87cUrR9cXrYOkMs= +go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= +go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 h1:Qskaf8d6oDKG5emNvGHZsd9iZRqz2GeouVNKY5paXlE= +go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31/go.mod h1:d6jrawloc02MCXi/EVc8FIN5h3C6XDiMs4fuFMcU0PU= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= diff --git a/wallet/wallet.go b/wallet/wallet.go index 93bc107..762d744 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -380,7 +380,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } // handle foundation subsidy - if relevant(cs.FoundationPrimaryAddress) { + if relevant(cs.FoundationManagementAddress) { element, ok := sces[cs.Index.ID.FoundationOutputID()] if ok { addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeFoundationSubsidy, wallet.EventPayout{ diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index dab8c1f..bdd6406 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "math" "math/bits" "path/filepath" "reflect" @@ -3419,14 +3418,10 @@ func TestEventTypes(t *testing.T) { cau.UpdateElementProof(&fce.StateElement) } - // finalize the contract - finalRevision := fce.V2FileContract - finalRevision.RevisionNumber = math.MaxUint64 - finalRevision.RenterSignature = types.Signature{} - finalRevision.HostSignature = types.Signature{} // create a renewal renewal := types.V2FileContractRenewal{ - FinalRevision: finalRevision, + FinalHostOutput: fc.HostOutput, + FinalRenterOutput: fc.RenterOutput, NewContract: types.V2FileContract{ RenterOutput: fc.RenterOutput, ProofHeight: fc.ProofHeight + 10, @@ -3441,12 +3436,15 @@ func TestEventTypes(t *testing.T) { renewalSig := pk.SignHash(renewalSigHash) renewal.RenterSignature = renewalSig renewal.HostSignature = renewalSig + contractSigHash := cm.TipState().ContractSigHash(renewal.NewContract) + renewal.NewContract.RenterSignature = pk.SignHash(contractSigHash) + renewal.NewContract.HostSignature = pk.SignHash(contractSigHash) sces = spendableSiacoinUTXOs() newContractValue := renterPayout.Add(cm.TipState().V2FileContractTax(renewal.NewContract)) - // renewals can't have change outputs - setupTxn := types.V2Transaction{ + // create the renewal transaction + resolutionTxn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { Parent: sces[0], @@ -3456,23 +3454,8 @@ func TestEventTypes(t *testing.T) { }, }, SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr, Value: newContractValue}, {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(newContractValue)}, }, - } - setupSigHash := cm.TipState().InputSigHash(setupTxn) - setupTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(setupSigHash)} - - // create the renewal transaction - resolutionTxn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{ - { - Parent: setupTxn.EphemeralSiacoinOutput(0), - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: policy, - }, - }, - }, FileContractResolutions: []types.V2FileContractResolution{ { Parent: fce, @@ -3484,105 +3467,7 @@ func TestEventTypes(t *testing.T) { resolutionTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(resolutionTxnSigHash)} // broadcast the renewal - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{setupTxn, resolutionTxn}); err != nil { - t.Fatal(err) - } - mineBlock(1, types.VoidAddress) - assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) - }) - - t.Run("v2 contract resolution - finalization", func(t *testing.T) { - sces := spendableSiacoinUTXOs() - - // using the UnlockConditions policy for brevity - policy := types.SpendPolicy{ - Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), - } - - // create a storage contract - renterPayout := types.Siacoins(10000) - fc := types.V2FileContract{ - RenterOutput: types.SiacoinOutput{ - Address: addr, - Value: renterPayout, - }, - HostOutput: types.SiacoinOutput{ - Address: types.VoidAddress, - Value: types.ZeroCurrency, - }, - ProofHeight: cm.TipState().Index.Height + 10, - ExpirationHeight: cm.TipState().Index.Height + 20, - - RenterPublicKey: pk.PublicKey(), - HostPublicKey: pk.PublicKey(), - } - contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) - sigHash := cm.TipState().ContractSigHash(fc) - sig := pk.SignHash(sigHash) - fc.RenterSignature = sig - fc.HostSignature = sig - - // create a transaction with the contract - txn := types.V2Transaction{ - FileContracts: []types.V2FileContract{fc}, - SiacoinInputs: []types.V2SiacoinInput{ - { - Parent: sces[0], - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: policy, - }, - }, - }, - SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(contractValue)}, - }, - } - sigHash = cm.TipState().InputSigHash(txn) - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} - - // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { - t.Fatal(err) - } - // current tip - tip := cm.Tip() - // mine until the contract proof window - mineBlock(1, types.VoidAddress) - - // this is even more annoying because we have to keep the file contract - // proof and the chain index proof up to date. - _, applied, err := cm.UpdatesSince(tip, 1000) - if err != nil { - t.Fatal(err) - } - - // get the confirmed file contract element - var fce types.V2FileContractElement - applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { - fce = ele - }) - for _, cau := range applied { - cau.UpdateElementProof(&fce.StateElement) - } - - // finalize the contract - fc = fce.V2FileContract - fc.RevisionNumber = types.MaxRevisionNumber - finalizationSigHash := cm.TipState().ContractSigHash(fc) - finalization := types.V2FileContractFinalization(pk.SignHash(finalizationSigHash)) - - // create the resolution transaction - finalizationTxn := types.V2Transaction{ - FileContractResolutions: []types.V2FileContractResolution{ - { - Parent: fce, - Resolution: &finalization, - }, - }, - } - - // broadcast the resolution - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{finalizationTxn}); err != nil { + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { t.Fatal(err) } mineBlock(1, types.VoidAddress) @@ -3616,7 +3501,7 @@ func TestEventTypes(t *testing.T) { } sigHash := cm.TipState().InputSigHash(txn) txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} - claimValue := cm.TipState().SiafundPool + claimValue := cm.TipState().SiafundTaxRevenue // broadcast the transaction if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { From ca3d83da27814d69457fe1ebf24f7e4577010d6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:00:05 +0000 Subject: [PATCH 298/630] build(deps): bump the all-dependencies group with 3 updates Bumps the all-dependencies group with 3 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/core` from 0.7.1-0.20241203090808-c6a988d759d6 to 0.7.1 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Commits](https://github.com/SiaFoundation/core/commits/v0.7.1) Updates `go.sia.tech/web/walletd` from 0.24.0 to 0.25.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.24.0...hostd@0.25.0) Updates `golang.org/x/term` from 0.26.0 to 0.27.0 - [Commits](https://github.com/golang/term/compare/v0.26.0...v0.27.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a64e890..3a8c100 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,12 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6 + go.sia.tech/core v0.7.1 go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.24.0 + go.sia.tech/web/walletd v0.25.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.26.0 + golang.org/x/term v0.27.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -25,6 +25,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.29.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index a9e0008..42e6d6a 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6 h1:52hztNcOJ+eql7dHMBl+g9VL4Lxr87cUrR9cXrYOkMs= -go.sia.tech/core v0.7.1-0.20241203090808-c6a988d759d6/go.mod h1:4v+aT/33857tMfqa5j5OYlAoLsoIrd4d7qMlgeP+VGk= +go.sia.tech/core v0.7.1 h1:PrKh19Ql5vJbQbB5YGtTHQ8W3fRF8hhYnR4kPOIOIME= +go.sia.tech/core v0.7.1/go.mod h1:gB8iXFJFSV8XIHRaL00CL6Be+hyykB+SYnvRPHCCc/E= go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 h1:Qskaf8d6oDKG5emNvGHZsd9iZRqz2GeouVNKY5paXlE= go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31/go.mod h1:d6jrawloc02MCXi/EVc8FIN5h3C6XDiMs4fuFMcU0PU= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= @@ -20,8 +20,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.24.0 h1:8l27tssquF/ONiXCd0m8707HIiohszGgjqJRHcGCTcE= -go.sia.tech/web/walletd v0.24.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.25.0 h1:RaI0ufYTPa+TWghu9I5JbOxpL7fW9BzCqWqxX5HbL5Y= +go.sia.tech/web/walletd v0.25.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -34,10 +34,10 @@ golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From e4f5880d4d0690d2b32c56ee0b48c190f2d7e2dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:51:39 +0000 Subject: [PATCH 299/630] build(deps): bump the all-dependencies group with 3 updates Bumps the all-dependencies group with 3 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/core` from 0.7.1 to 0.8.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.7.1...v0.8.0) Updates `go.sia.tech/coreutils` from 0.7.1-0.20241203172514-7bf95dd18f31 to 0.8.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/commits/v0.8.0) Updates `go.sia.tech/web/walletd` from 0.25.0 to 0.26.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.25.0...hostd@0.26.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 3a8c100..5c4a1b2 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.7.1 - go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 + go.sia.tech/core v0.8.0 + go.sia.tech/coreutils v0.8.0 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.25.0 + go.sia.tech/web/walletd v0.26.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.27.0 gopkg.in/yaml.v3 v3.0.1 @@ -24,7 +24,7 @@ require ( go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.29.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 42e6d6a..222b46e 100644 --- a/go.sum +++ b/go.sum @@ -10,26 +10,26 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.1 h1:PrKh19Ql5vJbQbB5YGtTHQ8W3fRF8hhYnR4kPOIOIME= -go.sia.tech/core v0.7.1/go.mod h1:gB8iXFJFSV8XIHRaL00CL6Be+hyykB+SYnvRPHCCc/E= -go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 h1:Qskaf8d6oDKG5emNvGHZsd9iZRqz2GeouVNKY5paXlE= -go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31/go.mod h1:d6jrawloc02MCXi/EVc8FIN5h3C6XDiMs4fuFMcU0PU= +go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= +go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= +go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0= +go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.25.0 h1:RaI0ufYTPa+TWghu9I5JbOxpL7fW9BzCqWqxX5HbL5Y= -go.sia.tech/web/walletd v0.25.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.26.0 h1:oQZbkN9ghDA7j5rg7k32DZiankf2HLVK9K5xCd82gIo= +go.sia.tech/web/walletd v0.26.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= From 8242676471a0086c06dae26995f11c7bf0bf7e84 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 13 Dec 2024 07:43:09 -0800 Subject: [PATCH 300/630] fix: Fix panic when resyncing after consensus database is deleted --- persist/sqlite/consensus.go | 48 +++++++++++++- wallet/manager.go | 9 ++- wallet/wallet_test.go | 125 ++++++++++++++++++++++++++++++++++-- 3 files changed, 174 insertions(+), 8 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 2129eab..25973d6 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -258,6 +258,52 @@ func (s *Store) SetIndexMode(mode wallet.IndexMode) error { }) } +// ResetChainState deletes all blockchain state from the database. +func (s *Store) ResetChainState() error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`UPDATE sia_addresses SET siacoin_balance=$1, siafund_balance=0, immature_siacoin_balance=$1`, encode(types.ZeroCurrency)) + if err != nil { + return fmt.Errorf("failed to reset sia addresses: %w", err) + } + + _, err = tx.Exec(`DELETE FROM siacoin_elements`) + if err != nil { + return fmt.Errorf("failed to delete siacoin elements: %w", err) + } + + _, err = tx.Exec(`DELETE FROM siafund_elements`) + if err != nil { + return fmt.Errorf("failed to delete siafund elements: %w", err) + } + + _, err = tx.Exec(`DELETE FROM state_tree`) + if err != nil { + return fmt.Errorf("failed to delete state tree: %w", err) + } + + _, err = tx.Exec(`DELETE FROM event_addresses`) + if err != nil { + return fmt.Errorf("failed to delete event addresses: %w", err) + } + + _, err = tx.Exec(`DELETE FROM events`) + if err != nil { + return fmt.Errorf("failed to delete events: %w", err) + } + + _, err = tx.Exec(`DELETE FROM chain_indices`) + if err != nil { + return fmt.Errorf("failed to delete chain indices: %w", err) + } + + _, err = tx.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1, element_num_leaves=0`, encode(types.BlockID{})) + if err != nil { + return fmt.Errorf("failed to reset global settings: %w", err) + } + return nil + }) +} + func getSiacoinStateElements(tx *txn) ([]stateElement, error) { const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` rows, err := tx.Query(query) @@ -1212,7 +1258,7 @@ RETURNING id, address_id, siafund_value`, index.Height, encode(index.ID)) func deleteOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { rows, err := tx.Query(`DELETE FROM siafund_elements WHERE id IN (SELECT se.id FROM siafund_elements se INNER JOIN chain_indices ci ON (ci.id=se.chain_index_id) -WHERE ci.height=$1 AND ci.block_id<>$2) +WHERE ci.height=$1 AND ci.block_id<>$2) RETURNING id, address_id, siafund_value, spent_index_id IS NOT NULL`, index.Height, encode(index.ID)) if err != nil { return nil, fmt.Errorf("failed to query siafund elements: %w", err) diff --git a/wallet/manager.go b/wallet/manager.go index dcda2ce..52f64cf 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "sync" "time" @@ -54,6 +55,7 @@ type ( // A Store is a persistent store of wallet data. Store interface { UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error + ResetChainState() error WalletUnconfirmedEvents(id ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []Event, err error) WalletEvents(walletID ID, offset, limit int) ([]Event, error) @@ -380,7 +382,12 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) m.mu.Lock() // update the store lastTip, err := store.LastCommittedIndex() - if err != nil { + if err != nil && strings.Contains(err.Error(), "missing block at index") { + log.Warn("missing block at index, resetting chain state", zap.Uint64("height", lastTip.Height), zap.Stringer("id", lastTip.ID)) + if err := store.ResetChainState(); err != nil { + log.Panic("failed to reset chain state", zap.Error(err)) + } + } else if err != nil { log.Panic("failed to get last committed index", zap.Error(err)) } else if err := syncStore(ctx, store, cm, lastTip, m.syncBatchSize); err != nil && !errors.Is(err, context.Canceled) { log.Panic("failed to sync store", zap.Error(err)) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index bdd6406..dfce746 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -21,6 +21,7 @@ import ( "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" + "lukechampine.com/frand" ) func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { @@ -577,13 +578,13 @@ func TestWalletAddresses(t *testing.T) { SpendPolicy: &spendPolicy, Description: "hello, world", } - err = db.AddWalletAddress(w.ID, addr) + err = wm.AddAddress(w.ID, addr) if err != nil { t.Fatal(err) } // Check that the address was added - addresses, err := db.WalletAddresses(w.ID) + addresses, err := wm.Addresses(w.ID) if err != nil { t.Fatal(err) } else if len(addresses) != 1 { @@ -600,12 +601,12 @@ func TestWalletAddresses(t *testing.T) { addr.Description = "goodbye, world" addr.Metadata = json.RawMessage(`{"foo": "bar"}`) - if err := db.AddWalletAddress(w.ID, addr); err != nil { + if err := wm.AddAddress(w.ID, addr); err != nil { t.Fatal(err) } // Check that the address was added - addresses, err = db.WalletAddresses(w.ID) + addresses, err = wm.Addresses(w.ID) if err != nil { t.Fatal(err) } else if len(addresses) != 1 { @@ -621,13 +622,13 @@ func TestWalletAddresses(t *testing.T) { } // Remove the address - err = db.RemoveWalletAddress(w.ID, address) + err = wm.RemoveAddress(w.ID, address) if err != nil { t.Fatal(err) } // Check that the address was removed - addresses, err = db.WalletAddresses(w.ID) + addresses, err = wm.Addresses(w.ID) if err != nil { t.Fatal(err) } else if len(addresses) != 0 { @@ -3512,3 +3513,115 @@ func TestEventTypes(t *testing.T) { assertEvent(t, types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) }) } + +func TestReset(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // helper to mine blocks + mineBlock := func(n int, addr types.Address) { + t.Helper() + for i := 0; i < n; i++ { + b, ok := coreutils.MineBlock(cm, addr, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + } + + assertBalance := func(t *testing.T, addr types.Address, siacoin types.Currency, siafund uint64) { + t.Helper() + + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(siacoin) { + t.Fatalf("expected %v SC, got %v", siacoin, balance.Siacoins) + } else if balance.Siafunds != siafund { + t.Fatalf("expected %v siafunds, got %v", siafund, balance.Siafunds) + } + } + + // mine a payout to the original address + mineBlock(1, addr) + + assertBalance(t, addr, types.ZeroCurrency, genesisState.SiafundCount()) + + // mine a bunch of payouts to random addresses + for i := 0; i < 50; i++ { + mineBlock(1, frand.Entropy256()) + } + + assertBalance(t, addr, genesisState.BlockReward(), genesisState.SiafundCount()) + + // close the wallet and reset it + if err := wm.Close(); err != nil { + t.Fatal(err) + } else if err := db.ResetChainState(); err != nil { + t.Fatal(err) + } + + index, err := db.LastCommittedIndex() + if err != nil { + t.Fatal(err) + } else if index.Height != 0 { + t.Fatalf("expected height 0, got %v", index.Height) + } else if index.ID != (types.BlockID{}) { + t.Fatalf("expected zero ID, got %v", index.ID) + } + + // balance should be reset + assertBalance(t, addr, types.ZeroCurrency, 0) + events, err := db.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // reopen the wallet + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // mine a block to trigger sync + mineBlock(1, types.VoidAddress) + + assertBalance(t, addr, genesisState.BlockReward(), genesisState.SiafundCount()) +} From d402a8760ff75892b613235bffd28be99733fbfa Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 18 Dec 2024 21:22:28 -0800 Subject: [PATCH 301/630] wallet: add test reset --- wallet/manager.go | 29 ++++++--- wallet/wallet_test.go | 139 +++++++++++++++++++++++------------------- 2 files changed, 97 insertions(+), 71 deletions(-) diff --git a/wallet/manager.go b/wallet/manager.go index 52f64cf..3c5b7ac 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -382,15 +382,28 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) m.mu.Lock() // update the store lastTip, err := store.LastCommittedIndex() - if err != nil && strings.Contains(err.Error(), "missing block at index") { - log.Warn("missing block at index, resetting chain state", zap.Uint64("height", lastTip.Height), zap.Stringer("id", lastTip.ID)) - if err := store.ResetChainState(); err != nil { - log.Panic("failed to reset chain state", zap.Error(err)) - } - } else if err != nil { + if err != nil { log.Panic("failed to get last committed index", zap.Error(err)) - } else if err := syncStore(ctx, store, cm, lastTip, m.syncBatchSize); err != nil && !errors.Is(err, context.Canceled) { - log.Panic("failed to sync store", zap.Error(err)) + } + err = syncStore(ctx, store, cm, lastTip, m.syncBatchSize) + if err != nil { + switch { + case errors.Is(err, context.Canceled): + m.mu.Unlock() + return + case strings.Contains(err.Error(), "missing block at index"): // unfortunate, but not exposed by coreutils + log.Warn("missing block at index, resetting chain state", zap.Stringer("id", lastTip.ID), zap.Uint64("height", lastTip.Height)) + if err := store.ResetChainState(); err != nil { + log.Panic("failed to reset wallet state", zap.Error(err)) + } + // trigger resync + select { + case reorgChan <- struct{}{}: + default: + } + default: + log.Panic("failed to sync store", zap.Error(err)) + } } m.mu.Unlock() } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index dfce746..d672093 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -21,7 +21,6 @@ import ( "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" - "lukechampine.com/frand" ) func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { @@ -3516,18 +3515,6 @@ func TestEventTypes(t *testing.T) { func TestReset(t *testing.T) { log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -3536,92 +3523,118 @@ func TestReset(t *testing.T) { // send the siafunds to the owned address genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) if err != nil { t.Fatal(err) } + defer bdb.Close() - cm := chain.NewManager(store, genesisState) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm1 := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + bdb2, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus2.db")) if err != nil { t.Fatal(err) } - defer wm.Close() + defer bdb2.Close() + store2, genesisState2, err := chain.NewDBStore(bdb2, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm2 := chain.NewManager(store2, genesisState2) - // helper to mine blocks - mineBlock := func(n int, addr types.Address) { - t.Helper() - for i := 0; i < n; i++ { - b, ok := coreutils.MineBlock(cm, addr, 15*time.Second) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } + // mine blocks before starting the wallet manager + for i := 0; i < 25; i++ { + // blocks on the first chain manager go to the void + b1, ok := coreutils.MineBlock(cm1, types.VoidAddress, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm1.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) } - waitForBlock(t, cm, db) + + // blocks on the second one go to the primary address + b2, ok := coreutils.MineBlock(cm2, addr, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm2.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + } + + db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) } + defer db.Close() + + // wait for the manager to sync to the first chain + wm, err := wallet.NewManager(cm1, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm1, db) - assertBalance := func(t *testing.T, addr types.Address, siacoin types.Currency, siafund uint64) { + assertBalance := func(t *testing.T, addr types.Address, siacoin, immature types.Currency, siafund uint64) { t.Helper() balance, err := db.AddressBalance(addr) if err != nil { t.Fatal(err) - } else if !balance.Siacoins.Equals(siacoin) { + } + switch { + case !balance.Siacoins.Equals(siacoin): t.Fatalf("expected %v SC, got %v", siacoin, balance.Siacoins) - } else if balance.Siafunds != siafund { + case !balance.ImmatureSiacoins.Equals(immature): + t.Fatalf("expected immature %v SC, got %v", siacoin, balance.Siacoins) + case balance.Siafunds != siafund: t.Fatalf("expected %v siafunds, got %v", siafund, balance.Siafunds) } } - // mine a payout to the original address - mineBlock(1, addr) - - assertBalance(t, addr, types.ZeroCurrency, genesisState.SiafundCount()) - - // mine a bunch of payouts to random addresses - for i := 0; i < 50; i++ { - mineBlock(1, frand.Entropy256()) - } + assertBalance(t, addr, types.ZeroCurrency, types.ZeroCurrency, 10000) - assertBalance(t, addr, genesisState.BlockReward(), genesisState.SiafundCount()) - - // close the wallet and reset it + // close the manager if err := wm.Close(); err != nil { - t.Fatal(err) - } else if err := db.ResetChainState(); err != nil { - t.Fatal(err) + t.Fatal() } - index, err := db.LastCommittedIndex() + // calculate the expected balances + _, applied, err := cm2.UpdatesSince(types.ChainIndex{}, 1000) if err != nil { t.Fatal(err) - } else if index.Height != 0 { - t.Fatalf("expected height 0, got %v", index.Height) - } else if index.ID != (types.BlockID{}) { - t.Fatalf("expected zero ID, got %v", index.ID) } - // balance should be reset - assertBalance(t, addr, types.ZeroCurrency, 0) - events, err := db.AddressEvents(addr, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(events) != 0 { - t.Fatalf("expected 0 events, got %v", len(events)) + var siacoinElements []types.SiacoinElement + for _, cau := range applied { + cau.ForEachSiacoinElement(func(sce types.SiacoinElement, created, spent bool) { + if created && sce.SiacoinOutput.Address == addr { + siacoinElements = append(siacoinElements, sce) + } + }) + } + + var expectedSiacoins, expectedImmature types.Currency + for _, sce := range siacoinElements { + if sce.MaturityHeight > cm2.Tip().Height { + expectedImmature = expectedImmature.Add(sce.SiacoinOutput.Value) + } else { + expectedSiacoins = expectedSiacoins.Add(sce.SiacoinOutput.Value) + } } - // reopen the wallet - wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + wm, err = wallet.NewManager(cm2, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) if err != nil { t.Fatal(err) } defer wm.Close() - // mine a block to trigger sync - mineBlock(1, types.VoidAddress) + waitForBlock(t, cm2, db) - assertBalance(t, addr, genesisState.BlockReward(), genesisState.SiafundCount()) + assertBalance(t, addr, expectedSiacoins, expectedImmature, genesisState.SiafundCount()) } From 850c2cd1708a3336d39d121328e06257ec1e1a5d Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 19 Dec 2024 14:32:41 -0800 Subject: [PATCH 302/630] chore: update dependencies --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5c4a1b2..601adb2 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.8.0 - go.sia.tech/coreutils v0.8.0 + go.sia.tech/core v0.9.0 + go.sia.tech/coreutils v0.9.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.26.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 222b46e..0b5b635 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= -go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= -go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0= -go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= +go.sia.tech/core v0.9.0 h1:qV7V8nkNaPvBEhkbwgrETTkb7JCMcAnKUQt9nUumP4k= +go.sia.tech/core v0.9.0/go.mod h1:3NAvYHuzAZg9vP6pyIMOxjTkgHBQ3vx9cXTqRF6oEa4= +go.sia.tech/coreutils v0.9.0 h1:5cnK0RtHOyErGhcmNkmCdEKeuj1tECwO9PYbErEbpDQ= +go.sia.tech/coreutils v0.9.0/go.mod h1:KFq1q5/YbPH6ZSWtXCxA1bRhBF5Zgcj8G3Wvu0jr/BA= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= From 2405a353c4ef81fd8766dfb67bc3b2cf54462084 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 21 Dec 2024 09:25:05 -0800 Subject: [PATCH 303/630] chore: document change --- .changeset/support_v2_hardfork.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .changeset/support_v2_hardfork.md diff --git a/.changeset/support_v2_hardfork.md b/.changeset/support_v2_hardfork.md new file mode 100644 index 0000000..a2c2045 --- /dev/null +++ b/.changeset/support_v2_hardfork.md @@ -0,0 +1,26 @@ +--- +default: major +--- + +# Support V2 Hardfork + +The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. + +#### V2 Highlights +- Drastically reduces blockchain size on disk +- Improves UTXO spend policies - including HTLC support for Atomic Swaps +- More efficient contract renewals - reducing lock up requirements for hosts and renters +- Improved transfer speeds - enables hot storage + +#### Phase 1 - Allow Height +- **Activation Height:** `513400` (March 10th, 2025) +- **New Features:** V2 transactions, contracts, and RHP4 +- **V1 Support:** Both V1 and V2 will be supported during this phase +- **Purpose:** This period gives time for integrators to transition from V1 to V2 +- **Requirements:** Users will need to update to support the hardfork before this block height + +#### Phase 2 - Require Height +- **Activation Height:** `526000` (June 6th, 2025) +- **New Features:** The consensus database can be trimmed to only store the Merkle proofs +- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted +- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height From ce674a376aa98cc50b47f8e357595aa3315839f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 17:04:16 +0000 Subject: [PATCH 304/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.26.0 to 0.27.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.26.0...hostd@0.27.0) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 601adb2..3989790 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.9.0 go.sia.tech/coreutils v0.9.0 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.26.0 + go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.27.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 0b5b635..ff6ff83 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.26.0 h1:oQZbkN9ghDA7j5rg7k32DZiankf2HLVK9K5xCd82gIo= -go.sia.tech/web/walletd v0.26.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.27.0 h1:oTCqqZHvvWbcy/jKMN7urBEFYXQs1+yVzuKu17vgQtk= +go.sia.tech/web/walletd v0.27.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= From 6c40e2b694ed0491dc7346332fc1d4e49825b6f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:58:33 +0000 Subject: [PATCH 305/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.27.0 to 0.28.0 - [Commits](https://github.com/golang/term/compare/v0.27.0...v0.28.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3989790..2bae8b3 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.27.0 + golang.org/x/term v0.28.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -25,6 +25,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index ff6ff83..80525cb 100644 --- a/go.sum +++ b/go.sum @@ -34,10 +34,10 @@ golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 90b3bd374d311a2796fd105d70de0561eac42d8d Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 2 Jan 2025 09:03:06 -0800 Subject: [PATCH 306/630] refactor: move utxo selection out of API and into wallet manager --- api/server.go | 181 ++++++++++--------------------- knope.toml | 1 + wallet/manager.go | 243 ++++++++++++++++++++++++++++++++++++++++++ wallet/wallet_test.go | 171 +++++++++++++++++++++++++++++ 4 files changed, 470 insertions(+), 126 deletions(-) diff --git a/api/server.go b/api/server.go index 4da2206..84b3dc0 100644 --- a/api/server.go +++ b/api/server.go @@ -6,14 +6,12 @@ import ( "fmt" "net/http" "net/http/pprof" - "reflect" "runtime" "sync" "time" "go.sia.tech/jape" "go.uber.org/zap" - "lukechampine.com/frand" "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" @@ -101,6 +99,8 @@ type ( Addresses(id wallet.ID) ([]wallet.Address, error) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) + SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool, expiration time.Duration) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) + SelectSiafundElements(walletID wallet.ID, amount uint64, expiration time.Duration) ([]types.SiafundElement, types.ChainIndex, uint64, error) UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) WalletBalance(id wallet.ID) (wallet.Balance, error) @@ -116,7 +116,8 @@ type ( SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) - Reserve(ids []types.Hash256, duration time.Duration) error + Reserve([]types.Hash256, time.Duration) error + Release([]types.Hash256) } ) @@ -131,10 +132,6 @@ type server struct { s Syncer wm WalletManager - // for walletsReserveHandler - mu sync.Mutex - used map[types.Hash256]bool - scanMu sync.Mutex // for resubscribe scanInProgress bool scanInfo RescanResponse @@ -581,88 +578,55 @@ func (s *server) walletsReserveHandler(jc jape.Context) { } func (s *server) walletsReleaseHandler(jc jape.Context) { - var name string var wrr WalletReleaseRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wrr) != nil { + if jc.Decode(&wrr) != nil { return } - s.mu.Lock() - defer s.mu.Unlock() + + ids := make([]types.Hash256, 0, len(wrr.SiacoinOutputs)+len(wrr.SiafundOutputs)) for _, id := range wrr.SiacoinOutputs { - delete(s.used, types.Hash256(id)) + ids = append(ids, types.Hash256(id)) } for _, id := range wrr.SiafundOutputs { - delete(s.used, types.Hash256(id)) + ids = append(ids, types.Hash256(id)) } + s.wm.Release(ids) jc.EmptyResonse() } func (s *server) walletsFundHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount types.Currency, utxos []types.SiacoinElement, changeAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { - s.mu.Lock() - defer s.mu.Unlock() - if amount.IsZero() { - return nil, nil - } - inPool := make(map[types.Hash256]bool) - for _, ptxn := range pool { - for _, in := range ptxn.SiacoinInputs { - inPool[types.Hash256(in.ParentID)] = true - } - } - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) - var outputSum types.Currency - var fundingElements []types.SiacoinElement - for _, sce := range utxos { - if s.used[types.Hash256(sce.ID)] || inPool[types.Hash256(sce.ID)] { - continue - } - fundingElements = append(fundingElements, sce) - outputSum = outputSum.Add(sce.SiacoinOutput.Value) - if outputSum.Cmp(amount) >= 0 { - break - } - } - if outputSum.Cmp(amount) < 0 { - return nil, errors.New("insufficient balance") - } else if outputSum.Cmp(amount) > 0 { - if changeAddr == types.VoidAddress { - return nil, errors.New("change address must be specified") - } - txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ - Value: outputSum.Sub(amount), - Address: changeAddr, - }) - } - - toSign := make([]types.Hash256, len(fundingElements)) - for i, sce := range fundingElements { - txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ - ParentID: types.SiacoinOutputID(sce.ID), - // UnlockConditions left empty for client to fill in - }) - toSign[i] = types.Hash256(sce.ID) - s.used[types.Hash256(sce.ID)] = true - } - - return toSign, nil - } - var id wallet.ID var wfr WalletFundRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiacoinOutputs(id, 0, 1000) + utxos, _, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false, time.Hour) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } txn := wfr.Transaction - toSign, err := fundTxn(&txn, wfr.Amount, utxos, wfr.ChangeAddress, s.cm.PoolTransactions()) - if jc.Check("couldn't fund transaction", err) != nil { - return + if !change.IsZero() { + if wfr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ + Value: change, + Address: wfr.ChangeAddress, + }) } + + toSign := make([]types.Hash256, 0, len(utxos)) + for _, sce := range utxos { + txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ + ParentID: sce.ID, + // UnlockConditions left empty for client to fill in + }) + toSign = append(toSign, types.Hash256(sce.ID)) + } + jc.Encode(WalletFundResponse{ Transaction: txn, ToSign: toSign, @@ -671,71 +635,37 @@ func (s *server) walletsFundHandler(jc jape.Context) { } func (s *server) walletsFundSFHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount uint64, utxos []types.SiafundElement, changeAddr, claimAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { - s.mu.Lock() - defer s.mu.Unlock() - if amount == 0 { - return nil, nil - } - inPool := make(map[types.Hash256]bool) - for _, ptxn := range pool { - for _, in := range ptxn.SiafundInputs { - inPool[types.Hash256(in.ParentID)] = true - } - } - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) - var outputSum uint64 - var fundingElements []types.SiafundElement - for _, sfe := range utxos { - if s.used[types.Hash256(sfe.ID)] || inPool[types.Hash256(sfe.ID)] { - continue - } - fundingElements = append(fundingElements, sfe) - outputSum += sfe.SiafundOutput.Value - if outputSum >= amount { - break - } - } - if outputSum < amount { - return nil, errors.New("insufficient balance") - } else if outputSum > amount { - if changeAddr == types.VoidAddress { - return nil, errors.New("change address must be specified") - } - txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ - Value: outputSum - amount, - Address: changeAddr, - }) - } - - toSign := make([]types.Hash256, len(fundingElements)) - for i, sfe := range fundingElements { - txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ - ParentID: types.SiafundOutputID(sfe.ID), - ClaimAddress: claimAddr, - // UnlockConditions left empty for client to fill in - }) - toSign[i] = types.Hash256(sfe.ID) - s.used[types.Hash256(sfe.ID)] = true - } - - return toSign, nil - } - var id wallet.ID var wfr WalletFundSFRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, err := s.wm.UnspentSiafundOutputs(id, 0, 1000) + utxos, _, change, err := s.wm.SelectSiafundElements(id, wfr.Amount, time.Hour) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } txn := wfr.Transaction - toSign, err := fundTxn(&txn, wfr.Amount, utxos, wfr.ChangeAddress, wfr.ClaimAddress, s.cm.PoolTransactions()) - if jc.Check("couldn't fund transaction", err) != nil { - return + if change > 0 { + if wfr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ + Value: change, + Address: wfr.ChangeAddress, + }) + } + + toSign := make([]types.Hash256, 0, len(utxos)) + for _, sce := range utxos { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sce.ID, + ClaimAddress: wfr.ChangeAddress, + // UnlockConditions left empty for client to fill in + }) + toSign = append(toSign, types.Hash256(sce.ID)) } jc.Encode(WalletFundResponse{ Transaction: txn, @@ -924,10 +854,9 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption publicEndpoints: false, startTime: time.Now(), - cm: cm, - s: s, - wm: wm, - used: make(map[types.Hash256]bool), + cm: cm, + s: s, + wm: wm, } for _, opt := range opts { opt(&srv) diff --git a/knope.toml b/knope.toml index 2cbe180..1034ad0 100644 --- a/knope.toml +++ b/knope.toml @@ -18,6 +18,7 @@ command = "git switch -c release" [[workflows.steps]] type = "PrepareRelease" +ignore_conventional_commits = true [[workflows.steps]] type = "Command" diff --git a/wallet/manager.go b/wallet/manager.go index 3c5b7ac..c8e51fc 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -36,6 +36,12 @@ const ( const defaultSyncBatchSize = 1 +var ( + // ErrInsufficientFunds is returned when there are not enough funds to + // fund a transaction. + ErrInsufficientFunds = errors.New("insufficient funds") +) + type ( // An IndexMode determines the chain state that the wallet manager stores. IndexMode uint8 @@ -63,6 +69,7 @@ type ( UpdateWallet(Wallet) (Wallet, error) DeleteWallet(walletID ID) error WalletBalance(walletID ID) (Balance, error) + WalletAddress(ID, types.Address) (Address, error) WalletSiacoinOutputs(walletID ID, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, error) WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) WalletAddresses(walletID ID) ([]Address, error) @@ -265,6 +272,240 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { return nil } +// Release releases the given ids. +func (m *Manager) Release(ids []types.Hash256) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, id := range ids { + delete(m.used, id) + } +} + +// SelectSiacoinElements selects siacoin elements from the wallet that sum to +// at least the given amount. Returns the elements, the element basis, and the +// change amount. The selected elements are locked for the given duration. +func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool, expiration time.Duration) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { + m.mu.Lock() + defer m.mu.Unlock() + + knownAddresses := make(map[types.Address]bool) + relevantAddr := func(addr types.Address) bool { + if exists, ok := knownAddresses[addr]; ok { + return exists + } + _, err := m.store.WalletAddress(walletID, addr) + if errors.Is(err, ErrNotFound) { + knownAddresses[addr] = false + return false + } else if err != nil { + panic(err) + } + knownAddresses[addr] = true + return true + } + + ephemeral := make(map[types.SiacoinOutputID]types.SiacoinElement) + inPool := make(map[types.SiacoinOutputID]bool) + for _, txn := range m.chain.PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + inPool[sci.ParentID] = true + delete(ephemeral, sci.ParentID) + } + for i, sco := range txn.SiacoinOutputs { + if relevantAddr(sco.Address) { + scoid := txn.SiacoinOutputID(i) + ephemeral[scoid] = types.SiacoinElement{ + ID: scoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiacoinOutput: sco, + } + } + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + inPool[sci.Parent.ID] = true + delete(ephemeral, sci.Parent.ID) + } + for i, sco := range txn.SiacoinOutputs { + if relevantAddr(sco.Address) { + sce := txn.EphemeralSiacoinOutput(i) + ephemeral[sce.ID] = sce + } + } + } + + tip := m.chain.Tip() + if amount.IsZero() { + return nil, tip, types.ZeroCurrency, nil + } + + var inputSum types.Currency + var selected []types.SiacoinElement + const utxoBatchSize = 100 +top: + for i := 0; ; i += utxoBatchSize { + // extra large wallets may need to paginate through utxos + // to find enough to cover the amount + utxos, err := m.store.WalletSiacoinOutputs(walletID, tip, i, utxoBatchSize) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to get siacoin elements: %w", err) + } else if len(utxos) == 0 { + return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + } + + for _, sce := range utxos { + if inPool[sce.ID] || m.used[types.Hash256(sce.ID)] { + continue + } + + selected = append(selected, sce) + inputSum = inputSum.Add(sce.SiacoinOutput.Value) + if inputSum.Cmp(amount) >= 0 { + break top + } + } + } + + if inputSum.Cmp(amount) < 0 { + if !useUnconfirmed { + return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + } + + for _, sce := range ephemeral { + if inPool[sce.ID] || m.used[types.Hash256(sce.ID)] { + continue + } + + selected = append(selected, sce) + inputSum = inputSum.Add(sce.SiacoinOutput.Value) + if inputSum.Cmp(amount) >= 0 { + break + } + } + } + + if inputSum.Cmp(amount) < 0 { + return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + } + + for _, sce := range selected { + m.used[types.Hash256(sce.ID)] = true + } + time.AfterFunc(expiration, func() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, sce := range selected { + delete(m.used, types.Hash256(sce.ID)) + } + }) + return selected, tip, inputSum.Sub(amount), nil +} + +// SelectSiafundElements selects siacoin elements from the wallet that sum to +// at least the given amount. Returns the elements, the element basis, and the +// change amount. The selected elements are locked for the given +// duration. +func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration time.Duration) ([]types.SiafundElement, types.ChainIndex, uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() + + tip := m.chain.Tip() + if amount == 0 { + return nil, tip, 0, nil + } + + knownAddresses := make(map[types.Address]bool) + relevantAddr := func(addr types.Address) bool { + if exists, ok := knownAddresses[addr]; ok { + return exists + } + _, err := m.store.WalletAddress(walletID, addr) + if errors.Is(err, ErrNotFound) { + knownAddresses[addr] = false + return false + } else if err != nil { + panic(err) + } + knownAddresses[addr] = true + return true + } + + ephemeral := make(map[types.SiafundOutputID]types.SiafundElement) + inPool := make(map[types.SiafundOutputID]bool) + for _, txn := range m.chain.PoolTransactions() { + for _, sfi := range txn.SiafundInputs { + inPool[sfi.ParentID] = true + delete(ephemeral, sfi.ParentID) + } + for i, sfo := range txn.SiafundOutputs { + if relevantAddr(sfo.Address) { + sfoid := txn.SiafundOutputID(i) + ephemeral[sfoid] = types.SiafundElement{ + ID: sfoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiafundOutput: sfo, + } + } + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, sfi := range txn.SiafundInputs { + inPool[sfi.Parent.ID] = true + delete(ephemeral, sfi.Parent.ID) + } + for i, sfo := range txn.SiafundOutputs { + if relevantAddr(sfo.Address) { + sfe := txn.EphemeralSiafundOutput(i) + ephemeral[sfe.ID] = sfe + } + } + } + + var inputSum uint64 + var selected []types.SiafundElement +top: + for i := 0; ; i++ { + utxos, err := m.store.WalletSiafundOutputs(walletID, i, 100) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to get siacoin elements: %w", err) + } else if len(utxos) == 0 { + return nil, types.ChainIndex{}, 0, ErrInsufficientFunds + } + + for _, sfe := range utxos { + if inPool[sfe.ID] || m.used[types.Hash256(sfe.ID)] { + continue + } + + selected = append(selected, sfe) + inputSum += sfe.SiafundOutput.Value + if inputSum >= amount { + break top + } + } + } + + if inputSum < amount { + return nil, types.ChainIndex{}, 0, ErrInsufficientFunds + } + + for _, sce := range selected { + m.used[types.Hash256(sce.ID)] = true + } + time.AfterFunc(expiration, func() { + m.mu.Lock() + defer m.mu.Unlock() + + for _, sce := range selected { + delete(m.used, types.Hash256(sce.ID)) + } + }) + return selected, tip, inputSum - amount, nil +} + // Scan rescans the chain starting from the given index. The scan will complete // when the chain manager reaches the current tip or the context is canceled. func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { @@ -338,6 +579,8 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) store: store, log: zap.NewNop(), tg: threadgroup.New(), + + used: make(map[types.Hash256]bool), } for _, opt := range opts { diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index d672093..13de1fa 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "math/bits" "path/filepath" @@ -40,6 +41,7 @@ func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) n, genesisBlock := chain.TestnetZen() genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} + n.MaturityDelay = 5 n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 n.HardforkStorageProof.Height = 1 @@ -56,6 +58,7 @@ func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) n, genesisBlock := chain.TestnetZen() genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} + n.MaturityDelay = 5 n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 n.HardforkStorageProof.Height = 1 @@ -98,6 +101,174 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty return b } +func TestSelectSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + network.InitialCoinbase = types.Siacoins(100) + network.MinimumCoinbase = types.Siacoins(100) + network.MaturityDelay = 5 + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sk := types.GeneratePrivateKey() + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{sk.PublicKey().UnlockKey()}, + SignaturesRequired: 1, + } + addr := uc.UnlockHash() + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + mineAndSync := func(t *testing.T, addr types.Address, n int) { + t.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(t, cm, addr, 1) + waitForBlock(t, cm, db) + } + } + // mine enough utxos to ensure the pagination works + mineAndSync(t, addr, 200) + // mine until all the wallet's outputs are mature + mineAndSync(t, types.VoidAddress, int(cm.TipState().Network.MaturityDelay)) + + // check that the wallet has 200 matured outputs + utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 200 { + t.Fatalf("expected 200 outputs, got %v", len(utxos)) + } + + balance, err := wm.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } + + // fund a transaction with more than the wallet balance + _, _, _, err = wm.SelectSiacoinElements(w.ID, balance.Siacoins.Add(types.Siacoins(1)), false, time.Minute) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + + // fund multiple overlapping transactions to ensure no double spends + var selected []types.Hash256 + seen := make(map[types.SiacoinOutputID]bool) + for i := 0; i < len(utxos); i++ { + utxos, _, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false, time.Minute) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { // one UTXO should always be enough to cover + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if seen[utxos[0].ID] { + t.Fatalf("double spend %v", utxos[0].ID) + } else if !change.Equals(types.Siacoins(99)) { + t.Fatalf("expected 99 SC change, got %v", change) + } + seen[utxos[0].ID] = true + selected = append(selected, types.Hash256(utxos[0].ID)) + } + + // all available outputs should be locked + _, _, _, err = wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false, time.Minute) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + // release the selected outputs + wm.Release(selected) + + // fund and broadcast a transaction + utxos, basis, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(101), false, time.Minute) // uses two outputs + if err != nil { + t.Fatal(err) + } else if len(utxos) != 2 { + t.Fatalf("expected 2 outputs, got %v", len(utxos)) + } else if !change.Equals(types.Siacoins(99)) { + t.Fatalf("expected 99 SC change, got %v", change) + } else if basis != cm.Tip() { + t.Fatalf("expected tip, got %v", basis) + } + txn := types.Transaction{ + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(101)}, + {Address: addr, Value: change}, + }, + } + for _, utxo := range utxos { + txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ + ParentID: types.SiacoinOutputID(utxo.ID), + UnlockConditions: uc, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxo.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + for i := range txn.Signatures { + sigHash := cm.TipState().WholeSigHash(txn, txn.Signatures[i].ParentID, 0, 0, nil) + sig := sk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + + known, err := cm.AddPoolTransactions([]types.Transaction{txn}) + if err != nil { + t.Fatal(err) + } else if known { + t.Fatal("transaction was already known") + } + + mineAndSync(t, types.VoidAddress, 1) + + events, err := wm.WalletEvents(w.ID, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if !events[0].SiacoinOutflow().Sub(events[0].SiacoinInflow()).Equals(types.Siacoins(101)) { + t.Fatalf("expected transaction value 101 SC, got %v", events[0].SiacoinOutflow().Sub(events[0].SiacoinInflow())) + } +} + func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) From 5bcbd15ae0ee5f2c105e0ae0d93f8c4fb0266a5d Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 2 Jan 2025 09:06:35 -0800 Subject: [PATCH 307/630] sqlite: add missing WalletAddress impl --- persist/sqlite/wallet.go | 66 ++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 45f572f..4400558 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -176,6 +176,24 @@ func (s *Store) RemoveWalletAddress(id wallet.ID, address types.Address) error { }) } +// WalletAddress returns an address registered to the wallet. +func (s *Store) WalletAddress(id wallet.ID, address types.Address) (addr wallet.Address, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data +FROM wallet_addresses wa +INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) +WHERE wa.wallet_id=$1 AND sa.sia_address=$2` + + addr, err = scanWalletAddress(tx.QueryRow(query, id, encode(address))) + return err + }) + return +} + // WalletAddresses returns a slice of addresses registered to the wallet. func (s *Store) WalletAddresses(id wallet.ID) (addresses []wallet.Address, err error) { err = s.transaction(func(tx *txn) error { @@ -195,27 +213,11 @@ WHERE wa.wallet_id=$1` defer rows.Close() for rows.Next() { - var address wallet.Address - var decodedPolicy any - if err := rows.Scan(decode(&address.Address), &address.Description, &decodedPolicy, (*[]byte)(&address.Metadata)); err != nil { + addr, err := scanWalletAddress(rows) + if err != nil { return fmt.Errorf("failed to scan address: %w", err) } - - if decodedPolicy != nil { - switch v := decodedPolicy.(type) { - case []byte: - dec := types.NewBufDecoder(v) - address.SpendPolicy = new(types.SpendPolicy) - address.SpendPolicy.DecodeFrom(dec) - if err := dec.Err(); err != nil { - return fmt.Errorf("failed to decode spend policy: %w", err) - } - default: - return fmt.Errorf("unexpected spend policy type: %T", decodedPolicy) - } - } - - addresses = append(addresses, address) + addresses = append(addresses, addr) } return rows.Err() }) @@ -611,6 +613,32 @@ RETURNING id` return } +func scanWalletAddress(s scanner) (wallet.Address, error) { + var address wallet.Address + var decodedPolicy any + if err := s.Scan(decode(&address.Address), &address.Description, &decodedPolicy, (*[]byte)(&address.Metadata)); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return wallet.Address{}, wallet.ErrNotFound + } + return wallet.Address{}, fmt.Errorf("failed to scan address: %w", err) + } + + if decodedPolicy != nil { + switch v := decodedPolicy.(type) { + case []byte: + dec := types.NewBufDecoder(v) + address.SpendPolicy = new(types.SpendPolicy) + address.SpendPolicy.DecodeFrom(dec) + if err := dec.Err(); err != nil { + return wallet.Address{}, fmt.Errorf("failed to decode spend policy: %w", err) + } + default: + return wallet.Address{}, fmt.Errorf("unexpected spend policy type: %T", decodedPolicy) + } + } + return address, nil +} + func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ error) { if len(indices) == 0 { return nil, nil From d1801ba8586e1ea3e29943bdc32523584c45f1da Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 2 Jan 2025 15:05:20 -0800 Subject: [PATCH 308/630] wallet: revert unnecessary test change --- wallet/wallet_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 13de1fa..ae7d43a 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -58,7 +58,6 @@ func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) n, genesisBlock := chain.TestnetZen() genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} - n.MaturityDelay = 5 n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 n.HardforkStorageProof.Height = 1 @@ -116,10 +115,9 @@ func TestSelectSiacoins(t *testing.T) { } defer bdb.Close() - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + network, genesisBlock := testutil.Network() network.InitialCoinbase = types.Siacoins(100) network.MinimumCoinbase = types.Siacoins(100) - network.MaturityDelay = 5 store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { From 33bad49eda2f05d15ab33785078ec55934d1e4ae Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 11:10:42 -0800 Subject: [PATCH 309/630] wallet: switch locks to global timer --- wallet/manager.go | 151 +++++++++++++++++++++++++----------------- wallet/options.go | 14 +++- wallet/wallet_test.go | 1 - 3 files changed, 103 insertions(+), 63 deletions(-) diff --git a/wallet/manager.go b/wallet/manager.go index c8e51fc..7fe3f46 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log" "strings" "sync" "time" @@ -97,6 +98,7 @@ type ( Manager struct { indexMode IndexMode syncBatchSize int + lockDuration time.Duration chain ChainManager store Store @@ -104,7 +106,7 @@ type ( tg *threadgroup.ThreadGroup mu sync.Mutex // protects the fields below - used map[types.Hash256]bool + used map[types.Hash256]time.Time } ) @@ -142,6 +144,27 @@ func (i IndexMode) MarshalText() ([]byte, error) { return []byte(i.String()), nil } +// lockUTXOs locks the given UTXOs for the duration of the lock duration. +// The lock duration is used to prevent double spending when building transactions. +// It is expected that the caller holds the manager's lock. +func (m *Manager) lockUTXOs(ids ...types.Hash256) { + ts := time.Now().Add(m.lockDuration) + for _, id := range ids { + m.used[id] = ts + } +} + +// utxosLocked returns an error if any of the given UTXOs are locked. +// It is expected that the caller holds the manager's lock. +func (m *Manager) utxosLocked(ids ...types.Hash256) error { + for _, id := range ids { + if m.used[id].After(time.Now()) { + return fmt.Errorf("output %q is locked", id) + } + } + return nil +} + // Tip returns the last scanned chain index of the manager. func (m *Manager) Tip() (types.ChainIndex, error) { return m.store.LastCommittedIndex() @@ -249,26 +272,10 @@ func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { defer m.mu.Unlock() // check if any of the ids are already reserved - for _, id := range ids { - if m.used[id] { - return fmt.Errorf("output %q already reserved", id) - } - } - - // reserve the ids - for _, id := range ids { - m.used[id] = true + if err := m.utxosLocked(ids...); err != nil { + return err } - - // sleep for the duration and then unreserve the ids - time.AfterFunc(duration, func() { - m.mu.Lock() - defer m.mu.Unlock() - - for _, id := range ids { - delete(m.used, id) - } - }) + m.lockUTXOs(ids...) return nil } @@ -290,19 +297,19 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU defer m.mu.Unlock() knownAddresses := make(map[types.Address]bool) - relevantAddr := func(addr types.Address) bool { + relevantAddr := func(addr types.Address) (bool, error) { if exists, ok := knownAddresses[addr]; ok { - return exists + return exists, nil } _, err := m.store.WalletAddress(walletID, addr) if errors.Is(err, ErrNotFound) { knownAddresses[addr] = false - return false + return false, nil } else if err != nil { - panic(err) + return false, err } knownAddresses[addr] = true - return true + return true, nil } ephemeral := make(map[types.SiacoinOutputID]types.SiacoinElement) @@ -313,7 +320,10 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU delete(ephemeral, sci.ParentID) } for i, sco := range txn.SiacoinOutputs { - if relevantAddr(sco.Address) { + exists, err := relevantAddr(sco.Address) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) + } else if exists { scoid := txn.SiacoinOutputID(i) ephemeral[scoid] = types.SiacoinElement{ ID: scoid, @@ -329,7 +339,10 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU delete(ephemeral, sci.Parent.ID) } for i, sco := range txn.SiacoinOutputs { - if relevantAddr(sco.Address) { + exists, err := relevantAddr(sco.Address) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) + } else if exists { sce := txn.EphemeralSiacoinOutput(i) ephemeral[sce.ID] = sce } @@ -343,6 +356,7 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU var inputSum types.Currency var selected []types.SiacoinElement + var utxoIDs []types.Hash256 const utxoBatchSize = 100 top: for i := 0; ; i += utxoBatchSize { @@ -356,11 +370,12 @@ top: } for _, sce := range utxos { - if inPool[sce.ID] || m.used[types.Hash256(sce.ID)] { + if inPool[sce.ID] || m.utxosLocked(types.Hash256(sce.ID)) != nil { continue } selected = append(selected, sce) + utxoIDs = append(utxoIDs, types.Hash256(sce.ID)) inputSum = inputSum.Add(sce.SiacoinOutput.Value) if inputSum.Cmp(amount) >= 0 { break top @@ -374,7 +389,7 @@ top: } for _, sce := range ephemeral { - if inPool[sce.ID] || m.used[types.Hash256(sce.ID)] { + if inPool[sce.ID] || m.utxosLocked(types.Hash256(sce.ID)) != nil { continue } @@ -389,18 +404,7 @@ top: if inputSum.Cmp(amount) < 0 { return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds } - - for _, sce := range selected { - m.used[types.Hash256(sce.ID)] = true - } - time.AfterFunc(expiration, func() { - m.mu.Lock() - defer m.mu.Unlock() - - for _, sce := range selected { - delete(m.used, types.Hash256(sce.ID)) - } - }) + m.lockUTXOs(utxoIDs...) return selected, tip, inputSum.Sub(amount), nil } @@ -418,19 +422,19 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration t } knownAddresses := make(map[types.Address]bool) - relevantAddr := func(addr types.Address) bool { + relevantAddr := func(addr types.Address) (bool, error) { if exists, ok := knownAddresses[addr]; ok { - return exists + return exists, nil } _, err := m.store.WalletAddress(walletID, addr) if errors.Is(err, ErrNotFound) { knownAddresses[addr] = false - return false + return false, nil } else if err != nil { - panic(err) + return false, err } knownAddresses[addr] = true - return true + return true, nil } ephemeral := make(map[types.SiafundOutputID]types.SiafundElement) @@ -441,7 +445,10 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration t delete(ephemeral, sfi.ParentID) } for i, sfo := range txn.SiafundOutputs { - if relevantAddr(sfo.Address) { + exists, err := relevantAddr(sfo.Address) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) + } else if exists { sfoid := txn.SiafundOutputID(i) ephemeral[sfoid] = types.SiafundElement{ ID: sfoid, @@ -457,7 +464,10 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration t delete(ephemeral, sfi.Parent.ID) } for i, sfo := range txn.SiafundOutputs { - if relevantAddr(sfo.Address) { + exists, err := relevantAddr(sfo.Address) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) + } else if exists { sfe := txn.EphemeralSiafundOutput(i) ephemeral[sfe.ID] = sfe } @@ -466,6 +476,7 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration t var inputSum uint64 var selected []types.SiafundElement + var utxoIDs []types.Hash256 top: for i := 0; ; i++ { utxos, err := m.store.WalletSiafundOutputs(walletID, i, 100) @@ -476,11 +487,12 @@ top: } for _, sfe := range utxos { - if inPool[sfe.ID] || m.used[types.Hash256(sfe.ID)] { + if inPool[sfe.ID] || m.utxosLocked(types.Hash256(sfe.ID)) != nil { continue } selected = append(selected, sfe) + utxoIDs = append(utxoIDs, types.Hash256(sfe.ID)) inputSum += sfe.SiafundOutput.Value if inputSum >= amount { break top @@ -492,17 +504,7 @@ top: return nil, types.ChainIndex{}, 0, ErrInsufficientFunds } - for _, sce := range selected { - m.used[types.Hash256(sce.ID)] = true - } - time.AfterFunc(expiration, func() { - m.mu.Lock() - defer m.mu.Unlock() - - for _, sce := range selected { - delete(m.used, types.Hash256(sce.ID)) - } - }) + m.lockUTXOs(utxoIDs...) return selected, tip, inputSum - amount, nil } @@ -574,13 +576,14 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) m := &Manager{ indexMode: IndexModePersonal, syncBatchSize: defaultSyncBatchSize, + lockDuration: time.Hour, chain: cm, store: store, log: zap.NewNop(), tg: threadgroup.New(), - used: make(map[types.Hash256]bool), + used: make(map[types.Hash256]time.Time), } for _, opt := range opts { @@ -605,6 +608,32 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) } }) + go func() { + ctx, cancel, err := m.tg.AddWithContext(context.Background()) + if err != nil { + log.Panic("failed to add to threadgroup", zap.Error(err)) + } + defer cancel() + + t := time.NewTicker(m.lockDuration / 2) + defer t.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + m.mu.Lock() + for id, ts := range m.used { + if ts.Before(time.Now()) { + delete(m.used, id) + } + } + m.mu.Unlock() + } + } + }() + go func() { defer unsubscribe() diff --git a/wallet/options.go b/wallet/options.go index 79075e1..34e19af 100644 --- a/wallet/options.go +++ b/wallet/options.go @@ -1,6 +1,10 @@ package wallet -import "go.uber.org/zap" +import ( + "time" + + "go.uber.org/zap" +) // An Option configures a wallet Manager. type Option func(*Manager) @@ -27,3 +31,11 @@ func WithSyncBatchSize(size int) Option { m.syncBatchSize = size } } + +// WithLockDuration sets the duration that a UTXO is locked after +// being selected as an input to a transaction. The default is 1 hour. +func WithLockDuration(d time.Duration) Option { + return func(m *Manager) { + m.lockDuration = d + } +} diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index ae7d43a..079231c 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -41,7 +41,6 @@ func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) n, genesisBlock := chain.TestnetZen() genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr n.InitialTarget = types.BlockID{0xFF} - n.MaturityDelay = 5 n.HardforkDevAddr.Height = 1 n.HardforkTax.Height = 1 n.HardforkStorageProof.Height = 1 From f8e7cefc52df86b288a1415ad064c43710f22a83 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 11:21:09 -0800 Subject: [PATCH 310/630] api,wallet: remove extra duration param --- api/api.go | 1 - api/client.go | 1 - api/server.go | 12 +++---- wallet/manager.go | 9 ++--- wallet/wallet_test.go | 83 ++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 90 insertions(+), 16 deletions(-) diff --git a/api/api.go b/api/api.go index 0014bd4..fb8f010 100644 --- a/api/api.go +++ b/api/api.go @@ -51,7 +51,6 @@ type BalanceResponse wallet.Balance type WalletReserveRequest struct { SiacoinOutputs []types.SiacoinOutputID `json:"siacoinOutputs"` SiafundOutputs []types.SiafundOutputID `json:"siafundOutputs"` - Duration time.Duration `json:"duration"` } // A WalletUpdateRequest is a request to update a wallet diff --git a/api/client.go b/api/client.go index 776617b..8ff757a 100644 --- a/api/client.go +++ b/api/client.go @@ -300,7 +300,6 @@ func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOut err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.id), WalletReserveRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, - Duration: duration, }, nil) return } diff --git a/api/server.go b/api/server.go index 84b3dc0..984cb86 100644 --- a/api/server.go +++ b/api/server.go @@ -99,8 +99,8 @@ type ( Addresses(id wallet.ID) ([]wallet.Address, error) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) - SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool, expiration time.Duration) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) - SelectSiafundElements(walletID wallet.ID, amount uint64, expiration time.Duration) ([]types.SiafundElement, types.ChainIndex, uint64, error) + SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) + SelectSiafundElements(walletID wallet.ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) WalletBalance(id wallet.ID) (wallet.Balance, error) @@ -116,7 +116,7 @@ type ( SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) - Reserve([]types.Hash256, time.Duration) error + Reserve([]types.Hash256) error Release([]types.Hash256) } ) @@ -571,7 +571,7 @@ func (s *server) walletsReserveHandler(jc jape.Context) { ids = append(ids, types.Hash256(id)) } - if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids, wrr.Duration)) != nil { + if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids)) != nil { return } jc.EmptyResonse() @@ -600,7 +600,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false, time.Hour) + utxos, _, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -640,7 +640,7 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, change, err := s.wm.SelectSiafundElements(id, wfr.Amount, time.Hour) + utxos, _, change, err := s.wm.SelectSiafundElements(id, wfr.Amount) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } diff --git a/wallet/manager.go b/wallet/manager.go index 7fe3f46..a5b5e2f 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -41,6 +41,7 @@ var ( // ErrInsufficientFunds is returned when there are not enough funds to // fund a transaction. ErrInsufficientFunds = errors.New("insufficient funds") + ErrAlreadyReserved = errors.New("output already reserved") ) type ( @@ -159,7 +160,7 @@ func (m *Manager) lockUTXOs(ids ...types.Hash256) { func (m *Manager) utxosLocked(ids ...types.Hash256) error { for _, id := range ids { if m.used[id].After(time.Now()) { - return fmt.Errorf("output %q is locked", id) + return fmt.Errorf("failed to lock output %q: %w", id, ErrAlreadyReserved) } } return nil @@ -267,7 +268,7 @@ func (m *Manager) UnconfirmedEvents() ([]Event, error) { } // Reserve reserves the given ids for the given duration. -func (m *Manager) Reserve(ids []types.Hash256, duration time.Duration) error { +func (m *Manager) Reserve(ids []types.Hash256) error { m.mu.Lock() defer m.mu.Unlock() @@ -292,7 +293,7 @@ func (m *Manager) Release(ids []types.Hash256) { // SelectSiacoinElements selects siacoin elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. The selected elements are locked for the given duration. -func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool, expiration time.Duration) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { +func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { m.mu.Lock() defer m.mu.Unlock() @@ -412,7 +413,7 @@ top: // at least the given amount. Returns the elements, the element basis, and the // change amount. The selected elements are locked for the given // duration. -func (m *Manager) SelectSiafundElements(walletID ID, amount uint64, expiration time.Duration) ([]types.SiafundElement, types.ChainIndex, uint64, error) { +func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { m.mu.Lock() defer m.mu.Unlock() diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 079231c..1d55387 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -22,6 +22,7 @@ import ( "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" + "lukechampine.com/frand" ) func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { @@ -99,6 +100,80 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty return b } +func TestReserve(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testutil.V2Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithLockDuration(2*time.Second)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sk := types.GeneratePrivateKey() + sp := types.SpendPolicy{Type: types.PolicyTypePublicKey(sk.PublicKey())} + addr := sp.Address() + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &sp, + }) + if err != nil { + t.Fatal(err) + } + + scoID := types.Hash256(frand.Entropy256()) + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } + + // output should be locked + if err := wm.Reserve([]types.Hash256{scoID}); !errors.Is(err, wallet.ErrAlreadyReserved) { + t.Fatalf("expected output locked error, got %v", err) + } + + time.Sleep(3 * time.Second) + + // output should be unlocked + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } + + // output should be locked + if err := wm.Reserve([]types.Hash256{scoID}); !errors.Is(err, wallet.ErrAlreadyReserved) { + t.Fatalf("expected output locked error, got %v", err) + } + + wm.Release([]types.Hash256{scoID}) + // output should be unlocked + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } +} + func TestSelectSiacoins(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() @@ -179,7 +254,7 @@ func TestSelectSiacoins(t *testing.T) { } // fund a transaction with more than the wallet balance - _, _, _, err = wm.SelectSiacoinElements(w.ID, balance.Siacoins.Add(types.Siacoins(1)), false, time.Minute) + _, _, _, err = wm.SelectSiacoinElements(w.ID, balance.Siacoins.Add(types.Siacoins(1)), false) if !errors.Is(err, wallet.ErrInsufficientFunds) { t.Fatal("expected insufficient funds error") } @@ -188,7 +263,7 @@ func TestSelectSiacoins(t *testing.T) { var selected []types.Hash256 seen := make(map[types.SiacoinOutputID]bool) for i := 0; i < len(utxos); i++ { - utxos, _, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false, time.Minute) + utxos, _, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { // one UTXO should always be enough to cover @@ -203,7 +278,7 @@ func TestSelectSiacoins(t *testing.T) { } // all available outputs should be locked - _, _, _, err = wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false, time.Minute) + _, _, _, err = wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false) if !errors.Is(err, wallet.ErrInsufficientFunds) { t.Fatal("expected insufficient funds error") } @@ -211,7 +286,7 @@ func TestSelectSiacoins(t *testing.T) { wm.Release(selected) // fund and broadcast a transaction - utxos, basis, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(101), false, time.Minute) // uses two outputs + utxos, basis, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(101), false) // uses two outputs if err != nil { t.Fatal(err) } else if len(utxos) != 2 { From dccf0f8894e70c0da5c3ea724d89dd138035ce73 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 11:22:38 -0800 Subject: [PATCH 311/630] wallet: fix lint --- wallet/manager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wallet/manager.go b/wallet/manager.go index a5b5e2f..4396bfa 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -41,7 +41,9 @@ var ( // ErrInsufficientFunds is returned when there are not enough funds to // fund a transaction. ErrInsufficientFunds = errors.New("insufficient funds") - ErrAlreadyReserved = errors.New("output already reserved") + // ErrAlreadyReserved is returned when trying to reserve an output that is + // already reserved. + ErrAlreadyReserved = errors.New("output already reserved") ) type ( From 627806dc23b4b39a136f2681f4c451cd6605c9b1 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 17:39:27 -0800 Subject: [PATCH 312/630] wallet: fix docstring --- wallet/manager.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wallet/manager.go b/wallet/manager.go index 4396bfa..f22a0ff 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -294,7 +294,7 @@ func (m *Manager) Release(ids []types.Hash256) { // SelectSiacoinElements selects siacoin elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the -// change amount. The selected elements are locked for the given duration. +// change amount. func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { m.mu.Lock() defer m.mu.Unlock() @@ -413,8 +413,7 @@ top: // SelectSiafundElements selects siacoin elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the -// change amount. The selected elements are locked for the given -// duration. +// change amount. func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { m.mu.Lock() defer m.mu.Unlock() From 9da769a7f50164c6f0a776f5852ab360a835b817 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 17:39:56 -0800 Subject: [PATCH 313/630] wallet: fix siafund docstring --- wallet/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/manager.go b/wallet/manager.go index f22a0ff..dbde785 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -411,7 +411,7 @@ top: return selected, tip, inputSum.Sub(amount), nil } -// SelectSiafundElements selects siacoin elements from the wallet that sum to +// SelectSiafundElements selects siafund elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { From be291e3d023f3a39a4e0a5f8a775ded6518e0ee7 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 17:41:44 -0800 Subject: [PATCH 314/630] wallet: fix comment --- wallet/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/manager.go b/wallet/manager.go index dbde785..2495199 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -483,7 +483,7 @@ top: for i := 0; ; i++ { utxos, err := m.store.WalletSiafundOutputs(walletID, i, 100) if err != nil { - return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to get siacoin elements: %w", err) + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to get siafund elements: %w", err) } else if len(utxos) == 0 { return nil, types.ChainIndex{}, 0, ErrInsufficientFunds } From 938b1fb829514329d2e81e2b3f244b8d2db491e9 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 8 Jan 2025 19:26:38 -0800 Subject: [PATCH 315/630] api,sqlite,wallet: address review comments --- api/server.go | 8 +- persist/sqlite/consensus_test.go | 4 +- persist/sqlite/wallet.go | 49 ++++++--- wallet/manager.go | 50 +++++---- wallet/wallet_test.go | 177 +++++++++++++++++++++++++++---- 5 files changed, 227 insertions(+), 61 deletions(-) diff --git a/api/server.go b/api/server.go index 984cb86..e247921 100644 --- a/api/server.go +++ b/api/server.go @@ -101,8 +101,8 @@ type ( WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) SelectSiafundElements(walletID wallet.ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) - UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, error) - UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, error) + UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) WalletBalance(id wallet.ID) (wallet.Balance, error) AddressBalance(address types.Address) (wallet.Balance, error) @@ -530,7 +530,7 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { return } - scos, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) + scos, _, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } @@ -549,7 +549,7 @@ func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { return } - sfos, err := s.wm.UnspentSiafundOutputs(id, offset, limit) + sfos, _, err := s.wm.UnspentSiafundOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 107bace..8480192 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -137,7 +137,7 @@ func TestPruneSiacoins(t *testing.T) { assertUTXOs(0, 1) // spend the utxo - utxos, err := db.WalletSiacoinOutputs(w.ID, cm.Tip(), 0, 100) + utxos, _, err := db.WalletSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatalf("failed to get wallet siacoin outputs: %v", err) } @@ -262,7 +262,7 @@ func TestPruneSiafunds(t *testing.T) { assertUTXOs(0, 1) // spend the utxo - utxos, err := db.WalletSiafundOutputs(w.ID, 0, 100) + utxos, _, err := db.WalletSiafundOutputs(w.ID, 0, 100) if err != nil { t.Fatalf("failed to get wallet siacoin outputs: %v", err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 4400558..6176e96 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -225,19 +225,24 @@ WHERE wa.wallet_id=$1` } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(id wallet.ID, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err } + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.spent_index_id IS NULL AND se.maturity_height <= $1 AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$2) LIMIT $3 OFFSET $4` - rows, err := tx.Query(query, index.Height, id, limit, offset) + rows, err := tx.Query(query, basis.Height, id, limit, offset) if err != nil { return err } @@ -276,13 +281,18 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, index types.ChainIndex, offse } // WalletSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []types.SiafundElement, err error) { +func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err } - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) @@ -419,7 +429,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti return se, nil } - siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) @@ -605,8 +615,8 @@ func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { } func insertAddress(tx *txn, addr types.Address) (id int64, err error) { - const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) -VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address + const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) +VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id` err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency)).Scan(&id) @@ -639,6 +649,11 @@ func scanWalletAddress(s scanner) (wallet.Address, error) { return address, nil } +func getScanBasis(tx *txn) (index types.ChainIndex, err error) { + err = tx.QueryRow(`SELECT last_indexed_id, last_indexed_height FROM global_settings`).Scan(decode(&index.ID), &index.Height) + return +} + func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ error) { if len(indices) == 0 { return nil, nil @@ -704,7 +719,7 @@ WITH last_chain_index AS ( SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 ), event_ids AS ( - SELECT + SELECT ev.id FROM events ev INNER JOIN event_addresses ea ON ev.id = ea.event_id @@ -714,18 +729,18 @@ event_ids AS ( ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3 ) -SELECT - ev.id, - ev.event_id, - ev.maturity_height, - ev.date_created, - ci.height, - ci.block_id, - CASE +SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + CASE WHEN last_chain_index.height < ci.height THEN 0 ELSE last_chain_index.height - ci.height END AS confirmations, - ev.event_type, + ev.event_type, ev.event_data FROM events ev INNER JOIN event_ids ei ON ev.id = ei.id diff --git a/wallet/manager.go b/wallet/manager.go index 2495199..753d661 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -74,8 +74,8 @@ type ( DeleteWallet(walletID ID) error WalletBalance(walletID ID) (Balance, error) WalletAddress(ID, types.Address) (Address, error) - WalletSiacoinOutputs(walletID ID, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, error) - WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) + WalletSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) @@ -215,13 +215,13 @@ func (m *Manager) WalletEvents(walletID ID, offset, limit int) ([]Event, error) // UnspentSiacoinOutputs returns a paginated list of matured siacoin outputs // relevant to the wallet -func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, error) { - return m.store.WalletSiacoinOutputs(walletID, m.chain.Tip(), offset, limit) +func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + return m.store.WalletSiacoinOutputs(walletID, offset, limit) } // UnspentSiafundOutputs returns a paginated list of siafund outputs relevant to // the wallet -func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, error) { +func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { return m.store.WalletSiafundOutputs(walletID, offset, limit) } @@ -296,6 +296,11 @@ func (m *Manager) Release(ids []types.Hash256) { // at least the given amount. Returns the elements, the element basis, and the // change amount. func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { + // sanity check that the wallet exists + if _, err := m.WalletBalance(walletID); err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, err + } + m.mu.Lock() defer m.mu.Unlock() @@ -352,24 +357,22 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU } } - tip := m.chain.Tip() - if amount.IsZero() { - return nil, tip, types.ZeroCurrency, nil - } - var inputSum types.Currency var selected []types.SiacoinElement var utxoIDs []types.Hash256 + var basis types.ChainIndex const utxoBatchSize = 100 top: for i := 0; ; i += utxoBatchSize { + var utxos []types.SiacoinElement + var err error // extra large wallets may need to paginate through utxos // to find enough to cover the amount - utxos, err := m.store.WalletSiacoinOutputs(walletID, tip, i, utxoBatchSize) + utxos, basis, err = m.store.WalletSiacoinOutputs(walletID, i, utxoBatchSize) if err != nil { return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to get siacoin elements: %w", err) } else if len(utxos) == 0 { - return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + break top } for _, sce := range utxos { @@ -408,19 +411,23 @@ top: return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds } m.lockUTXOs(utxoIDs...) - return selected, tip, inputSum.Sub(amount), nil + return selected, basis, inputSum.Sub(amount), nil } // SelectSiafundElements selects siafund elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { + // sanity check that the wallet exists + if _, err := m.WalletBalance(walletID); err != nil { + return nil, types.ChainIndex{}, 0, err + } + m.mu.Lock() defer m.mu.Unlock() - tip := m.chain.Tip() if amount == 0 { - return nil, tip, 0, nil + return nil, m.chain.Tip(), 0, nil } knownAddresses := make(map[types.Address]bool) @@ -479,13 +486,18 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.Sia var inputSum uint64 var selected []types.SiafundElement var utxoIDs []types.Hash256 + var basis types.ChainIndex + const utxoBatchSize = 100 top: - for i := 0; ; i++ { - utxos, err := m.store.WalletSiafundOutputs(walletID, i, 100) + for i := 0; ; i += utxoBatchSize { + var utxos []types.SiafundElement + var err error + + utxos, basis, err = m.store.WalletSiafundOutputs(walletID, i, utxoBatchSize) if err != nil { return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to get siafund elements: %w", err) } else if len(utxos) == 0 { - return nil, types.ChainIndex{}, 0, ErrInsufficientFunds + break top } for _, sfe := range utxos { @@ -507,7 +519,7 @@ top: } m.lockUTXOs(utxoIDs...) - return selected, tip, inputSum - amount, nil + return selected, basis, inputSum - amount, nil } // Scan rescans the chain starting from the given index. The scan will complete diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 1d55387..142c27c 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -241,7 +241,7 @@ func TestSelectSiacoins(t *testing.T) { mineAndSync(t, types.VoidAddress, int(cm.TipState().Network.MaturityDelay)) // check that the wallet has 200 matured outputs - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 1000) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 1000) if err != nil { t.Fatal(err) } else if len(utxos) != 200 { @@ -341,6 +341,145 @@ func TestSelectSiacoins(t *testing.T) { } } +func TestSelectSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + sk := types.GeneratePrivateKey() + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{sk.PublicKey().UnlockKey()}, + SignaturesRequired: 1, + } + addr := uc.UnlockHash() + + network, genesisBlock := testutil.Network() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + network.InitialCoinbase = types.Siacoins(100) + network.MinimumCoinbase = types.Siacoins(100) + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + mineAndSync := func(t *testing.T, addr types.Address, n int) { + t.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(t, cm, addr, 1) + waitForBlock(t, cm, db) + } + } + mineAndSync(t, types.VoidAddress, 1) + + // check that the wallet has a siafund utxo + utxos, _, err := wm.UnspentSiafundOutputs(w.ID, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 outputs, got %v", len(utxos)) + } + + balance, err := wm.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } + + // fund a transaction with more than the wallet balance + _, _, _, err = wm.SelectSiafundElements(w.ID, balance.Siafunds+1) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + + // fund and broadcast a transaction + utxos, basis, change, err := wm.SelectSiafundElements(w.ID, balance.Siafunds/2) // uses two outputs + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 utxo, got %v", len(utxos)) + } else if change != balance.Siafunds/2 { + t.Fatalf("expected %v SF change, got %v", balance.Siafunds/2, change) + } else if basis != cm.Tip() { + t.Fatalf("expected tip, got %v", basis) + } + txn := types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: types.VoidAddress, Value: balance.Siafunds / 2}, + {Address: addr, Value: change}, + }, + } + for _, utxo := range utxos { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: types.SiafundOutputID(utxo.ID), + UnlockConditions: uc, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxo.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + for i := range txn.Signatures { + sigHash := cm.TipState().WholeSigHash(txn, txn.Signatures[i].ParentID, 0, 0, nil) + sig := sk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + + known, err := cm.AddPoolTransactions([]types.Transaction{txn}) + if err != nil { + t.Fatal(err) + } else if known { + t.Fatal("transaction was already known") + } + + mineAndSync(t, types.VoidAddress, 1) + + events, err := wm.WalletEvents(w.ID, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if events[0].SiafundOutflow()-events[0].SiafundInflow() != balance.Siafunds/2 { + t.Fatalf("expected transaction value %v SF, got %v", balance.Siafunds/2, events[0].SiafundOutflow()-events[0].SiafundInflow()) + } +} + func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) @@ -420,7 +559,7 @@ func TestReorg(t *testing.T) { } // check that the utxo has not matured - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -455,7 +594,7 @@ func TestReorg(t *testing.T) { } // check that the utxo was removed - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -486,7 +625,7 @@ func TestReorg(t *testing.T) { } // check that the utxo has not matured - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -529,7 +668,7 @@ func TestReorg(t *testing.T) { } // check that only the single utxo still exists - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -630,7 +769,7 @@ func TestEphemeralBalance(t *testing.T) { waitForBlock(t, cm, db) // create a transaction that spends the matured payout - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -1286,7 +1425,7 @@ func TestOrphans(t *testing.T) { } // check that the utxo was created - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -1387,7 +1526,7 @@ func TestOrphans(t *testing.T) { } // check that the utxo was reverted - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -1944,7 +2083,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } } - utxos, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -2152,7 +2291,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } } - utxos, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -2387,7 +2526,7 @@ func TestV2(t *testing.T) { waitForBlock(t, cm, db) // create a v2 transaction that spends the matured payout - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } @@ -2697,7 +2836,7 @@ func TestReorgV2(t *testing.T) { } // check that the utxo has not matured - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -2732,7 +2871,7 @@ func TestReorgV2(t *testing.T) { } // check that the utxo was removed - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -2763,7 +2902,7 @@ func TestReorgV2(t *testing.T) { } // check that the utxo has not matured - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -2806,7 +2945,7 @@ func TestReorgV2(t *testing.T) { } // check that only the single utxo still exists - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -2844,7 +2983,7 @@ func TestReorgV2(t *testing.T) { } // check that all UTXOs have been spent - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { @@ -2932,7 +3071,7 @@ func TestOrphansV2(t *testing.T) { } // check that the utxo was created - utxos, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -3024,7 +3163,7 @@ func TestOrphansV2(t *testing.T) { } // check that the utxo was reverted - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 1 { @@ -3058,7 +3197,7 @@ func TestOrphansV2(t *testing.T) { } // check that all UTXOs have been spent - utxos, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } else if len(utxos) != 0 { From 65735111e166dd44d11f537725902f26fcba1ba7 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 8 Jan 2025 19:32:24 -0800 Subject: [PATCH 316/630] wallet: fix test NDF --- wallet/wallet_test.go | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 142c27c..c0f44dc 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -37,6 +37,15 @@ func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { tb.Fatal("timed out waiting for block") } +func mineAndSync(tb testing.TB, cm *chain.Manager, ws wallet.Store, addr types.Address, n int) { + tb.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(tb, cm, addr, 1) + waitForBlock(tb, cm, ws) + } +} + func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { // use a modified version of Zen n, genesisBlock := chain.TestnetZen() @@ -2273,23 +2282,9 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // mine a block sending the payout to the wallet - b, ok := coreutils.MineBlock(cm, addr1, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - + mineAndSync(t, cm, db, addr1, 1) // mine until the payout matures - maturityHeight := cm.TipState().MaturityHeight() - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - } + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { @@ -2429,12 +2424,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // mine the transactions - b, ok = coreutils.MineBlock(cm, types.VoidAddress, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } + mineAndSync(t, cm, db, types.VoidAddress, 1) // check that the unconfirmed events were removed events, err = wm.AddressUnconfirmedEvents(addr1) From f0819d56e25d9b29e843f0be9262d0d87521c72d Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 8 Jan 2025 19:38:30 -0800 Subject: [PATCH 317/630] wallet: fix test NDF --- wallet/wallet_test.go | 46 ++++++++----------------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index c0f44dc..dbfb663 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -2074,23 +2074,8 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // mine a block sending the payout to the wallet - b, ok := coreutils.MineBlock(cm, addr1, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - - // mine until the payout matures - maturityHeight := cm.TipState().MaturityHeight() - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - } + mineAndSync(t, cm, db, addr1, 1) + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { @@ -2221,12 +2206,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // mine the transactions - b, ok = coreutils.MineBlock(cm, types.VoidAddress, time.Minute) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } + mineAndSync(t, cm, db, types.VoidAddress, 1) // check that the unconfirmed events were removed events, err = wm.WalletUnconfirmedEvents(w1.ID) @@ -2482,11 +2462,7 @@ func TestV2(t *testing.T) { } expectedPayout := cm.TipState().BlockReward() - // mine a block sending the payout to the wallet - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, db) + mineAndSync(t, cm, db, addr, 1) // check that the payout was received balance, err := db.AddressBalance(addr) @@ -2507,16 +2483,10 @@ func TestV2(t *testing.T) { } // mine until the payout matures - maturityHeight := cm.TipState().MaturityHeight() + 1 - for i := cm.TipState().Index.Height; i < maturityHeight; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm, db) + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) // create a v2 transaction that spends the matured payout - utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + utxos, basis, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) if err != nil { t.Fatal(err) } @@ -2537,10 +2507,10 @@ func TestV2(t *testing.T) { } txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} - if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - waitForBlock(t, cm, db) + mineAndSync(t, cm, db, types.VoidAddress, 1) // check that the change was received balance, err = wm.AddressBalance(addr) From cf4ad0aa8651e1a2e916855232a82e6298cefd94 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 8 Jan 2025 14:28:25 -0800 Subject: [PATCH 318/630] cmd: changes startup errors to log to stderr for easier parsing in scripts --- cmd/walletd/config.go | 34 ++++++++++------------------------ cmd/walletd/main.go | 43 ++++++++++++++----------------------------- cmd/walletd/miner.go | 6 +++--- 3 files changed, 27 insertions(+), 56 deletions(-) diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go index c1ae246..5c41067 100644 --- a/cmd/walletd/config.go +++ b/cmd/walletd/config.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "errors" "fmt" "net" "os" @@ -18,9 +19,7 @@ import ( func readPasswordInput(context string) string { fmt.Printf("%s: ", context) input, err := term.ReadPassword(int(os.Stdin.Fd())) - if err != nil { - fatalError(fmt.Errorf("could not read password input: %w", err)) - } + checkFatalError("failed to read password input", err) fmt.Println("") return string(input) } @@ -29,9 +28,7 @@ func readInput(context string) string { fmt.Printf("%s: ", context) r := bufio.NewReader(os.Stdin) input, err := r.ReadString('\n') - if err != nil { - fatalError(fmt.Errorf("could not read input: %w", err)) - } + checkFatalError("failed to read input", err) return strings.TrimSpace(input) } @@ -115,9 +112,7 @@ func setDataDirectory() { } dir, err := filepath.Abs(cfg.Directory) - if err != nil { - fatalError(fmt.Errorf("failed to get absolute path of data directory: %w", err)) - } + checkFatalError("failed to get absolute path of data directory", err) fmt.Println("The data directory is where walletd will store its metadata and consensus data.") fmt.Println("This directory should be on a fast, reliable storage device, preferably an SSD.") @@ -200,7 +195,7 @@ func setAdvancedConfig() { case strings.EqualFold(mode, "full"): cfg.Index.Mode = wallet.IndexModeFull default: - fatalError(fmt.Errorf("invalid index mode: %q", mode)) + checkFatalError("invalid index mode", errors.New("must be either 'personal' or 'full'")) } fmt.Println("") @@ -236,21 +231,12 @@ func buildConfig() { // write the config file f, err := os.Create(configPath) - if err != nil { - fatalError(fmt.Errorf("failed to create config file: %w", err)) - return - } + checkFatalError("failed to create config file", err) defer f.Close() enc := yaml.NewEncoder(f) - if err := enc.Encode(cfg); err != nil { - fatalError(fmt.Errorf("failed to encode config file: %w", err)) - return - } else if err := f.Sync(); err != nil { - fatalError(fmt.Errorf("failed to sync config file: %w", err)) - return - } else if err := f.Close(); err != nil { - fatalError(fmt.Errorf("failed to close config file: %w", err)) - return - } + defer enc.Close() + + checkFatalError("failed to encode config file", enc.Encode(cfg)) + checkFatalError("failed to sync config file", f.Sync()) } diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 536844c..b33ad03 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -85,12 +85,6 @@ var cfg = config.Config{ }, } -func check(context string, err error) { - if err != nil { - log.Fatalf("%v: %v", context, err) - } -} - func mustSetAPIPassword() { if cfg.HTTP.Password != "" { return @@ -111,8 +105,12 @@ func mustSetAPIPassword() { } } -func fatalError(err error) { - os.Stderr.WriteString(err.Error() + "\n") +// checkFatalError prints an error message to stderr and exits with a 1 exit code. If err is nil, this is a no-op. +func checkFatalError(context string, err error) { + if err == nil { + return + } + os.Stderr.WriteString(fmt.Sprintf("%s: %s\n", context, err)) os.Exit(1) } @@ -130,19 +128,13 @@ func tryLoadConfig() { } f, err := os.Open(configPath) - if err != nil { - fatalError(fmt.Errorf("failed to open config file: %w", err)) - return - } + checkFatalError("failed to open config file", err) defer f.Close() dec := yaml.NewDecoder(f) dec.KnownFields(true) - if err := dec.Decode(&cfg); err != nil { - fmt.Println("failed to decode config file:", err) - os.Exit(1) - } + checkFatalError("failed to decode config file", dec.Decode(&cfg)) } // jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for @@ -243,12 +235,14 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) defer cancel() - if err := os.MkdirAll(cfg.Directory, 0700); err != nil { - fatalError(fmt.Errorf("failed to create data directory: %w", err)) + if cfg.Directory != "" { + checkFatalError("failed to create data directory", os.MkdirAll(cfg.Directory, 0700)) } mustSetAPIPassword() + checkFatalError("failed to parse index mode", cfg.Index.Mode.UnmarshalText([]byte(indexModeStr))) + var logCores []zapcore.Core if cfg.Log.StdOut.Enabled { // if no log level is set for stdout, use the global log level @@ -290,10 +284,7 @@ func main() { } fileWriter, closeFn, err := zap.Open(cfg.Log.File.Path) - if err != nil { - fatalError(fmt.Errorf("failed to open log file: %w", err)) - return - } + checkFatalError("failed to open log file", err) defer closeFn() // create the file logger @@ -312,13 +303,7 @@ func main() { // redirect stdlib log to zap zap.RedirectStdLog(log.Named("stdlib")) - if err := cfg.Index.Mode.UnmarshalText([]byte(indexModeStr)); err != nil { - log.Fatal("failed to parse index mode", zap.Error(err)) - } - - if err := runNode(ctx, cfg, log, enableDebug); err != nil { - log.Fatal("failed to run node", zap.Error(err)) - } + checkFatalError("failed to run node", runNode(ctx, cfg, log, enableDebug)) case versionCmd: if len(cmd.Args()) != 0 { cmd.Usage() diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index a531fcd..05be576 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -23,13 +23,13 @@ func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { } elapsed := time.Since(start) cs, err := c.ConsensusTipState() - check("Couldn't get consensus tip state:", err) + checkFatalError("failed to get consensus tip state:", err) d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) d.Mul(d, big.NewInt(int64(1+elapsed))) fmt.Printf("\rMining block %4v...(%.2f blocks/day), difficulty %v)", cs.Index.Height+1, float64(blocksFound)*float64(24*time.Hour)/float64(elapsed), cs.Difficulty) txns, v2txns, err := c.TxpoolTransactions() - check("Couldn't get txpool transactions:", err) + checkFatalError("failed to get pool transactions:", err) b := types.Block{ ParentID: cs.Index.ID, Nonce: cs.NonceFactor() * frand.Uint64n(100), @@ -56,7 +56,7 @@ func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { blocksFound++ index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} tip, err := c.ConsensusTip() - check("Couldn't get consensus tip:", err) + checkFatalError("failed to get consensus tip:", err) if tip != cs.Index { fmt.Printf("\nMined %v but tip changed, starting over\n", index) } else if err := c.SyncerBroadcastBlock(b); err != nil { From 10faf1764ab7e82041273a61886ccc031705325a Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 8 Jan 2025 14:29:20 -0800 Subject: [PATCH 319/630] document change --- .changeset/log_startup_errors_to_stderr.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/log_startup_errors_to_stderr.md diff --git a/.changeset/log_startup_errors_to_stderr.md b/.changeset/log_startup_errors_to_stderr.md new file mode 100644 index 0000000..347e743 --- /dev/null +++ b/.changeset/log_startup_errors_to_stderr.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Log startup errors to stderr From fdca6b2cb1a9553b86c417372bc35b8a8d56c4c2 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 9 Jan 2025 07:25:16 -0800 Subject: [PATCH 320/630] cmd: remove all log.Fatal calls in main.go --- cmd/walletd/main.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index b33ad03..3fcbfd9 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "log" "os" "os/signal" "path/filepath" @@ -319,9 +318,7 @@ func main() { } recoveryPhrase := cwallet.NewSeedPhrase() var seed [32]byte - if err := cwallet.SeedFromPhrase(&seed, recoveryPhrase); err != nil { - log.Fatal(err) - } + checkFatalError("failed to parse mnemonic phrase", cwallet.SeedFromPhrase(&seed, recoveryPhrase)) addr := types.StandardUnlockHash(cwallet.KeyFromSeed(&seed, 0).PublicKey()) fmt.Println("Recovery Phrase:", recoveryPhrase) @@ -340,10 +337,7 @@ func main() { } minerAddr, err := types.ParseAddress(minerAddrStr) - if err != nil { - log.Fatal(err) - } - + checkFatalError("failed to parse miner address", err) mustSetAPIPassword() c := api.NewClient("http://"+cfg.HTTP.Address+"/api", cfg.HTTP.Password) runCPUMiner(c, minerAddr, minerBlocks) From f1226092e914ac8153942b89f40eb45f84338303 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jan 2025 20:29:33 -0800 Subject: [PATCH 321/630] api,wallet: add transaction construction endpoints --- .../add_transaction_construction_api.md | 15 + api/api.go | 31 + api/api_test.go | 614 +++++++++++++++++- api/client.go | 32 +- api/server.go | 412 +++++++++++- cmd/walletd/miner.go | 2 +- wallet/manager.go | 5 + 7 files changed, 1083 insertions(+), 28 deletions(-) create mode 100644 .changeset/add_transaction_construction_api.md diff --git a/.changeset/add_transaction_construction_api.md b/.changeset/add_transaction_construction_api.md new file mode 100644 index 0000000..18f82df --- /dev/null +++ b/.changeset/add_transaction_construction_api.md @@ -0,0 +1,15 @@ +--- +default: minor +--- + +# Add transaction construction API + +Added two new endpoints to construct transactions with a few restrictions for ease of use. Clients can still use the existing fund endpoints for "advanced" transactions. All addresses in the wallet must all have either unlock conditions with a single required signature or a public key spend policy. + +This is a two step process. The private keys are never transmitted to the server. + +The client first calls `[POST] /api/:wallet/transaction/construct` with the recipients. The server will construct the transaction and return a list of hashes that the client needs to sign to broadcast the transaction. The client needs to match the returned public keys to their ed25519 private key and sign all of the hashes. + +After signing, the client calls `[POST] /api/:wallet/transaction/construct/:id` with the array of signatures. The server will add the signatures to the transaction and broadcast it. If the client provided the correct signatures, the transaction will be added to the tpool and broadcast. + +See API docs for request and response bodies \ No newline at end of file diff --git a/api/api.go b/api/api.go index fb8f010..407ff36 100644 --- a/api/api.go +++ b/api/api.go @@ -34,12 +34,14 @@ type GatewayPeer struct { // TxpoolBroadcastRequest is the request type for /txpool/broadcast. type TxpoolBroadcastRequest struct { + Basis types.ChainIndex `json:"basis"` Transactions []types.Transaction `json:"transactions"` V2Transactions []types.V2Transaction `json:"v2transactions"` } // TxpoolTransactionsResponse is the response type for /txpool/transactions. type TxpoolTransactionsResponse struct { + Basis types.ChainIndex `json:"basis"` Transactions []types.Transaction `json:"transactions"` V2Transactions []types.V2Transaction `json:"v2transactions"` } @@ -88,6 +90,35 @@ type WalletFundResponse struct { DependsOn []types.Transaction `json:"dependsOn"` } +// WalletConstructRequest is the request type for /wallets/:id/construct. +type WalletConstructRequest struct { + Siacoins []types.SiacoinOutput `json:"siacoins"` + Siafunds []types.SiafundOutput `json:"siafunds"` + ChangeAddress types.Address `json:"changeAddress"` +} + +// SignaturePayload is a signature that is required to finalize a transaction. +type SignaturePayload struct { + PublicKey types.PublicKey `json:"publicKey"` + SigHash types.Hash256 `json:"sigHash"` +} + +// WalletConstructResponse is the response type for /wallets/:id/construct/transaction. +type WalletConstructResponse struct { + Basis types.ChainIndex `json:"basis"` + ID types.TransactionID `json:"id"` + Transaction types.Transaction `json:"transaction"` + EstimatedFee types.Currency `json:"estimatedFee"` +} + +// WalletConstructV2Response is the response type for /wallets/:id/construct/v2/transaction. +type WalletConstructV2Response struct { + Basis types.ChainIndex `json:"basis"` + ID types.TransactionID `json:"id"` + Transaction types.V2Transaction `json:"transaction"` + EstimatedFee types.Currency `json:"estimatedFee"` +} + // SeedSignRequest requests that a transaction be signed using the keys derived // from the given indices. type SeedSignRequest struct { diff --git a/api/api_test.go b/api/api_test.go index fbf5039..4633620 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -367,7 +367,7 @@ func TestWallet(t *testing.T) { txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { + if err := c.TxpoolBroadcast(cm.Tip(), []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } @@ -1018,7 +1018,7 @@ func TestP2P(t *testing.T) { return err } - txns, v2txns, err := c.TxpoolTransactions() + _, txns, v2txns, err := c.TxpoolTransactions() if err != nil { return err } @@ -1114,7 +1114,7 @@ func TestP2P(t *testing.T) { } sig := key.SignHash(cs.WholeSigHash(txn, types.Hash256(sce.ID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] - if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { + if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { return err } else if err := addBlock(); err != nil { return err @@ -1165,7 +1165,7 @@ func TestP2P(t *testing.T) { return err } txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cs.InputSigHash(txn))} - if err := c.TxpoolBroadcast(nil, []types.V2Transaction{txn}); err != nil { + if err := c.TxpoolBroadcast(cs.Index, nil, []types.V2Transaction{txn}); err != nil { return err } else if err := addBlock(); err != nil { return err @@ -1286,6 +1286,612 @@ func TestConsensusUpdates(t *testing.T) { } } +func TestConstructSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create the syncer + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + resp, err := wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if err != nil { + t.Fatal(err) + } + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(1)): + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiacoinOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[1].Address) + case !resp.Transaction.SiacoinOutputs[1].Value.Equals(types.Siacoins(99).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[1].Value) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + for i, sig := range resp.Transaction.Signatures { + sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.Signatures[i].Signature = sig[:] + } + + if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + expectedValue := types.Siacoins(1).Add(resp.EstimatedFee) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected confirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected confirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected confirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } +} + +func TestConstructSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create the syncer + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + resp, err := wc.Construct(nil, []types.SiafundOutput{ + {Value: 1, Address: receiverAddr}, + }, senderAddr) + if err != nil { + t.Fatal(err) + } + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(100).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiafundOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiafundOutputs[0].Address) + case resp.Transaction.SiafundOutputs[0].Value != 1: + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiafundOutputs[0].Value) + case resp.Transaction.SiafundOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiafundOutputs[1].Address) + case resp.Transaction.SiafundOutputs[1].Value != 9999: + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiafundOutputs[1].Value) + case resp.Transaction.SiafundInputs[0].ClaimAddress != senderAddr: + t.Fatalf("expected transaction to have siafund input claim address %q, got %q", senderAddr, resp.Transaction.SiafundInputs[0].ClaimAddress) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + for i, sig := range resp.Transaction.Signatures { + sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.Signatures[i].Signature = sig[:] + } + + if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 3 { + t.Fatalf("expected 3 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim + } + sent = confirmed[1] // confirmed[0] is the siafund claim + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } +} + +func TestConstructV2Siacoins(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create the syncer + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(1)): + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiacoinOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[1].Address) + case !resp.Transaction.SiacoinOutputs[1].Value.Equals(types.Siacoins(99).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[1].Value) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i := range resp.Transaction.SiacoinInputs { + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + expectedValue := types.Siacoins(1).Add(resp.EstimatedFee) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected confirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected confirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected confirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } +} + +func TestConstructV2Siafunds(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create the syncer + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + resp, err := wc.ConstructV2(nil, []types.SiafundOutput{ + {Value: 1, Address: receiverAddr}, + }, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + sig := senderPrivateKey.SignHash(sigHash) + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 3 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim + } + sent = confirmed[1] // confirmed[0] is the siafund claim + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } +} + func TestDebugMine(t *testing.T) { log := zaptest.NewLogger(t) n, genesisBlock := testNetwork() diff --git a/api/client.go b/api/client.go index 8ff757a..0b55a84 100644 --- a/api/client.go +++ b/api/client.go @@ -46,16 +46,20 @@ func (c *Client) State() (resp StateResponse, err error) { } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(txns []types.Transaction, v2txns []types.V2Transaction) (err error) { - err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) +func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (err error) { + err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{ + Basis: basis, + Transactions: txns, + V2Transactions: v2txns, + }, nil) return } // TxpoolTransactions returns all transactions in the transaction pool. -func (c *Client) TxpoolTransactions() (txns []types.Transaction, v2txns []types.V2Transaction, err error) { +func (c *Client) TxpoolTransactions() (basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction, err error) { var resp TxpoolTransactionsResponse err = c.c.GET("/txpool/transactions", &resp) - return resp.Transactions, resp.V2Transactions, err + return resp.Basis, resp.Transactions, resp.V2Transactions, err } // TxpoolParents returns the parents of a transaction that are currently in the @@ -334,6 +338,26 @@ func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, return } +// Construct constructs a transaction and returns its ID +func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructResponse, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/transaction", c.id), WalletConstructRequest{ + Siacoins: siacoins, + Siafunds: siafunds, + ChangeAddress: change, + }, &resp) + return +} + +// ConstructV2 constructs a v2 transaction and returns its ID +func (c *WalletClient) ConstructV2(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructV2Response, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ + Siacoins: siacoins, + Siafunds: siafunds, + ChangeAddress: change, + }, &resp) + return +} + // NewClient returns a client that communicates with a walletd server listening // on the specified address. func NewClient(addr, password string) *Client { diff --git a/api/server.go b/api/server.go index e247921..17c4bd0 100644 --- a/api/server.go +++ b/api/server.go @@ -69,6 +69,7 @@ type ( AddPoolTransactions(txns []types.Transaction) (bool, error) AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) (bool, error) UnconfirmedParents(txn types.Transaction) []types.Transaction + UpdateV2TransactionSet(txns []types.V2Transaction, from types.ChainIndex, to types.ChainIndex) ([]types.V2Transaction, error) } // A Syncer can connect to other peers and synchronize the blockchain. @@ -97,6 +98,7 @@ type ( AddAddress(id wallet.ID, addr wallet.Address) error RemoveAddress(id wallet.ID, addr types.Address) error Addresses(id wallet.ID) ([]wallet.Address, error) + WalletAddress(wallet.ID, types.Address) (wallet.Address, error) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) @@ -273,6 +275,7 @@ func (s *server) txpoolParentsHandler(jc jape.Context) { func (s *server) txpoolTransactionsHandler(jc jape.Context) { jc.Encode(TxpoolTransactionsResponse{ + Basis: s.cm.Tip(), Transactions: s.cm.PoolTransactions(), V2Transactions: s.cm.V2PoolTransactions(), }) @@ -296,13 +299,12 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { s.s.BroadcastTransactionSet(tbr.Transactions) } if len(tbr.V2Transactions) != 0 { - index := s.cm.TipState().Index - _, err := s.cm.AddV2PoolTransactions(index, tbr.V2Transactions) + _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions) if err != nil { jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return } - s.s.BroadcastV2TransactionSet(index, tbr.V2Transactions) + s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions) } jc.EmptyResonse() @@ -674,6 +676,376 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { }) } +func (s *server) walletsConstructHandler(jc jape.Context) { + cs := s.cm.TipState() + if cs.Index.Height >= cs.Network.HardforkV2.RequireHeight { + jc.Error(errors.New("v1 transactions are not allowed after the v2 require height"), http.StatusBadRequest) + } + + var walletID wallet.ID + if err := jc.DecodeParam("id", &walletID); err != nil { + return + } + var wcr WalletConstructRequest + if err := jc.Decode(&wcr); err != nil { + return + } + + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + var siacoinInput types.Currency + for i, sco := range wcr.Siacoins { + switch { + case sco.Value.IsZero(): + jc.Error(fmt.Errorf("siacoin output %d has zero value", i), http.StatusBadRequest) + return + case sco.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siacoin output %d has void address", i), http.StatusBadRequest) + return + } + siacoinInput = siacoinInput.Add(sco.Value) + } + + var siafundInput uint64 + for i, sfo := range wcr.Siafunds { + switch { + case sfo.Value == 0: + jc.Error(fmt.Errorf("siafund output %d has zero value", i), http.StatusBadRequest) + return + case sfo.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siafund output %d has void address", i), http.StatusBadRequest) + return + } + siafundInput += sfo.Value + } + + if siacoinInput.IsZero() && siafundInput == 0 { + jc.Error(errors.New("no inputs provided"), http.StatusBadRequest) + } + + fee := s.cm.RecommendedFee().Mul64(2000) // use a const for simplicity + + var sent bool + var locked []types.Hash256 + defer func() { + if sent { + return + } + s.wm.Release(locked) + }() + + sces, basis, siacoinChange, err := s.wm.SelectSiacoinElements(walletID, siacoinInput.Add(fee), false) + if err != nil { + jc.Error(fmt.Errorf("failed to select siacoin elements: %w", err), http.StatusInternalServerError) + return + } + for _, sce := range sces { + locked = append(locked, types.Hash256(sce.ID)) + } + + if !siacoinChange.IsZero() { + wcr.Siacoins = append(wcr.Siacoins, types.SiacoinOutput{ + Value: siacoinChange, + Address: wcr.ChangeAddress, + }) + } + + sfes, _, siafundChange, err := s.wm.SelectSiafundElements(walletID, siafundInput) + if err != nil { + jc.Error(fmt.Errorf("failed to select siafund elements: %w", err), http.StatusInternalServerError) + return + } + for _, sfe := range sfes { + locked = append(locked, types.Hash256(sfe.ID)) + } + + if siafundChange > 0 { + wcr.Siafunds = append(wcr.Siafunds, types.SiafundOutput{ + Value: siafundChange, + Address: wcr.ChangeAddress, + }) + } + + knownAddresses := make(map[types.Address]wallet.Address) + getAddress := func(addr types.Address) (wallet.Address, error) { + if a, ok := knownAddresses[addr]; ok { + return a, nil + } + a, err := s.wm.WalletAddress(walletID, addr) + + if err != nil { + return wallet.Address{}, err + } + knownAddresses[addr] = a + return a, nil + } + + resp := WalletConstructResponse{ + Basis: basis, + EstimatedFee: fee, + } + + txn := types.Transaction{ + MinerFees: []types.Currency{fee}, + SiacoinInputs: make([]types.SiacoinInput, 0, len(sces)), + SiacoinOutputs: wcr.Siacoins, + SiafundInputs: make([]types.SiafundInput, 0, len(sfes)), + SiafundOutputs: wcr.Siafunds, + } + + for _, sce := range sces { + addr, err := getAddress(sce.SiacoinOutput.Address) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return + } + + sci := types.SiacoinInput{ + ParentID: sce.ID, + } + + if addr.SpendPolicy != nil { + // best effort to fill unlock conditions + uc, ok := addr.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) + if !ok { + jc.Error(fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr.Address), http.StatusBadRequest) + return + } + sci.UnlockConditions = types.UnlockConditions(uc) + } + + txn.SiacoinInputs = append(txn.SiacoinInputs, sci) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sce.ID), + CoveredFields: types.CoveredFields{ + WholeTransaction: true, + }, + }) + } + + for _, sfe := range sfes { + addr, err := getAddress(sfe.SiafundOutput.Address) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return + } + + sfi := types.SiafundInput{ + ParentID: sfe.ID, + ClaimAddress: wcr.ChangeAddress, + } + if addr.SpendPolicy != nil { + // best effort to fill unlock conditions + uc, ok := addr.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) + if !ok { + jc.Error(fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr.Address), http.StatusBadRequest) + return + } + sfi.UnlockConditions = types.UnlockConditions(uc) + } + txn.SiafundInputs = append(txn.SiafundInputs, sfi) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{ + WholeTransaction: true, + }, + }) + } + + resp.ID = txn.ID() + resp.Transaction = txn + sent = true // locks are released in defer + jc.Encode(resp) +} + +func (s *server) walletsConstructV2Handler(jc jape.Context) { + cs := s.cm.TipState() + if cs.Index.Height < cs.Network.HardforkV2.AllowHeight { + jc.Error(errors.New("v2 transactions are not allowed before the v2 allow height"), http.StatusBadRequest) + } + + var walletID wallet.ID + if err := jc.DecodeParam("id", &walletID); err != nil { + return + } + var wcr WalletConstructRequest + if err := jc.Decode(&wcr); err != nil { + return + } + + var siacoinInput types.Currency + for i, sco := range wcr.Siacoins { + switch { + case sco.Value.IsZero(): + jc.Error(fmt.Errorf("siacoin output %d has zero value", i), http.StatusBadRequest) + return + case sco.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siacoin output %d has void address", i), http.StatusBadRequest) + return + } + siacoinInput = siacoinInput.Add(sco.Value) + } + + var siafundInput uint64 + for i, sfo := range wcr.Siafunds { + switch { + case sfo.Value == 0: + jc.Error(fmt.Errorf("siafund output %d has zero value", i), http.StatusBadRequest) + return + case sfo.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siafund output %d has void address", i), http.StatusBadRequest) + return + } + siafundInput += sfo.Value + } + + if siacoinInput.IsZero() && siafundInput == 0 { + jc.Error(errors.New("no inputs provided"), http.StatusBadRequest) + } + + fee := s.cm.RecommendedFee().Mul64(2000) // use a const for simplicity + + var sent bool + var locked []types.Hash256 + defer func() { + if sent { + return + } + s.wm.Release(locked) + }() + + sces, basis, siacoinChange, err := s.wm.SelectSiacoinElements(walletID, siacoinInput.Add(fee), false) + if err != nil { + jc.Error(fmt.Errorf("failed to select siacoin elements: %w", err), http.StatusInternalServerError) + return + } + for _, sce := range sces { + locked = append(locked, types.Hash256(sce.ID)) + } + + if !siacoinChange.IsZero() { + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + wcr.Siacoins = append(wcr.Siacoins, types.SiacoinOutput{ + Value: siacoinChange, + Address: wcr.ChangeAddress, + }) + } + + sfes, sfBasis, siafundChange, err := s.wm.SelectSiafundElements(walletID, siafundInput) + if err != nil { + jc.Error(fmt.Errorf("failed to select siafund elements: %w", err), http.StatusInternalServerError) + return + } + for _, sfe := range sfes { + locked = append(locked, types.Hash256(sfe.ID)) + } + + if siafundChange > 0 { + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + wcr.Siafunds = append(wcr.Siafunds, types.SiafundOutput{ + Value: siafundChange, + Address: wcr.ChangeAddress, + }) + } + + knownAddresses := make(map[types.Address]wallet.Address) + getAddress := func(addr types.Address) (wallet.Address, error) { + if a, ok := knownAddresses[addr]; ok { + return a, nil + } + a, err := s.wm.WalletAddress(walletID, addr) + + if err != nil { + return wallet.Address{}, err + } + knownAddresses[addr] = a + return a, nil + } + + resp := WalletConstructV2Response{ + Basis: basis, + EstimatedFee: fee, + } + + txn := types.V2Transaction{ + MinerFee: fee, + SiacoinInputs: make([]types.V2SiacoinInput, 0, len(sces)), + SiacoinOutputs: wcr.Siacoins, + SiafundInputs: make([]types.V2SiafundInput, 0, len(sfes)), + SiafundOutputs: wcr.Siafunds, + } + + for _, sfe := range sfes { + addr, err := getAddress(sfe.SiafundOutput.Address) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return + } + + sfi := types.V2SiafundInput{ + Parent: sfe, + ClaimAddress: wcr.ChangeAddress, + } + + if addr.SpendPolicy != nil { + // best effort to fill spend policy + sfi.SatisfiedPolicy = types.SatisfiedPolicy{ + Policy: *addr.SpendPolicy, + } + } + txn.SiafundInputs = append(txn.SiafundInputs, sfi) + } + + if len(sfes) > 0 && basis != sfBasis { + txnset, err := s.cm.UpdateV2TransactionSet([]types.V2Transaction{txn}, sfBasis, basis) + if err != nil { + jc.Error(fmt.Errorf("failed to update transaction set: %w", err), http.StatusInternalServerError) + return + } + txn = txnset[0] + } + + for _, sce := range sces { + addr, err := getAddress(sce.SiacoinOutput.Address) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return + } + + sci := types.V2SiacoinInput{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: *addr.SpendPolicy, + }, + } + + if addr.SpendPolicy != nil { + // best effort to fill spend policy + sci.SatisfiedPolicy = types.SatisfiedPolicy{ + Policy: *addr.SpendPolicy, + } + } + + txn.SiacoinInputs = append(txn.SiacoinInputs, sci) + } + + resp.ID = txn.ID() + resp.Transaction = txn + sent = true // locks are released in defer + jc.Encode(resp) +} + func (s *server) addressesAddrBalanceHandler(jc jape.Context) { var addr types.Address if jc.DecodeParam("addr", &addr) != nil { @@ -932,22 +1304,24 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /rescan": wrapAuthHandler(srv.rescanHandlerGET), "POST /rescan": wrapAuthHandler(srv.rescanHandlerPOST), - "GET /wallets": wrapAuthHandler(srv.walletsHandler), - "POST /wallets": wrapAuthHandler(srv.walletsHandlerPOST), - "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), - "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), - "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), - "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), - "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), - "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), - "GET /wallets/:id/events": wrapAuthHandler(srv.walletsEventsHandler), - "GET /wallets/:id/events/unconfirmed": wrapAuthHandler(srv.walletsEventsUnconfirmedHandlerGET), - "GET /wallets/:id/outputs/siacoin": wrapAuthHandler(srv.walletsOutputsSiacoinHandler), - "GET /wallets/:id/outputs/siafund": wrapAuthHandler(srv.walletsOutputsSiafundHandler), - "POST /wallets/:id/reserve": wrapAuthHandler(srv.walletsReserveHandler), - "POST /wallets/:id/release": wrapAuthHandler(srv.walletsReleaseHandler), - "POST /wallets/:id/fund": wrapAuthHandler(srv.walletsFundHandler), - "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), + "GET /wallets": wrapAuthHandler(srv.walletsHandler), + "POST /wallets": wrapAuthHandler(srv.walletsHandlerPOST), + "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), + "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), + "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), + "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), + "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), + "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), + "GET /wallets/:id/events": wrapAuthHandler(srv.walletsEventsHandler), + "POST /wallets/:id/construct/transaction": wrapAuthHandler(srv.walletsConstructHandler), + "POST /wallets/:id/construct/v2/transaction": wrapAuthHandler(srv.walletsConstructV2Handler), + "GET /wallets/:id/events/unconfirmed": wrapAuthHandler(srv.walletsEventsUnconfirmedHandlerGET), + "GET /wallets/:id/outputs/siacoin": wrapAuthHandler(srv.walletsOutputsSiacoinHandler), + "GET /wallets/:id/outputs/siafund": wrapAuthHandler(srv.walletsOutputsSiafundHandler), + "POST /wallets/:id/reserve": wrapAuthHandler(srv.walletsReserveHandler), + "POST /wallets/:id/release": wrapAuthHandler(srv.walletsReleaseHandler), + "POST /wallets/:id/fund": wrapAuthHandler(srv.walletsFundHandler), + "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), } if srv.debugEnabled { diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index 05be576..a4e6767 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -28,7 +28,7 @@ func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { d.Mul(d, big.NewInt(int64(1+elapsed))) fmt.Printf("\rMining block %4v...(%.2f blocks/day), difficulty %v)", cs.Index.Height+1, float64(blocksFound)*float64(24*time.Hour)/float64(elapsed), cs.Difficulty) - txns, v2txns, err := c.TxpoolTransactions() + _, txns, v2txns, err := c.TxpoolTransactions() checkFatalError("failed to get pool transactions:", err) b := types.Block{ ParentID: cs.Index.ID, diff --git a/wallet/manager.go b/wallet/manager.go index 753d661..d679fb3 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -292,6 +292,11 @@ func (m *Manager) Release(ids []types.Hash256) { } } +// WalletAddress returns an address from the wallet. +func (m *Manager) WalletAddress(id ID, addr types.Address) (Address, error) { + return m.store.WalletAddress(id, addr) +} + // SelectSiacoinElements selects siacoin elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. From d683cd4a111d00afde5f217337275617ffedcd98 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 9 Jan 2025 09:52:48 -0800 Subject: [PATCH 322/630] api: fix test error text --- api/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api_test.go b/api/api_test.go index 4633620..486f255 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1877,7 +1877,7 @@ func TestConstructV2Siafunds(t *testing.T) { if err != nil { t.Fatal(err) } else if len(confirmed) != 3 { - t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim + t.Fatalf("expected 3 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim } sent = confirmed[1] // confirmed[0] is the siafund claim switch { From fc9c315c1600a815c4cf6f1f910b5c64db21cff9 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 09:35:27 -0800 Subject: [PATCH 323/630] refactor: address review comments --- .../add_transaction_construction_api.md | 8 +--- api/api_test.go | 47 +++++++++++++++++++ api/client.go | 6 ++- api/server.go | 39 ++++++++------- 4 files changed, 75 insertions(+), 25 deletions(-) diff --git a/.changeset/add_transaction_construction_api.md b/.changeset/add_transaction_construction_api.md index 18f82df..f598024 100644 --- a/.changeset/add_transaction_construction_api.md +++ b/.changeset/add_transaction_construction_api.md @@ -4,12 +4,6 @@ default: minor # Add transaction construction API -Added two new endpoints to construct transactions with a few restrictions for ease of use. Clients can still use the existing fund endpoints for "advanced" transactions. All addresses in the wallet must all have either unlock conditions with a single required signature or a public key spend policy. - -This is a two step process. The private keys are never transmitted to the server. - -The client first calls `[POST] /api/:wallet/transaction/construct` with the recipients. The server will construct the transaction and return a list of hashes that the client needs to sign to broadcast the transaction. The client needs to match the returned public keys to their ed25519 private key and sign all of the hashes. - -After signing, the client calls `[POST] /api/:wallet/transaction/construct/:id` with the array of signatures. The server will add the signatures to the transaction and broadcast it. If the client provided the correct signatures, the transaction will be added to the tpool and broadcast. +Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for sending siacoin and siafund transactions. will be added to the tpool and broadcast. See API docs for request and response bodies \ No newline at end of file diff --git a/api/api_test.go b/api/api_test.go index 486f255..63b086c 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -10,6 +10,7 @@ import ( "net/http" "path/filepath" "reflect" + "strings" "testing" "time" @@ -1365,8 +1366,21 @@ func TestConstructSiacoins(t *testing.T) { testutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, ws) + // try to construct a transaction with more siafunds than the wallet holds. + // this will lock all of the wallet's siacoins resp, err := wc.Construct([]types.SiacoinOutput{ {Value: types.Siacoins(1), Address: receiverAddr}, + }, []types.SiafundOutput{ + {Value: 100000, Address: senderAddr}, + }, senderAddr) + if !strings.Contains(err.Error(), "insufficient funds") { + t.Fatal(err) + } + + // construct a transaction with a single siacoin output + // this will fail if the utxos were not unlocked + resp, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, }, nil, senderAddr) if err != nil { t.Fatal(err) @@ -1594,6 +1608,16 @@ func TestConstructSiafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } + + claim := confirmed[0] + switch { + case claim.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected claim event to have type %q, got %q", wallet.EventTypeSiafundClaim, claim.Type) + case !claim.SiacoinOutflow().IsZero(): + t.Fatalf("expected claim event to have siacoin outflow of 0, got %v", claim.SiacoinOutflow()) + case !claim.SiacoinInflow().IsZero(): + t.Fatalf("expected claim event to have siacoin inflow of 0, got %v", claim.SiacoinInflow()) + } } func TestConstructV2Siacoins(t *testing.T) { @@ -1675,8 +1699,21 @@ func TestConstructV2Siacoins(t *testing.T) { testutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, ws) + // try to construct a transaction with more siafunds than the wallet holds. + // this will lock all of the wallet's Siacoin UTXOs resp, err := wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(1), Address: receiverAddr}, + }, []types.SiafundOutput{ + {Value: 100000, Address: senderAddr}, + }, senderAddr) + if !strings.Contains(err.Error(), "insufficient funds") { + t.Fatal(err) + } + + // this will fail if the utxos were not properly + // unlocked when the previous request failed + resp, err = wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, }, nil, senderAddr) if err != nil { t.Fatal(err) @@ -1890,6 +1927,16 @@ func TestConstructV2Siafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } + + claim := confirmed[0] + switch { + case claim.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected claim event to have type %q, got %q", wallet.EventTypeSiafundClaim, claim.Type) + case !claim.SiacoinOutflow().IsZero(): + t.Fatalf("expected claim event to have siacoin outflow of 0, got %v", claim.SiacoinOutflow()) + case !claim.SiacoinInflow().IsZero(): + t.Fatalf("expected claim event to have siacoin inflow of 0, got %v", claim.SiacoinInflow()) + } } func TestDebugMine(t *testing.T) { diff --git a/api/client.go b/api/client.go index 0b55a84..518ed8c 100644 --- a/api/client.go +++ b/api/client.go @@ -338,7 +338,8 @@ func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, return } -// Construct constructs a transaction and returns its ID +// Construct constructs a transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned +// along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructResponse, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/transaction", c.id), WalletConstructRequest{ Siacoins: siacoins, @@ -348,7 +349,8 @@ func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []type return } -// ConstructV2 constructs a v2 transaction and returns its ID +// Construct constructs a V2 transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned +// along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. func (c *WalletClient) ConstructV2(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructV2Response, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ Siacoins: siacoins, diff --git a/api/server.go b/api/server.go index 17c4bd0..6ebadf2 100644 --- a/api/server.go +++ b/api/server.go @@ -686,6 +686,15 @@ func (s *server) walletsConstructHandler(jc jape.Context) { if err := jc.DecodeParam("id", &walletID); err != nil { return } + + _, err := s.wm.WalletBalance(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to get wallet", err) != nil { + return + } + var wcr WalletConstructRequest if err := jc.Decode(&wcr); err != nil { return @@ -775,7 +784,6 @@ func (s *server) walletsConstructHandler(jc jape.Context) { return a, nil } a, err := s.wm.WalletAddress(walletID, addr) - if err != nil { return wallet.Address{}, err } @@ -871,11 +879,25 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { if err := jc.DecodeParam("id", &walletID); err != nil { return } + + _, err := s.wm.WalletBalance(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to get wallet", err) != nil { + return + } + var wcr WalletConstructRequest if err := jc.Decode(&wcr); err != nil { return } + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + var siacoinInput types.Currency for i, sco := range wcr.Siacoins { switch { @@ -927,11 +949,6 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } if !siacoinChange.IsZero() { - if wcr.ChangeAddress == types.VoidAddress { - jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) - return - } - wcr.Siacoins = append(wcr.Siacoins, types.SiacoinOutput{ Value: siacoinChange, Address: wcr.ChangeAddress, @@ -948,11 +965,6 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } if siafundChange > 0 { - if wcr.ChangeAddress == types.VoidAddress { - jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) - return - } - wcr.Siafunds = append(wcr.Siafunds, types.SiafundOutput{ Value: siafundChange, Address: wcr.ChangeAddress, @@ -965,7 +977,6 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { return a, nil } a, err := s.wm.WalletAddress(walletID, addr) - if err != nil { return wallet.Address{}, err } @@ -1025,9 +1036,6 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { sci := types.V2SiacoinInput{ Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: *addr.SpendPolicy, - }, } if addr.SpendPolicy != nil { @@ -1036,7 +1044,6 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { Policy: *addr.SpendPolicy, } } - txn.SiacoinInputs = append(txn.SiacoinInputs, sci) } From 5b44bf33156abe9bd45cef22e7c55f60644e16de Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 09:35:58 -0800 Subject: [PATCH 324/630] fix lint --- api/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client.go b/api/client.go index 518ed8c..82ca049 100644 --- a/api/client.go +++ b/api/client.go @@ -349,7 +349,7 @@ func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []type return } -// Construct constructs a V2 transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned +// ConstructV2 constructs a V2 transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned // along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. func (c *WalletClient) ConstructV2(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructV2Response, err error) { err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ From d3decb87fe180b01febaebf196275e19d3b123b7 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 09:43:33 -0800 Subject: [PATCH 325/630] docs: update changeset --- .changeset/add_transaction_construction_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/add_transaction_construction_api.md b/.changeset/add_transaction_construction_api.md index f598024..053b46e 100644 --- a/.changeset/add_transaction_construction_api.md +++ b/.changeset/add_transaction_construction_api.md @@ -4,6 +4,6 @@ default: minor # Add transaction construction API -Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for sending siacoin and siafund transactions. will be added to the tpool and broadcast. +Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for simple send transactions. See API docs for request and response bodies \ No newline at end of file From 72244ffee0becb1ad817d8890036ddfd3a9ed72e Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 09:58:28 -0800 Subject: [PATCH 326/630] document construct v2 ordering --- api/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/server.go b/api/server.go index 6ebadf2..563b62f 100644 --- a/api/server.go +++ b/api/server.go @@ -997,6 +997,10 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { SiafundOutputs: wcr.Siafunds, } + // the siafund elements are added to the transaction first because `UpdateV2TransactionSet` takes + // a V2 transaction as an argument. The Siacoin basis is our target because the transaction is + // guaranteed to have a non-zero Siacoin basis while the Siafund basis will be zero when not + // sending Siafunds. for _, sfe := range sfes { addr, err := getAddress(sfe.SiafundOutput.Address) if err != nil { From 6c798e1fa819cac70508b107550b4798ec8bab2d Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 10 Jan 2025 10:49:53 -0800 Subject: [PATCH 327/630] wallet: skip indexing zero-valued Siafund claim events --- wallet/update.go | 4 +- wallet/wallet.go | 4 +- wallet/wallet_test.go | 497 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 494 insertions(+), 11 deletions(-) diff --git a/wallet/update.go b/wallet/update.go index 9392c7a..af410c7 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -94,7 +94,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e // add new siacoin elements to the store cau.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { - if created && spent { + if (created && spent) || se.SiacoinOutput.Value.IsZero() { return } @@ -113,7 +113,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e }) cau.ForEachSiafundElement(func(se types.SiafundElement, created, spent bool) { - if created && spent { + if (created && spent) || se.SiafundOutput.Value == 0 { return } diff --git a/wallet/wallet.go b/wallet/wallet.go index 762d744..e28e5a0 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -220,7 +220,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } sce, ok := sces[sfi.ParentID.ClaimOutputID()] - if ok && relevant(sce.SiacoinOutput.Address) { + if ok && relevant(sce.SiacoinOutput.Address) && !sce.SiacoinOutput.Value.IsZero() { addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) @@ -267,7 +267,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f addresses[sfi.Parent.SiafundOutput.Address] = true sce, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] - if ok && relevant(sfi.ClaimAddress) { + if ok && relevant(sfi.ClaimAddress) && !sce.SiacoinOutput.Value.IsZero() { addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ SiacoinElement: sce, }, []types.Address{sfi.ClaimAddress}) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index dbfb663..73bfc16 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1755,10 +1755,8 @@ func TestFullIndex(t *testing.T) { // check the events for the transaction if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { t.Fatal(err) - } else if len(events) != 4 { - t.Fatalf("expected 4 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeSiafundClaim { - t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) } // check the events for the first address @@ -1996,10 +1994,8 @@ func TestEvents(t *testing.T) { events, err = wm.AddressEvents(addr2, 0, 100) if err != nil { t.Fatal(err) - } else if len(events) != 4 { + } else if len(events) != 3 { t.Fatalf("expected 4 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeSiafundClaim { - t.Fatalf("expected transaction event, got %v", events[0].Type) } expected = events[0] @@ -3855,6 +3851,493 @@ func TestEventTypes(t *testing.T) { }) } +func TestSiafundClaims(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesis := testutil.Network() + // send the siafunds to the owned address + genesis.Transactions[0].SiafundOutputs[0].Address = addr + siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value + + store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + uc := types.StandardUnlockConditions(pk.PublicKey()) + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + // rescan to index the genesis block + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // claim the siafunds. Since tax revenue is 0, no claim event or utxo should be indexed. + siafunds, _, change, err := wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn := types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: addr, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + cs := cm.TipState() + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(txn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(siacoins) != 0 { + t.Fatalf("expected no siacoin outputs, got %v", siacoins) + } + + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { // airdrop + siafund transaction + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // fund the wallet with some siacoins + testutil.MineBlocks(t, cm, addr, 5) + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + payout := types.Siacoins(100000) + fundAmount := taxAdjustedPayout(payout) + expectedTaxRevenue := fundAmount.Sub(payout) + fc := types.FileContract{ + UnlockHash: addr, + Payout: fundAmount, + ValidProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + WindowStart: cm.Tip().Height + 10, + WindowEnd: cm.Tip().Height + 20, + } + + fcTxn := types.Transaction{ + FileContracts: []types.FileContract{fc}, + } + + siacoins, _, scChange, err := wm.SelectSiacoinElements(w.ID, fundAmount, false) + if err != nil { + t.Fatal(err) + } + + if !scChange.IsZero() { + fcTxn.SiacoinOutputs = append(fcTxn.SiacoinOutputs, types.SiacoinOutput{ + Address: addr, + Value: scChange, + }) + } + + for _, sce := range siacoins { + fcTxn.SiacoinInputs = append(fcTxn.SiacoinInputs, types.SiacoinInput{ + ParentID: sce.ID, + UnlockConditions: uc, + }) + fcTxn.Signatures = append(fcTxn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sce.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + + cs = cm.TipState() + for i, sig := range fcTxn.Signatures { + sigHash := cs.WholeSigHash(fcTxn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + fcTxn.Signatures[i].Signature = sig[:] + } + + if _, err := cm.AddPoolTransactions([]types.Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + cs = cm.TipState() + if !cs.SiafundTaxRevenue.Equals(expectedTaxRevenue) { + t.Fatalf("expected %v tax revenue, got %v", expectedTaxRevenue, cs.SiafundTaxRevenue) + } + + // claim the siafunds again. A claim event should be created to account for the + // tax revenue. + siafunds, _, change, err = wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn = types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: addr, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + cs = cm.TipState() + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(txn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 10 { // airdrop + 2x siafund transaction + 5x miner payouts + 1x file contract + 1x siafund claim + t.Fatalf("expected 10 events, got %v", len(events)) + } + + // check the siafund claim event + expectedID := txn.SiafundInputs[0].ParentID.ClaimOutputID() + claimEvent := events[0] + switch { + case claimEvent.ID != types.Hash256(expectedID): + t.Fatalf("expected siafund claim output %q, got %q", expectedID, claimEvent.ID) + case claimEvent.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected siafund claim event, got %v", claimEvent.Type) + case !claimEvent.SiacoinInflow().Equals(expectedTaxRevenue): + t.Fatalf("expected %v tax revenue, got %v", expectedTaxRevenue, claimEvent.SiacoinInflow()) + case !claimEvent.SiacoinOutflow().IsZero(): + t.Fatalf("expected no outflow, got %v", claimEvent.SiacoinOutflow()) + } + + // mine until the siafund claim output is mature + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + // check that the output is now spendable + siacoins, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + for _, sce := range siacoins { + if sce.ID == expectedID && sce.SiacoinOutput.Value.Equals(expectedTaxRevenue) { + return + } + } + t.Fatalf("expected siafund claim output %q with value %v not found", expectedID, expectedTaxRevenue) +} + +func TestV2SiafundClaims(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardAddress(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesis := testutil.V2Network() + // send the siafunds to the owned address + genesis.Transactions[0].SiafundOutputs[0].Address = addr + siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value + + store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // activate the v2 hardfork + testutil.MineBlocks(t, cm, types.VoidAddress, 2) + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sp := types.SpendPolicy{ + Type: types.PolicyTypePublicKey(pk.PublicKey()), + } + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &sp, + }) + if err != nil { + t.Fatal(err) + } + + // rescan to index the genesis block + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // claim the siafunds. Since tax revenue is 0, no claim event or utxo should be indexed. + siafunds, basis, change, err := wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn := types.V2Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ + Parent: sfe, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + ClaimAddress: addr, + }) + } + cs := cm.TipState() + sigHash := cs.InputSigHash(txn) + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(siacoins) != 0 { + t.Fatalf("expected no siacoin outputs, got %v", siacoins) + } + + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { // airdrop + siafund transaction + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // fund the wallet with some siacoins + testutil.MineBlocks(t, cm, addr, 5) + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + payout := types.Siacoins(100000) + cs = cm.TipState() + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: payout, + }, + ProofHeight: cs.Index.Height + 10, + ExpirationHeight: cs.Index.Height + 20, + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + sigHash = cs.ContractSigHash(fc) + fc.RenterSignature = pk.SignHash(sigHash) + fc.HostSignature = pk.SignHash(sigHash) + + expectedTax := cs.V2FileContractTax(fc) + fundAmount := payout.Add(expectedTax) + + fcTxn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + } + + siacoins, basis, scChange, err := wm.SelectSiacoinElements(w.ID, fundAmount, false) + if err != nil { + t.Fatal(err) + } + + if !scChange.IsZero() { + fcTxn.SiacoinOutputs = append(fcTxn.SiacoinOutputs, types.SiacoinOutput{ + Address: addr, + Value: scChange, + }) + } + + for _, sce := range siacoins { + fcTxn.SiacoinInputs = append(fcTxn.SiacoinInputs, types.V2SiacoinInput{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }) + } + + sigHash = cs.InputSigHash(fcTxn) + for i := range fcTxn.SiacoinInputs { + fcTxn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + cs = cm.TipState() + if !cs.SiafundTaxRevenue.Equals(expectedTax) { + t.Fatalf("expected %v tax revenue, got %v", expectedTax, cs.SiafundTaxRevenue) + } + + // claim the siafunds again. A claim event should be created to account for the + // tax revenue. + siafunds, basis, change, err = wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn = types.V2Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ + Parent: sfe, + SatisfiedPolicy: types.SatisfiedPolicy{Policy: sp}, + ClaimAddress: addr, + }) + } + + cs = cm.TipState() + sigHash = cs.InputSigHash(txn) + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 10 { // airdrop + 2x siafund transaction + 5x miner payouts + 1x file contract + 1x siafund claim + t.Fatalf("expected 10 events, got %v", len(events)) + } + + // check the siafund claim event + expectedID := txn.SiafundInputs[0].Parent.ID.V2ClaimOutputID() + claimEvent := events[0] + switch { + case claimEvent.ID != types.Hash256(expectedID): + t.Fatalf("expected siafund claim output %q, got %q", expectedID, claimEvent.ID) + case claimEvent.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected siafund claim event, got %v", claimEvent.Type) + case !claimEvent.SiacoinInflow().Equals(expectedTax): + t.Fatalf("expected %v tax revenue, got %v", expectedTax, claimEvent.SiacoinInflow()) + case !claimEvent.SiacoinOutflow().IsZero(): + t.Fatalf("expected no outflow, got %v", claimEvent.SiacoinOutflow()) + } + + // mine until the siafund claim output is mature + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + // check that the output is now spendable + siacoins, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + for _, sce := range siacoins { + if sce.ID == expectedID && sce.SiacoinOutput.Value.Equals(expectedTax) { + return + } + } + t.Fatalf("expected siafund claim output %q with value %v not found", expectedID, expectedTax) +} + func TestReset(t *testing.T) { log := zaptest.NewLogger(t) From 86f4307c57ec96c000f1b09d8571c8853bdf2ccd Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 13 Jan 2025 08:11:05 -0800 Subject: [PATCH 328/630] fix api test after merging zero valued utxos --- api/api_test.go | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 63b086c..6e0fbe1 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1594,10 +1594,10 @@ func TestConstructSiafunds(t *testing.T) { confirmed, err := wc.Events(0, 5) if err != nil { t.Fatal(err) - } else if len(confirmed) != 3 { - t.Fatalf("expected 3 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction } - sent = confirmed[1] // confirmed[0] is the siafund claim + sent = confirmed[0] switch { case types.TransactionID(sent.ID) != resp.ID: t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) @@ -1608,16 +1608,6 @@ func TestConstructSiafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } - - claim := confirmed[0] - switch { - case claim.Type != wallet.EventTypeSiafundClaim: - t.Fatalf("expected claim event to have type %q, got %q", wallet.EventTypeSiafundClaim, claim.Type) - case !claim.SiacoinOutflow().IsZero(): - t.Fatalf("expected claim event to have siacoin outflow of 0, got %v", claim.SiacoinOutflow()) - case !claim.SiacoinInflow().IsZero(): - t.Fatalf("expected claim event to have siacoin inflow of 0, got %v", claim.SiacoinInflow()) - } } func TestConstructV2Siacoins(t *testing.T) { @@ -1913,10 +1903,11 @@ func TestConstructV2Siafunds(t *testing.T) { confirmed, err := wc.Events(0, 5) if err != nil { t.Fatal(err) - } else if len(confirmed) != 3 { - t.Fatalf("expected 3 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + siafund claim + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction } - sent = confirmed[1] // confirmed[0] is the siafund claim + + sent = confirmed[0] switch { case types.TransactionID(sent.ID) != resp.ID: t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) @@ -1927,16 +1918,6 @@ func TestConstructV2Siafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } - - claim := confirmed[0] - switch { - case claim.Type != wallet.EventTypeSiafundClaim: - t.Fatalf("expected claim event to have type %q, got %q", wallet.EventTypeSiafundClaim, claim.Type) - case !claim.SiacoinOutflow().IsZero(): - t.Fatalf("expected claim event to have siacoin outflow of 0, got %v", claim.SiacoinOutflow()) - case !claim.SiacoinInflow().IsZero(): - t.Fatalf("expected claim event to have siacoin inflow of 0, got %v", claim.SiacoinInflow()) - } } func TestDebugMine(t *testing.T) { From 441cc8e2cb55c3e51af068ce828aa90d48c9b1a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:13:53 +0000 Subject: [PATCH 329/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.9.0 to 0.9.1 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.9.0...v0.9.1) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2bae8b3..df24630 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.9.0 - go.sia.tech/coreutils v0.9.0 + go.sia.tech/coreutils v0.9.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 80525cb..f1a03f4 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.9.0 h1:qV7V8nkNaPvBEhkbwgrETTkb7JCMcAnKUQt9nUumP4k= go.sia.tech/core v0.9.0/go.mod h1:3NAvYHuzAZg9vP6pyIMOxjTkgHBQ3vx9cXTqRF6oEa4= -go.sia.tech/coreutils v0.9.0 h1:5cnK0RtHOyErGhcmNkmCdEKeuj1tECwO9PYbErEbpDQ= -go.sia.tech/coreutils v0.9.0/go.mod h1:KFq1q5/YbPH6ZSWtXCxA1bRhBF5Zgcj8G3Wvu0jr/BA= +go.sia.tech/coreutils v0.9.1 h1:2SukWrF9o18HIG+BNmNk4SZw48+h+cM0gR8jMtG2cQ4= +go.sia.tech/coreutils v0.9.1/go.mod h1:A/tNYSBdryxCFaIpvW04YvdQ4e2j8iGuHoIw70+ZYXc= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= From 40bada153060ee9692a33ea356e1f42f953fda1f Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 14 Jan 2025 18:30:00 -0800 Subject: [PATCH 330/630] use standard directories --- ...standard_locations_for_application_data.md | 25 +++++++ .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/workflows/publish.yml | 6 +- Dockerfile | 22 +++--- cmd/walletd/config.go | 43 ++++++++++-- cmd/walletd/main.go | 70 +++++++++++-------- cmd/walletd/node.go | 54 ++++++++++++++ config/config.go | 29 +++++++- 8 files changed, 198 insertions(+), 53 deletions(-) create mode 100644 .changeset/use_standard_locations_for_application_data.md diff --git a/.changeset/use_standard_locations_for_application_data.md b/.changeset/use_standard_locations_for_application_data.md new file mode 100644 index 0000000..63811ed --- /dev/null +++ b/.changeset/use_standard_locations_for_application_data.md @@ -0,0 +1,25 @@ +--- +default: major +--- + +# Use standard locations for application data + +# Use standard locations for application data + + Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. + + #### Linux, FreeBSD, OpenBSD + - Configuration: `/etc/walletd/walletd.yml` + - Data directory: `/var/lib/walletd` + + #### macOS + - Configuration: `~/Library/Application Support/walletd.yml` + - Data directory: `~/Library/Application Support/walletd` + + #### Windows + - Configuration: `%APPDATA%\SiaFoundation\walletd.yml` + - Data directory: `%APPDATA%\SiaFoundation\walletd` + + #### Docker + - Configuration: `/data/walletd.yml` + - Data directory: `/data` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 1a50396..d8e8e7d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Sia Community Discord url: https://discord.gg/sia - about: Join the Sia community discord for more help with Sia or hostd. \ No newline at end of file + about: Join the Sia community discord for more help with Sia or walletd. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 741e4ff..42468b3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,9 +19,9 @@ jobs: uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master secrets: inherit with: - linux-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' - windows-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' - macos-build-args: -tags=timetzdata -trimpath -a -ldflags '-s -w' + linux-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + windows-build-args: -tags='timetzdata netgo -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + macos-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w' cgo-enabled: 1 project: walletd project-desc: "walletd: The new Sia wallet" diff --git a/Dockerfile b/Dockerfile index 2318f1f..3ad2c4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,22 +10,18 @@ RUN go mod download COPY . . # Enable CGO for sqlite3 support -ENV CGO_ENABLED=1 +ENV CGO_ENABLED=1 RUN go generate ./... RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd -FROM docker.io/library/alpine:3 +FROM debian:bookworm-slim LABEL maintainer="The Sia Foundation " \ - org.opencontainers.image.description.vendor="The Sia Foundation" \ - org.opencontainers.image.description="A walletd container - send and receive Siacoins and Siafunds" \ - org.opencontainers.image.source="https://github.com/SiaFoundation/walletd" \ - org.opencontainers.image.licenses=MIT + org.opencontainers.image.description.vendor="The Sia Foundation" \ + org.opencontainers.image.description="A walletd container - send and receive Siacoins and Siafunds" \ + org.opencontainers.image.source="https://github.com/SiaFoundation/walletd" \ + org.opencontainers.image.licenses=MIT -ENV PUID=0 -ENV PGID=0 - -ENV WALLETD_API_PASSWORD= # copy binary and prepare data dir. COPY --from=builder /walletd/bin/* /usr/bin/ @@ -36,7 +32,7 @@ EXPOSE 9980/tcp # RPC port EXPOSE 9981/tcp -USER ${PUID}:${PGID} - +ENV WALLETD_DATA_DIR=/data ENV WALLETD_CONFIG_FILE=/data/walletd.yml -ENTRYPOINT [ "walletd", "--dir", "/data", "--http", ":9980" ] \ No newline at end of file + +ENTRYPOINT [ "walletd", "--http", ":9980" ] diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go index 5c41067..353998f 100644 --- a/cmd/walletd/config.go +++ b/cmd/walletd/config.go @@ -7,6 +7,7 @@ import ( "net" "os" "path/filepath" + "runtime" "strconv" "strings" @@ -207,17 +208,45 @@ func setAdvancedConfig() { cfg.Consensus.Network = readInput(`Enter network ("mainnet" or "zen")`) } -func buildConfig() { +func configPath() string { + if str := os.Getenv(configFileEnvVar); str != "" { + return str + } + + switch runtime.GOOS { + case "windows": + return filepath.Join(os.Getenv("APPDATA"), "walletd", "walletd.yml") + case "darwin": + return filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd", "walletd.yml") + case "linux", "freebsd", "openbsd": + return filepath.Join(string(filepath.Separator), "etc", "walletd", "walletd.yml") + default: + return "walletd.yml" + } +} + +func buildConfig(fp string) { + fmt.Println("walletd Configuration Wizard") + fmt.Println("This wizard will help you configure walletd for the first time.") + fmt.Println("You can always change these settings with the config command or by editing the config file.") + // write the config file - configPath := "walletd.yml" - if str := os.Getenv("WALLETD_CONFIG_FILE"); str != "" { - configPath = str + if fp == "" { + fp = configPath() } - if _, err := os.Stat(configPath); err == nil { - if !promptYesNo("walletd.yml already exists. Would you like to overwrite it?") { + fmt.Println("") + fmt.Printf("Config Location %q\n", fp) + + if _, err := os.Stat(fp); err == nil { + if !promptYesNo(fmt.Sprintf("%q already exists. Would you like to overwrite it?", fp)) { return } + } else if !errors.Is(err, os.ErrNotExist) { + checkFatalError("failed to check if config file exists", err) + } else { + // ensure the config directory exists + checkFatalError("failed to create config directory", os.MkdirAll(filepath.Dir(fp), 0700)) } fmt.Println("") @@ -230,7 +259,7 @@ func buildConfig() { setAdvancedConfig() // write the config file - f, err := os.Create(configPath) + f, err := os.Create(fp) checkFatalError("failed to create config file", err) defer f.Close() diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 3fcbfd9..07da3d2 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -17,10 +18,16 @@ import ( "go.sia.tech/walletd/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "gopkg.in/yaml.v3" "lukechampine.com/flagg" ) +const ( + apiPasswordEnvVar = "WALLETD_API_PASSWORD" + configFileEnvVar = "WALLETD_CONFIG_FILE" + dataDirEnvVar = "WALLETD_DATA_DIR" + logFileEnvVar = "WALLETD_LOG_FILE_PATH" +) + const ( rootUsage = `Usage: walletd [flags] [action] @@ -51,11 +58,11 @@ Runs a CPU miner. Not intended for production use. var cfg = config.Config{ Name: "walletd", - Directory: ".", + Directory: os.Getenv(dataDirEnvVar), AutoOpenWebUI: true, HTTP: config.HTTP{ Address: "localhost:9980", - Password: os.Getenv("WALLETD_API_PASSWORD"), + Password: os.Getenv(apiPasswordEnvVar), PublicEndpoints: false, }, Syncer: config.Syncer{ @@ -74,7 +81,7 @@ var cfg = config.Config{ File: config.LogFile{ Enabled: true, Format: "json", - Path: os.Getenv("WALLETD_LOG_FILE"), + Path: os.Getenv(logFileEnvVar), }, StdOut: config.StdOut{ Enabled: true, @@ -113,27 +120,20 @@ func checkFatalError(context string, err error) { os.Exit(1) } -// tryLoadConfig loads the config file specified by the WALLETD_CONFIG_FILE. If -// the config file does not exist, it will not be loaded. -func tryLoadConfig() { - configPath := "walletd.yml" - if str := os.Getenv("WALLETD_CONFIG_FILE"); str != "" { - configPath = str - } - - // If the config file doesn't exist, don't try to load it. - if _, err := os.Stat(configPath); os.IsNotExist(err) { - return +// tryLoadConfig tries to load the config file. It will try multiple locations +// based on GOOS starting with PWD/walletd.yml. If the file does not exist, it will +// try the next location. If an error occurs while loading the file, it will +// print the error and exit. If the config is successfully loaded, the path to +// the config file is returned. +func tryLoadConfig() string { + for _, fp := range tryConfigPaths() { + if err := config.LoadFile(fp, &cfg); err == nil { + return fp + } else if !errors.Is(err, os.ErrNotExist) { + checkFatalError("failed to load config file", err) + } } - - f, err := os.Open(configPath) - checkFatalError("failed to open config file", err) - defer f.Close() - - dec := yaml.NewDecoder(f) - dec.KnownFields(true) - - checkFatalError("failed to decode config file", dec.Decode(&cfg)) + return "" } // jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for @@ -180,10 +180,24 @@ func parseLogLevel(level string) zap.AtomicLevel { panic("unreachable") } +func initStdoutLog(colored bool, levelStr string) *zap.Logger { + level := parseLogLevel(levelStr) + core := zapcore.NewCore(humanEncoder(colored), zapcore.Lock(os.Stdout), level) + return zap.New(core, zap.AddCaller()) +} + func main() { - // attempt to load the config file first, command line flags will override - // any values set in the config file - tryLoadConfig() + log := initStdoutLog(cfg.Log.StdOut.EnableANSI, cfg.Log.Level) + defer log.Sync() + + // attempt to load the config file, command line flags will override any + // values set in the config file + configPath := tryLoadConfig() + if configPath != "" { + log.Info("loaded config file", zap.String("path", configPath)) + } + // set the data directory to the default if it is not set + cfg.Directory = defaultDataDirectory(cfg.Directory) indexModeStr := cfg.Index.Mode.String() @@ -329,7 +343,7 @@ func main() { return } - buildConfig() + buildConfig(configPath) case mineCmd: if len(cmd.Args()) != 0 { cmd.Usage() diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index d419563..e9a5b57 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -6,7 +6,9 @@ import ( "fmt" "net" "net/http" + "os" "path/filepath" + "runtime" "strconv" "strings" "time" @@ -27,6 +29,58 @@ import ( "lukechampine.com/upnp" ) +func tryConfigPaths() []string { + if str := os.Getenv(configFileEnvVar); str != "" { + return []string{str} + } + + paths := []string{ + "walletd.yml", + } + if str := os.Getenv(dataDirEnvVar); str != "" { + paths = append(paths, filepath.Join(str, "walletd.yml")) + } + + switch runtime.GOOS { + case "windows": + paths = append(paths, filepath.Join(os.Getenv("APPDATA"), "walletd", "walletd.yml")) + case "darwin": + paths = append(paths, filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd", "walletd.yml")) + case "linux", "freebsd", "openbsd": + paths = append(paths, + filepath.Join(string(filepath.Separator), "etc", "walletd", "walletd.yml"), + filepath.Join(string(filepath.Separator), "var", "lib", "walletd", "walletd.yml"), // old default for the Linux service + ) + } + return paths +} + +func defaultDataDirectory(fp string) string { + // use the provided path if it's not empty + if fp != "" { + return fp + } + + // check for databases in the current directory + if _, err := os.Stat("walletd.db"); err == nil { + return "." + } else if _, err := os.Stat("walletd.sqlite3"); err == nil { + return "." + } + + // default to the operating system's application directory + switch runtime.GOOS { + case "windows": + return filepath.Join(os.Getenv("APPDATA"), "walletd") + case "darwin": + return filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd") + case "linux", "freebsd", "openbsd": + return filepath.Join(string(filepath.Separator), "var", "lib", "walletd") + default: + return "." + } +} + func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() diff --git a/config/config.go b/config/config.go index dd18249..90ba010 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,13 @@ package config -import "go.sia.tech/walletd/wallet" +import ( + "bytes" + "fmt" + "os" + + "go.sia.tech/walletd/wallet" + "gopkg.in/yaml.v3" +) type ( // HTTP contains the configuration for the HTTP server. @@ -66,3 +73,23 @@ type ( Index Index `yaml:"index,omitempty"` } ) + +// LoadFile loads the configuration from the provided file path. +// If the file does not exist, an error is returned. +// If the file exists but cannot be decoded, the function will attempt +// to upgrade the config file. +func LoadFile(fp string, cfg *Config) error { + buf, err := os.ReadFile(fp) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + r := bytes.NewReader(buf) + dec := yaml.NewDecoder(r) + dec.KnownFields(true) + + if err := dec.Decode(cfg); err != nil { + return fmt.Errorf("failed to decode config file: %w", err) + } + return nil +} From d711ac97aa56e5e2fe53b03241f969bd18ae93c8 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 16 Jan 2025 07:53:22 -0800 Subject: [PATCH 331/630] fix copy pasta --- ...standard_locations_for_application_data.md | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/.changeset/use_standard_locations_for_application_data.md b/.changeset/use_standard_locations_for_application_data.md index 63811ed..d53f435 100644 --- a/.changeset/use_standard_locations_for_application_data.md +++ b/.changeset/use_standard_locations_for_application_data.md @@ -4,22 +4,20 @@ default: major # Use standard locations for application data -# Use standard locations for application data - - Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. +Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. - #### Linux, FreeBSD, OpenBSD - - Configuration: `/etc/walletd/walletd.yml` - - Data directory: `/var/lib/walletd` +#### Linux, FreeBSD, OpenBSD +- Configuration: `/etc/walletd/walletd.yml` +- Data directory: `/var/lib/walletd` - #### macOS - - Configuration: `~/Library/Application Support/walletd.yml` - - Data directory: `~/Library/Application Support/walletd` +#### macOS +- Configuration: `~/Library/Application Support/walletd.yml` +- Data directory: `~/Library/Application Support/walletd` - #### Windows - - Configuration: `%APPDATA%\SiaFoundation\walletd.yml` - - Data directory: `%APPDATA%\SiaFoundation\walletd` +#### Windows +- Configuration: `%APPDATA%\SiaFoundation\walletd.yml` +- Data directory: `%APPDATA%\SiaFoundation\walletd` - #### Docker - - Configuration: `/data/walletd.yml` - - Data directory: `/data` +#### Docker +- Configuration: `/data/walletd.yml` +- Data directory: `/data` From 7446b2d01a7155edbcb41423eb0d5e1ad2ae4457 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 14 Jan 2025 16:58:06 -0800 Subject: [PATCH 332/630] sqlite: add migration test --- ...re_consistency_between_database_schemas.md | 5 + persist/sqlite/init.go | 70 ++-- persist/sqlite/migrations_test.go | 323 ++++++++++++++++++ persist/sqlite/store.go | 31 +- 4 files changed, 405 insertions(+), 24 deletions(-) create mode 100644 .changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md create mode 100644 persist/sqlite/migrations_test.go diff --git a/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md b/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md new file mode 100644 index 0000000..87d2e57 --- /dev/null +++ b/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Added a test for migrations to ensure consistency between database schemas diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go index 2949588..95f39a5 100644 --- a/persist/sqlite/init.go +++ b/persist/sqlite/init.go @@ -34,31 +34,28 @@ func (s *Store) initNewDatabase(target int64) error { } func (s *Store) upgradeDatabase(current, target int64) error { - log := s.log.Named("migrations") - log.Info("migrating database", zap.Int64("current", current), zap.Int64("target", target)) - - return s.transaction(func(tx *txn) error { - // defer foreign key constraints until commit - if _, err := tx.Exec("PRAGMA defer_foreign_keys=ON"); err != nil { - return fmt.Errorf("failed to enable foreign key deferral: %w", err) - } - - for _, fn := range migrations[current-1:] { - current++ - start := time.Now() - if err := fn(tx, log.With(zap.Int64("version", current))); err != nil { - return fmt.Errorf("failed to migrate database to version %v: %w", current, err) - } - // check that no foreign key constraints were violated - if err := tx.QueryRow("PRAGMA foreign_key_check").Scan(); !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("foreign key constraints are not satisfied") + log := s.log.Named("migrations").With(zap.Int64("target", target)) + for ; current < target; current++ { + version := current + 1 // initial schema is version 1, migration 0 is version 2, etc. + log := log.With(zap.Int64("version", version)) + start := time.Now() + fn := migrations[current-1] + err := s.transaction(func(tx *txn) error { + if _, err := tx.Exec("PRAGMA defer_foreign_keys=ON"); err != nil { + return fmt.Errorf("failed to enable foreign key deferral: %w", err) + } else if err := fn(tx, log); err != nil { + return err + } else if err := foreignKeyCheck(tx, log); err != nil { + return fmt.Errorf("failed foreign key check: %w", err) } - log.Debug("migration complete", zap.Int64("current", current), zap.Int64("target", target), zap.Duration("elapsed", time.Since(start))) + return setDBVersion(tx, version) + }) + if err != nil { + return fmt.Errorf("migration %d failed: %w", version, err) } - - // set the final database version - return setDBVersion(tx, target) - }) + log.Info("migration complete", zap.Duration("elapsed", time.Since(start))) + } + return nil } func (s *Store) init() error { @@ -77,3 +74,30 @@ func (s *Store) init() error { // nothing to do return nil } + +func foreignKeyCheck(txn *txn, log *zap.Logger) error { + rows, err := txn.Query("PRAGMA foreign_key_check") + if err != nil { + return fmt.Errorf("failed to run foreign key check: %w", err) + } + defer rows.Close() + var hasErrors bool + for rows.Next() { + var table string + var rowid sql.NullInt64 + var fkTable string + var fkRowid sql.NullInt64 + + if err := rows.Scan(&table, &rowid, &fkTable, &fkRowid); err != nil { + return fmt.Errorf("failed to scan foreign key check result: %w", err) + } + hasErrors = true + log.Error("foreign key constraint violated", zap.String("table", table), zap.Int64("rowid", rowid.Int64), zap.String("fkTable", fkTable), zap.Int64("fkRowid", fkRowid.Int64)) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to iterate foreign key check results: %w", err) + } else if hasErrors { + return errors.New("foreign key constraint violated") + } + return nil +} diff --git a/persist/sqlite/migrations_test.go b/persist/sqlite/migrations_test.go new file mode 100644 index 0000000..91ef642 --- /dev/null +++ b/persist/sqlite/migrations_test.go @@ -0,0 +1,323 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +// nolint:misspell +const initialSchema = `CREATE TABLE chain_indices ( + id INTEGER PRIMARY KEY, + block_id BLOB UNIQUE NOT NULL, + height INTEGER UNIQUE NOT NULL +); +CREATE INDEX chain_indices_height ON chain_indices (block_id, height); + +CREATE TABLE sia_addresses ( + id INTEGER PRIMARY KEY, + sia_address BLOB UNIQUE NOT NULL, + siacoin_balance BLOB NOT NULL, + immature_siacoin_balance BLOB NOT NULL, + siafund_balance INTEGER NOT NULL +); + +CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id ON siacoin_elements(address_id, spent_index_id); + +CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id ON siafund_elements(address_id, spent_index_id); + +CREATE TABLE state_tree ( + row INTEGER, + column INTEGER, + value BLOB NOT NULL, + PRIMARY KEY (row, column) +); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + event_id BLOB UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, + date_created INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_data BLOB NOT NULL +); +CREATE INDEX events_chain_index_id ON events (chain_index_id); + +CREATE TABLE event_addresses ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + PRIMARY KEY (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); + +CREATE TABLE wallets ( + id INTEGER PRIMARY KEY, + friendly_name TEXT NOT NULL, + description TEXT NOT NULL, + date_created INTEGER NOT NULL, + last_updated INTEGER NOT NULL, + extra_data BLOB +); + +CREATE TABLE wallet_addresses ( + wallet_id INTEGER NOT NULL REFERENCES wallets (id), + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + description TEXT NOT NULL, + spend_policy BLOB, + extra_data BLOB, + UNIQUE (wallet_id, address_id) +); +CREATE INDEX wallet_addresses_wallet_id ON wallet_addresses (wallet_id); +CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); + +CREATE TABLE syncer_peers ( + peer_address TEXT PRIMARY KEY NOT NULL, + first_seen INTEGER NOT NULL +); + +CREATE TABLE syncer_bans ( + net_cidr TEXT PRIMARY KEY NOT NULL, + expiration INTEGER NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); + +CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + index_mode INTEGER, -- the mode of the data store + last_indexed_tip BLOB NOT NULL, -- the last chain index that was processed + element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree +);` + +func TestMigrationConsistency(t *testing.T) { + fp := filepath.Join(t.TempDir(), "hostd.sqlite3") + db, err := sql.Open("sqlite3", sqliteFilepath(fp)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if _, err := db.Exec(initialSchema); err != nil { + t.Fatal(err) + } + + // initialize the settings table + _, err = db.Exec(`INSERT INTO global_settings (id, db_version, index_mode, element_num_leaves, last_indexed_tip) VALUES (0, 1, 0, 0, ?)`, encode(types.ChainIndex{})) + if err != nil { + t.Fatal(err) + } + + if err := db.Close(); err != nil { + t.Fatal(err) + } + + expectedVersion := int64(len(migrations) + 1) + log := zaptest.NewLogger(t) + store, err := OpenDatabase(fp, log) + if err != nil { + t.Fatal(err) + } + defer store.Close() + v := getDBVersion(store.db) + if v != expectedVersion { + t.Fatalf("expected version %d, got %d", expectedVersion, v) + } else if err := store.Close(); err != nil { + t.Fatal(err) + } + + // ensure the database does not change version when opened again + store, err = OpenDatabase(fp, log) + if err != nil { + t.Fatal(err) + } + defer store.Close() + v = getDBVersion(store.db) + if v != expectedVersion { + t.Fatalf("expected version %d, got %d", expectedVersion, v) + } + + fp2 := filepath.Join(t.TempDir(), "hostd.sqlite3") + baseline, err := OpenDatabase(fp2, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + defer baseline.Close() + + getTableIndices := func(db *sql.DB) (map[string]bool, error) { + const query = `SELECT name, tbl_name, sql FROM sqlite_schema WHERE type='index'` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + indices := make(map[string]bool) + for rows.Next() { + var name, table string + var sqlStr sql.NullString // auto indices have no sql + if err := rows.Scan(&name, &table, &sqlStr); err != nil { + return nil, err + } + indices[fmt.Sprintf("%s.%s.%s", name, table, sqlStr.String)] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return indices, nil + } + + // ensure the migrated database has the same indices as the baseline + baselineIndices, err := getTableIndices(baseline.db) + if err != nil { + t.Fatal(err) + } + + migratedIndices, err := getTableIndices(store.db) + if err != nil { + t.Fatal(err) + } + + for k := range baselineIndices { + if !migratedIndices[k] { + t.Errorf("missing index %s", k) + } + } + + for k := range migratedIndices { + if !baselineIndices[k] { + t.Errorf("unexpected index %s", k) + } + } + + getTables := func(db *sql.DB) (map[string]bool, error) { + const query = `SELECT name FROM sqlite_schema WHERE type='table'` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + tables := make(map[string]bool) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + tables[name] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return tables, nil + } + + // ensure the migrated database has the same tables as the baseline + baselineTables, err := getTables(baseline.db) + if err != nil { + t.Fatal(err) + } + + migratedTables, err := getTables(store.db) + if err != nil { + t.Fatal(err) + } + + for k := range baselineTables { + if !migratedTables[k] { + t.Errorf("missing table %s", k) + } + } + for k := range migratedTables { + if !baselineTables[k] { + t.Errorf("unexpected table %s", k) + } + } + + // ensure each table has the same columns as the baseline + getTableColumns := func(db *sql.DB, table string) (map[string]bool, error) { + query := fmt.Sprintf(`PRAGMA table_info(%s)`, table) // cannot use parameterized query for PRAGMA statements + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + columns := make(map[string]bool) + for rows.Next() { + var cid int + var name, colType string + var defaultValue sql.NullString + var notNull bool + var primaryKey int // composite keys are indices + if err := rows.Scan(&cid, &name, &colType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, err + } + // column ID is ignored since it may not match between the baseline and migrated databases + key := fmt.Sprintf("%s.%s.%s.%t.%d", name, colType, defaultValue.String, notNull, primaryKey) + columns[key] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return columns, nil + } + + for k := range baselineTables { + baselineColumns, err := getTableColumns(baseline.db, k) + if err != nil { + t.Fatal(err) + } + migratedColumns, err := getTableColumns(store.db, k) + if err != nil { + t.Fatal(err) + } + + for c := range baselineColumns { + if !migratedColumns[c] { + t.Errorf("missing column %s.%s", k, c) + } + } + + for c := range migratedColumns { + if !baselineColumns[c] { + t.Errorf("unexpected column %s.%s", k, c) + } + } + } +} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 1237413..a9dedf6 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/mattn/go-sqlite3" "go.sia.tech/walletd/wallet" "go.uber.org/zap" "lukechampine.com/frand" @@ -105,19 +106,47 @@ func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { return nil } +func integrityCheck(db *sql.DB, log *zap.Logger) error { + rows, err := db.Query("PRAGMA integrity_check") + if err != nil { + return fmt.Errorf("failed to run integrity check: %w", err) + } + defer rows.Close() + var hasErrors bool + for rows.Next() { + var result string + if err := rows.Scan(&result); err != nil { + return fmt.Errorf("failed to scan integrity check result: %w", err) + } else if result != "ok" { + log.Error("integrity check failed", zap.String("result", result)) + hasErrors = true + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to iterate integrity check results: %w", err) + } else if hasErrors { + return errors.New("integrity check failed") + } + return nil +} + // OpenDatabase creates a new SQLite store and initializes the database. If the // database does not exist, it is created. func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { db, err := sql.Open("sqlite3", sqliteFilepath(fp)) if err != nil { return nil, err + } else if err := integrityCheck(db, log.Named("integrity")); err != nil { + return nil, fmt.Errorf("integrity check failed: %w", err) } store := &Store{ db: db, log: log, } if err := store.init(); err != nil { - return nil, fmt.Errorf("failed to initialize database: %w", err) + return nil, err } + sqliteVersion, _, _ := sqlite3.Version() + log.Debug("database initialized", zap.String("sqliteVersion", sqliteVersion), zap.Int("schemaVersion", len(migrations)+1), zap.String("path", fp)) return store, nil } From 88d98cd90f0b4c0e6df7c7aa0ae82422c376ba6e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 16 Jan 2025 09:11:19 -0800 Subject: [PATCH 333/630] Update persist/sqlite/migrations_test.go Co-authored-by: Peter-Jan Brone --- persist/sqlite/migrations_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/migrations_test.go b/persist/sqlite/migrations_test.go index 91ef642..52813dd 100644 --- a/persist/sqlite/migrations_test.go +++ b/persist/sqlite/migrations_test.go @@ -126,7 +126,7 @@ CREATE TABLE global_settings ( );` func TestMigrationConsistency(t *testing.T) { - fp := filepath.Join(t.TempDir(), "hostd.sqlite3") + fp := filepath.Join(t.TempDir(), "walletd.sqlite3") db, err := sql.Open("sqlite3", sqliteFilepath(fp)) if err != nil { t.Fatal(err) From eb2bc57d3d7baac4e1efba7653efd236147c7e30 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Thu, 16 Jan 2025 09:11:24 -0800 Subject: [PATCH 334/630] Update persist/sqlite/migrations_test.go Co-authored-by: Peter-Jan Brone --- persist/sqlite/migrations_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/migrations_test.go b/persist/sqlite/migrations_test.go index 52813dd..369e05e 100644 --- a/persist/sqlite/migrations_test.go +++ b/persist/sqlite/migrations_test.go @@ -172,7 +172,7 @@ func TestMigrationConsistency(t *testing.T) { t.Fatalf("expected version %d, got %d", expectedVersion, v) } - fp2 := filepath.Join(t.TempDir(), "hostd.sqlite3") + fp2 := filepath.Join(t.TempDir(), "walletd.sqlite3") baseline, err := OpenDatabase(fp2, zap.NewNop()) if err != nil { t.Fatal(err) From 16b1f6dd531d356a5e0f02185990243c93f0c6c6 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 16 Jan 2025 10:38:44 -0800 Subject: [PATCH 335/630] chore: update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index df24630..49178b0 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.9.0 - go.sia.tech/coreutils v0.9.1 + go.sia.tech/coreutils v0.10.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index f1a03f4..57a77f0 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.9.0 h1:qV7V8nkNaPvBEhkbwgrETTkb7JCMcAnKUQt9nUumP4k= go.sia.tech/core v0.9.0/go.mod h1:3NAvYHuzAZg9vP6pyIMOxjTkgHBQ3vx9cXTqRF6oEa4= -go.sia.tech/coreutils v0.9.1 h1:2SukWrF9o18HIG+BNmNk4SZw48+h+cM0gR8jMtG2cQ4= -go.sia.tech/coreutils v0.9.1/go.mod h1:A/tNYSBdryxCFaIpvW04YvdQ4e2j8iGuHoIw70+ZYXc= +go.sia.tech/coreutils v0.10.0 h1:uKkxq2llz49vDRUDdGlPbHCMKEf5OJhCDkGxIZiLedA= +go.sia.tech/coreutils v0.10.0/go.mod h1:/m6/PFV377MeJ42uO1xd+4//B1TNaQYF7DURrILHDvA= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= From 24b3faed518316b2a9767feeb05aaa1a2594f5aa Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 16 Jan 2025 10:38:50 -0800 Subject: [PATCH 336/630] ci: fix publish --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 42468b3..5849c36 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,7 @@ jobs: secrets: inherit with: linux-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' - windows-build-args: -tags='timetzdata netgo -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + windows-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' macos-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w' cgo-enabled: 1 project: walletd From 8a532424b7bbf44be1aba6ffe81267d5539dc261 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:32:39 +0000 Subject: [PATCH 337/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.9.0 to 0.9.1 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.9.0...v0.9.1) Updates `go.sia.tech/coreutils` from 0.10.0 to 0.10.1 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.10.0...v0.10.1) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 49178b0..def24bb 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.9.0 - go.sia.tech/coreutils v0.10.0 + go.sia.tech/core v0.9.1 + go.sia.tech/coreutils v0.10.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 @@ -24,7 +24,7 @@ require ( go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 57a77f0..3516842 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,10 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.9.0 h1:qV7V8nkNaPvBEhkbwgrETTkb7JCMcAnKUQt9nUumP4k= -go.sia.tech/core v0.9.0/go.mod h1:3NAvYHuzAZg9vP6pyIMOxjTkgHBQ3vx9cXTqRF6oEa4= -go.sia.tech/coreutils v0.10.0 h1:uKkxq2llz49vDRUDdGlPbHCMKEf5OJhCDkGxIZiLedA= -go.sia.tech/coreutils v0.10.0/go.mod h1:/m6/PFV377MeJ42uO1xd+4//B1TNaQYF7DURrILHDvA= +go.sia.tech/core v0.9.1 h1:p65iVQP4OnLRvPHBbZDhUR0LFserNIY82M/4de/gNPo= +go.sia.tech/core v0.9.1/go.mod h1:7buI+3k5xO+9PdzBQJlogOAc5h+twDUxEpV6EuXWZ5A= +go.sia.tech/coreutils v0.10.1 h1:qs6JIUhzQGcWYdMoE0KURz8g+Wt+OI65KMmyc4or/DA= +go.sia.tech/coreutils v0.10.1/go.mod h1:99k+BlLKYsKHNdZAr5KqYIhoamPEbwhKZdq4FDV4HtU= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= @@ -28,8 +28,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= From 4b6c04568ffdae08ba78298cbca1bfd2f852df4b Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 16 Jan 2025 11:09:55 -0800 Subject: [PATCH 338/630] sqlite: skip integrity check --- persist/sqlite/store.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index a9dedf6..e64b512 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -136,8 +136,6 @@ func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { db, err := sql.Open("sqlite3", sqliteFilepath(fp)) if err != nil { return nil, err - } else if err := integrityCheck(db, log.Named("integrity")); err != nil { - return nil, fmt.Errorf("integrity check failed: %w", err) } store := &Store{ db: db, From 3d4f781c0ff55e25814a625431fa1b181bead168 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 24 Jan 2025 13:40:37 -0800 Subject: [PATCH 339/630] api, sqlite, wallet: add basis to UTXO API responses --- ...erkle_proof_basis_to_utxo_api_responses.md | 39 ++++ api/api.go | 27 +++ api/api_test.go | 169 ++++++++++++++++-- api/client.go | 40 +++-- api/server.go | 57 ++++-- persist/sqlite/addresses.go | 14 +- wallet/addresses.go | 4 +- wallet/manager.go | 4 +- wallet/wallet_test.go | 60 ++++--- 9 files changed, 346 insertions(+), 68 deletions(-) create mode 100644 .changeset/add_merkle_proof_basis_to_utxo_api_responses.md diff --git a/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md b/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md new file mode 100644 index 0000000..94ef47f --- /dev/null +++ b/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md @@ -0,0 +1,39 @@ +--- +default: major +--- + +# Add Merkle Proof Basis to UTXO API Responses + +Changes the response to include the Merkle proof basis for the following endpoints: +- `[GET] /addresses/:address/outputs/siacoin` +- `[GET] /addresses/:address/outputs/siafund` +- `[GET] /wallets/:id/outputs/siacoin` +- `[GET] /wallets/:id/outputs/siafund` + + +```json +{ + "basis": { + "height": 1, + "id": "f362385eea61f81627f283a31af9faf6417fbb88d53b794639a34e18515996e9" + }, + "outputs": [ + { + "id": "ed556177482e70822a5dcad9343efb51998425884788415349bef8eba7e063ae", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "01048fc792904f156844a5524671304d3a020861da144afa4acc6553db63c1fd", + "33efdfaf9bb212842292ab6f298c454e1b3d412aa7beb7efdccdfccf09f5b4ee", + "102345919e408540d240460b0d84aa2f6da9a3d8f74765fd7c6daae6e46dd7f3" + ] + }, + "siacoinOutput": { + "value": "500000000000000000000000", + "address": "fbfc3d034b1eb45f63e0087571ec1f3028a9a2f8c180381d47713e6112467d91f474059476f2" + }, + "maturityHeight": 0 + } + ] +} +``` \ No newline at end of file diff --git a/api/api.go b/api/api.go index 407ff36..49d3422 100644 --- a/api/api.go +++ b/api/api.go @@ -46,6 +46,19 @@ type TxpoolTransactionsResponse struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } +// TxpoolUpdateV2TransactionsRequest is the request type for /txpool/transactions/v2/basis. +type TxpoolUpdateV2TransactionsRequest struct { + Basis types.ChainIndex `json:"basis"` + Target types.ChainIndex `json:"target"` + Transactions []types.V2Transaction `json:"transactions"` +} + +// TxpoolUpdateV2TransactionsResponse is the response type for /txpool/transactions/v2/basis. +type TxpoolUpdateV2TransactionsResponse struct { + Basis types.ChainIndex `json:"basis"` + Transactions []types.V2Transaction `json:"transactions"` +} + // BalanceResponse is the response type for /wallets/:id/balance. type BalanceResponse wallet.Balance @@ -159,3 +172,17 @@ type DebugMineRequest struct { Blocks int `json:"blocks"` Address types.Address `json:"address"` } + +// SiacoinElementsResponse is the response type for any endpoint that returns +// siacoin UTXOs +type SiacoinElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []types.SiacoinElement `json:"outputs"` +} + +// SiafundElementsResponse is the response type for any endpoint that returns +// siafund UTXOs +type SiafundElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []types.SiafundElement `json:"outputs"` +} diff --git a/api/api_test.go b/api/api_test.go index 6e0fbe1..21342e2 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -421,11 +421,13 @@ func TestWallet(t *testing.T) { t.Fatal("transaction should appear in history") } - outputs, err := wc.SiacoinOutputs(0, 100) + outputs, basis, err := wc.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) + } else if basis != cm.Tip() { + t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) } // mine a block to add an immature balance @@ -618,11 +620,13 @@ func TestAddresses(t *testing.T) { t.Fatal("transaction should appear in history") } - outputs, err := c.AddressSiacoinOutputs(addr.Address, 0, 100) + outputs, basis, err := c.AddressSiacoinOutputs(addr.Address, 0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) + } else if basis != cm.Tip() { + t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) } // mine a block to add an immature balance @@ -775,14 +779,18 @@ func TestV2(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, err := primary.SiacoinOutputs(0, 100) + sces, basis, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) } if len(sces) == 0 { - sces, err = secondary.SiacoinOutputs(0, 100) + sces, basis, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) } key = secondaryPrivateKey dest = primaryAddress @@ -822,12 +830,14 @@ func TestV2(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, err := primary.SiacoinOutputs(0, 100) + sces, basis, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) } if len(sces) == 0 { - sces, err = secondary.SiacoinOutputs(0, 100) + sces, _, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -1077,7 +1087,7 @@ func TestP2P(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, err := primary.SiacoinOutputs(0, 100) + sces, _, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -1085,7 +1095,7 @@ func TestP2P(t *testing.T) { c = c2 key = secondaryPrivateKey dest = primaryAddress - sces, err = secondary.SiacoinOutputs(0, 100) + sces, _, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -1131,7 +1141,7 @@ func TestP2P(t *testing.T) { key := primaryPrivateKey dest := secondaryAddress pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, err := primary.SiacoinOutputs(0, 100) + sces, _, err := primary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -1139,7 +1149,7 @@ func TestP2P(t *testing.T) { c = c2 key = secondaryPrivateKey dest = primaryAddress - sces, err = secondary.SiacoinOutputs(0, 100) + sces, _, err = secondary.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } @@ -2162,3 +2172,142 @@ func TestAPINoContent(t *testing.T) { t.Fatalf("expected no content, got %v bytes", resp.ContentLength) } } + +func TestV2TransactionUpdateBasis(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V2Network() + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + l, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + ps, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + s := syncer.New(l, cm, ps, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l.Addr().String(), + }) + defer s.Close() + go s.Run(context.Background()) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + mineAndSync := func(addr types.Address, n int) { + testutil.MineBlocks(t, cm, addr, n) + waitForBlock(t, cm, ws) + } + + // create a wallet + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + + pk := types.GeneratePrivateKey() + policy := types.SpendPolicy{Type: types.PolicyTypePublicKey(pk.PublicKey())} + addr := policy.Address() + + err = wc.AddAddress(wallet.Address{ + Address: addr, + SpendPolicy: &policy, + }) + if err != nil { + t.Fatal(err) + } + + // fund the wallet + mineAndSync(addr, 5) + mineAndSync(types.VoidAddress, int(n.MaturityDelay)) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(100), Address: addr}, + }, nil, addr) + if err != nil { + t.Fatal(err) + } + parentTxn, basis := resp.Transaction, resp.Basis + + // sign the transaction + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + sigHash := cs.InputSigHash(parentTxn) + sig := pk.SignHash(sigHash) + for i := range parentTxn.SiacoinInputs { + parentTxn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + // broadcast the transaction + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { + t.Fatal(err) + } + + mineAndSync(addr, 1) + + // create a child transaction + sce := parentTxn.EphemeralSiacoinOutput(0) + childTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + childSigHash := cs.InputSigHash(childTxn) + childTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(childSigHash)} + + txnset := []types.V2Transaction{parentTxn, childTxn} + + basis, txnset, err = c.V2UpdateTransactionSetBasis(txnset, basis, cm.Tip()) + if err != nil { + t.Fatal(err) + } else if len(txnset) != 1 { + t.Fatalf("expected 1 transactions, got %v", len(txnset)) + } else if txnset[0].ID() != childTxn.ID() { + t.Fatalf("expected parent transaction to be removed") + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be %v, got %v", cm.Tip(), basis) + } + + if err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { + t.Fatal(err) + } + + mineAndSync(addr, 1) +} diff --git a/api/client.go b/api/client.go index 82ca049..456dbe1 100644 --- a/api/client.go +++ b/api/client.go @@ -62,6 +62,18 @@ func (c *Client) TxpoolTransactions() (basis types.ChainIndex, txns []types.Tran return resp.Basis, resp.Transactions, resp.V2Transactions, err } +// V2UpdateTransactionSetBasis updates a V2 transaction set's basis to the target index. +func (c *Client) V2UpdateTransactionSetBasis(txnset []types.V2Transaction, from, to types.ChainIndex) (types.ChainIndex, []types.V2Transaction, error) { + req := TxpoolUpdateV2TransactionsRequest{ + Basis: from, + Target: to, + Transactions: txnset, + } + var resp TxpoolUpdateV2TransactionsResponse + err := c.c.POST("/txpool/transactions/v2/basis", req, &resp) + return resp.Basis, resp.Transactions, err +} + // TxpoolParents returns the parents of a transaction that are currently in the // transaction pool. func (c *Client) TxpoolParents(txn types.Transaction) (resp []types.Transaction, err error) { @@ -226,15 +238,17 @@ func (c *Client) AddressUnconfirmedEvents(addr types.Address) (resp []wallet.Eve } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) (resp []types.SiacoinElement, err error) { - err = c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) - return +func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + var resp SiacoinElementsResponse + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) (resp []types.SiafundElement, err error) { - err = c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) - return +func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + var resp SiafundElementsResponse + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } // Event returns the event with the specified ID. @@ -288,15 +302,17 @@ func (c *WalletClient) UnconfirmedEvents() (resp []wallet.Event, err error) { } // SiacoinOutputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) SiacoinOutputs(offset, limit int) (sc []types.SiacoinElement, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &sc) - return +func (c *WalletClient) SiacoinOutputs(offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + var resp SiacoinElementsResponse + err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } // SiafundOutputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) SiafundOutputs(offset, limit int) (sf []types.SiafundElement, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &sf) - return +func (c *WalletClient) SiafundOutputs(offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + var resp SiafundElementsResponse + err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } // Reserve reserves a set outputs for use in a transaction. diff --git a/api/server.go b/api/server.go index 563b62f..22cc5c6 100644 --- a/api/server.go +++ b/api/server.go @@ -110,8 +110,8 @@ type ( AddressBalance(address types.Address) (wallet.Balance, error) AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) AddressUnconfirmedEvents(address types.Address) ([]wallet.Event, error) - AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, error) - AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, error) + AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) Events(eventIDs []types.Hash256) ([]wallet.Event, error) @@ -310,6 +310,22 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { jc.EmptyResonse() } +func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { + var req TxpoolUpdateV2TransactionsRequest + if jc.Decode(&req) != nil { + return + } + + txnset, err := s.cm.UpdateV2TransactionSet(req.Transactions, req.Basis, req.Target) + if jc.Check("couldn't update v2 transaction set", err) != nil { + return + } + jc.Encode(TxpoolUpdateV2TransactionsResponse{ + Basis: req.Target, + Transactions: txnset, + }) +} + func (s *server) walletsHandler(jc jape.Context) { wallets, err := s.wm.Wallets() if jc.Check("couldn't load wallets", err) != nil { @@ -532,12 +548,15 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { return } - scos, _, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) + scos, basis, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } - jc.Encode(scos) + jc.Encode(SiacoinElementsResponse{ + Basis: basis, + Outputs: scos, + }) } func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { @@ -551,11 +570,14 @@ func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { return } - sfos, _, err := s.wm.UnspentSiafundOutputs(id, offset, limit) + sfos, basis, err := s.wm.UnspentSiafundOutputs(id, offset, limit) if jc.Check("couldn't load siacoin outputs", err) != nil { return } - jc.Encode(sfos) + jc.Encode(SiafundElementsResponse{ + Basis: basis, + Outputs: sfos, + }) } func (s *server) walletsReserveHandler(jc jape.Context) { @@ -1111,11 +1133,14 @@ func (s *server) addressesAddrOutputsSCHandler(jc jape.Context) { return } - utxos, err := s.wm.AddressSiacoinOutputs(addr, offset, limit) + utxos, basis, err := s.wm.AddressSiacoinOutputs(addr, offset, limit) if jc.Check("couldn't load utxos", err) != nil { return } - jc.Encode(utxos) + jc.Encode(SiacoinElementsResponse{ + Basis: basis, + Outputs: utxos, + }) } func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { @@ -1129,11 +1154,14 @@ func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { return } - utxos, err := s.wm.AddressSiafundOutputs(addr, offset, limit) + utxos, basis, err := s.wm.AddressSiafundOutputs(addr, offset, limit) if jc.Check("couldn't load utxos", err) != nil { return } - jc.Encode(utxos) + jc.Encode(SiafundElementsResponse{ + Basis: basis, + Outputs: utxos, + }) } func (s *server) eventsHandlerGET(jc jape.Context) { @@ -1296,10 +1324,11 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /syncer/peers": wrapPublicAuthHandler(srv.syncerPeersHandler), "POST /syncer/broadcast/block": wrapPublicAuthHandler(srv.syncerBroadcastBlockHandler), - "GET /txpool/transactions": wrapPublicAuthHandler(srv.txpoolTransactionsHandler), - "GET /txpool/fee": wrapPublicAuthHandler(srv.txpoolFeeHandler), - "POST /txpool/parents": wrapPublicAuthHandler(srv.txpoolParentsHandler), - "POST /txpool/broadcast": wrapPublicAuthHandler(srv.txpoolBroadcastHandler), + "GET /txpool/transactions": wrapPublicAuthHandler(srv.txpoolTransactionsHandler), + "POST /txpool/transactions/v2/basis": wrapPublicAuthHandler(srv.txpoolV2TransactionsBasisHandler), + "GET /txpool/fee": wrapPublicAuthHandler(srv.txpoolFeeHandler), + "POST /txpool/parents": wrapPublicAuthHandler(srv.txpoolParentsHandler), + "POST /txpool/broadcast": wrapPublicAuthHandler(srv.txpoolBroadcastHandler), "GET /addresses/:addr/balance": wrapPublicAuthHandler(srv.addressesAddrBalanceHandler), "GET /addresses/:addr/events": wrapPublicAuthHandler(srv.addressesAddrEventsHandlerGET), diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 526e08e..e3e1ebe 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -73,7 +73,7 @@ LIMIT $2 OFFSET $3` } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se @@ -113,13 +113,18 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIn siacoins[i].StateElement.MerkleProof = proof } } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } return nil }) return } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) { +func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se @@ -158,6 +163,11 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) siafunds[i].StateElement.MerkleProof = proof } } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } return nil }) return diff --git a/wallet/addresses.go b/wallet/addresses.go index 2358631..e14dae2 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -12,12 +12,12 @@ func (m *Manager) AddressBalance(address types.Address) (balance Balance, err er } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) (siacoins []types.SiacoinElement, err error) { +func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { return m.store.AddressSiacoinOutputs(address, m.chain.Tip(), offset, limit) } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) { +func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { return m.store.AddressSiafundOutputs(address, offset, limit) } diff --git a/wallet/manager.go b/wallet/manager.go index d679fb3..43ad162 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -84,8 +84,8 @@ type ( AddressBalance(address types.Address) (balance Balance, err error) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) - AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, err error) - AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, err error) + AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) Events(eventIDs []types.Hash256) ([]Event, error) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 73bfc16..a5ad9e4 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1654,7 +1654,7 @@ func TestFullIndex(t *testing.T) { assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) // send half siacoins to the second address - utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, _, err := wm.AddressSiacoinOutputs(addr, 0, 100) if err != nil { t.Fatal(err) } @@ -1711,7 +1711,7 @@ func TestFullIndex(t *testing.T) { t.Fatalf("expected transaction event, got %v", events[0].Type) } - sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) + sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) if err != nil { t.Fatal(err) } @@ -1888,9 +1888,11 @@ func TestEvents(t *testing.T) { assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) // send half siacoins to the second address - utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") } policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) @@ -1912,10 +1914,10 @@ func TestEvents(t *testing.T) { } txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} - if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - waitForBlock(t, cm, db) + mineAndSync(t, cm, db, types.VoidAddress, 1) assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, 0) assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) @@ -1957,7 +1959,7 @@ func TestEvents(t *testing.T) { t.Fatalf("expected event %v to match %v", expected, events2[0]) } - sf, err := wm.AddressSiafundOutputs(addr2, 0, 100) + sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) if err != nil { t.Fatal(err) } @@ -2687,9 +2689,11 @@ func TestScanV2(t *testing.T) { t.Fatal(err) } - utxos, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") } // spend the payout @@ -2708,10 +2712,10 @@ func TestScanV2(t *testing.T) { } txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} - if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - waitForBlock(t, cm, db) + mineAndSync(t, cm, db, types.VoidAddress, 1) // check that the first address has a balance of zero if err := checkBalance(expectedBalance2, types.ZeroCurrency); err != nil { @@ -3307,12 +3311,14 @@ func TestEventTypes(t *testing.T) { } defer wm.Close() - spendableSiacoinUTXOs := func() []types.SiacoinElement { + spendableSiacoinUTXOs := func(t *testing.T) ([]types.SiacoinElement, types.ChainIndex) { t.Helper() - sces, err := wm.AddressSiacoinOutputs(addr, 0, 100) + sces, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") } filtered := sces[:0] height := cm.Tip().Height @@ -3325,7 +3331,7 @@ func TestEventTypes(t *testing.T) { sort.Slice(filtered, func(i, j int) bool { return filtered[i].SiacoinOutput.Value.Cmp(filtered[j].SiacoinOutput.Value) < 0 }) - return filtered + return filtered, basis } assertEvent := func(t *testing.T, id types.Hash256, eventType string, expectedInflow, expectedOutflow types.Currency, maturityHeight uint64) { @@ -3364,7 +3370,7 @@ func TestEventTypes(t *testing.T) { // v1 transaction t.Run("v1 transaction", func(t *testing.T) { - sce := spendableSiacoinUTXOs() + sce, _ := spendableSiacoinUTXOs(t) // v1 only supports unlock conditions uc := types.StandardUnlockConditions(pk.PublicKey()) @@ -3406,7 +3412,7 @@ func TestEventTypes(t *testing.T) { // v1 contract resolution - only one type of resolution is supported. // The only difference is `missed == true` or `missed == false` - sce := spendableSiacoinUTXOs() + sce, _ := spendableSiacoinUTXOs(t) uc := types.StandardUnlockConditions(pk.PublicKey()) // create a storage contract @@ -3461,7 +3467,7 @@ func TestEventTypes(t *testing.T) { }) t.Run("v2 transaction", func(t *testing.T) { - sce := spendableSiacoinUTXOs() + sce, basis := spendableSiacoinUTXOs(t) // using the UnlockConditions policy for brevity policy := types.SpendPolicy{ @@ -3486,7 +3492,7 @@ func TestEventTypes(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } // mine a block to confirm the transaction @@ -3495,7 +3501,7 @@ func TestEventTypes(t *testing.T) { }) t.Run("v2 contract resolution - expired", func(t *testing.T) { - sce := spendableSiacoinUTXOs() + sce, basis := spendableSiacoinUTXOs(t) // using the UnlockConditions policy for brevity policy := types.SpendPolicy{ @@ -3544,7 +3550,7 @@ func TestEventTypes(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } // current tip @@ -3586,7 +3592,7 @@ func TestEventTypes(t *testing.T) { }) t.Run("v2 contract resolution - storage proof", func(t *testing.T) { - sce := spendableSiacoinUTXOs() + sce, basis := spendableSiacoinUTXOs(t) // using the UnlockConditions policy for brevity policy := types.SpendPolicy{ @@ -3635,7 +3641,7 @@ func TestEventTypes(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } // current tip @@ -3683,7 +3689,7 @@ func TestEventTypes(t *testing.T) { }) t.Run("v2 contract resolution - renewal", func(t *testing.T) { - sces := spendableSiacoinUTXOs() + sces, basis := spendableSiacoinUTXOs(t) // using the UnlockConditions policy for brevity policy := types.SpendPolicy{ @@ -3732,7 +3738,7 @@ func TestEventTypes(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } // current tip @@ -3778,7 +3784,7 @@ func TestEventTypes(t *testing.T) { renewal.NewContract.RenterSignature = pk.SignHash(contractSigHash) renewal.NewContract.HostSignature = pk.SignHash(contractSigHash) - sces = spendableSiacoinUTXOs() + sces, basis = spendableSiacoinUTXOs(t) newContractValue := renterPayout.Add(cm.TipState().V2FileContractTax(renewal.NewContract)) // create the renewal transaction @@ -3805,7 +3811,7 @@ func TestEventTypes(t *testing.T) { resolutionTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(resolutionTxnSigHash)} // broadcast the renewal - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{resolutionTxn}); err != nil { t.Fatal(err) } mineBlock(1, types.VoidAddress) @@ -3813,9 +3819,11 @@ func TestEventTypes(t *testing.T) { }) t.Run("siafund claim", func(t *testing.T) { - sfe, err := wm.AddressSiafundOutputs(addr, 0, 100) + sfe, basis, err := wm.AddressSiafundOutputs(addr, 0, 100) if err != nil { t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") } policy := types.SpendPolicy{ @@ -3842,7 +3850,7 @@ func TestEventTypes(t *testing.T) { claimValue := cm.TipState().SiafundTaxRevenue // broadcast the transaction - if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } // mine a block to confirm the transaction From 99696057e4dd4d2f43dd530d30aa1cdc8c34e8f8 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 4 Feb 2025 14:17:57 -0800 Subject: [PATCH 340/630] add basis to fund responses --- .changeset/add_basis_to_wallet_fund_endpoints.md | 5 +++++ api/api.go | 1 + api/server.go | 6 ++++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 .changeset/add_basis_to_wallet_fund_endpoints.md diff --git a/.changeset/add_basis_to_wallet_fund_endpoints.md b/.changeset/add_basis_to_wallet_fund_endpoints.md new file mode 100644 index 0000000..b42fb16 --- /dev/null +++ b/.changeset/add_basis_to_wallet_fund_endpoints.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Add basis to wallet fund endpoints diff --git a/api/api.go b/api/api.go index 49d3422..e974bbb 100644 --- a/api/api.go +++ b/api/api.go @@ -98,6 +98,7 @@ type WalletFundSFRequest struct { // WalletFundResponse is the response type for /wallets/:id/fund. type WalletFundResponse struct { + Basis types.ChainIndex `json:"basis"` Transaction types.Transaction `json:"transaction"` ToSign []types.Hash256 `json:"toSign"` DependsOn []types.Transaction `json:"dependsOn"` diff --git a/api/server.go b/api/server.go index 22cc5c6..48afef9 100644 --- a/api/server.go +++ b/api/server.go @@ -624,7 +624,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false) + utxos, basis, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -652,6 +652,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { } jc.Encode(WalletFundResponse{ + Basis: basis, Transaction: txn, ToSign: toSign, DependsOn: s.cm.UnconfirmedParents(txn), @@ -664,7 +665,7 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, change, err := s.wm.SelectSiafundElements(id, wfr.Amount) + utxos, basis, change, err := s.wm.SelectSiafundElements(id, wfr.Amount) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } @@ -692,6 +693,7 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { toSign = append(toSign, types.Hash256(sce.ID)) } jc.Encode(WalletFundResponse{ + Basis: basis, Transaction: txn, ToSign: toSign, DependsOn: s.cm.UnconfirmedParents(txn), From 1f95e6431e5f44a468c65c56fc705e5d82dcf997 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Wed, 5 Feb 2025 11:47:44 -0500 Subject: [PATCH 341/630] mod: Update core dependency --- api/api_test.go | 12 +++--- cmd/walletd/node.go | 2 +- go.mod | 18 ++++++-- go.sum | 67 +++++++++++++++++++++++++----- wallet/update.go | 96 ++++++++++++++++++------------------------- wallet/wallet.go | 62 +++++++++++++++------------- wallet/wallet_test.go | 30 +++++--------- 7 files changed, 161 insertions(+), 126 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 21342e2..1c29341 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -956,7 +956,7 @@ func TestP2P(t *testing.T) { UniqueID: gateway.GenerateUniqueID(), NetAddress: l1.Addr().String(), }) - go s1.Run(context.Background()) + go s1.Run() defer s1.Close() c1 := runServer(t, cm1, s1, wm1) w1, err := c1.AddWallet(api.WalletUpdateRequest{Name: "primary"}) @@ -999,7 +999,7 @@ func TestP2P(t *testing.T) { UniqueID: gateway.GenerateUniqueID(), NetAddress: l2.Addr().String(), }, syncer.WithLogger(zaptest.NewLogger(t))) - go s2.Run(context.Background()) + go s2.Run() defer s2.Close() c2 := runServer(t, cm2, s2, wm2) @@ -1964,7 +1964,7 @@ func TestDebugMine(t *testing.T) { NetAddress: l.Addr().String(), }) defer s.Close() - go s.Run(context.Background()) + go s.Run() wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) if err != nil { @@ -2025,7 +2025,7 @@ func TestAPISecurity(t *testing.T) { NetAddress: syncerListener.Addr().String(), }) defer s.Close() - go s.Run(context.Background()) + go s.Run() wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) if err != nil { @@ -2140,7 +2140,7 @@ func TestAPINoContent(t *testing.T) { NetAddress: l.Addr().String(), }) defer s.Close() - go s.Run(context.Background()) + go s.Run() wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) if err != nil { @@ -2207,7 +2207,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { NetAddress: l.Addr().String(), }) defer s.Close() - go s.Run(context.Background()) + go s.Run() wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) if err != nil { diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index e9a5b57..c0ec30c 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -193,7 +193,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu s := syncer.New(syncerListener, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) defer s.Close() - go s.Run(ctx) + go s.Run() wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(cfg.Index.Mode), wallet.WithSyncBatchSize(cfg.Index.BatchSize)) if err != nil { diff --git a/go.mod b/go.mod index def24bb..c8a5ab9 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.9.1 - go.sia.tech/coreutils v0.10.1 + go.sia.tech/core v0.10.0 + go.sia.tech/coreutils v0.11.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 @@ -19,12 +19,24 @@ require ( ) require ( + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.49.0 // indirect + github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.3.11 // indirect go.sia.tech/mux v1.3.0 // indirect - go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect + go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect + go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.32.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 3516842..ad9e4da 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,92 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94= +github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.9.1 h1:p65iVQP4OnLRvPHBbZDhUR0LFserNIY82M/4de/gNPo= -go.sia.tech/core v0.9.1/go.mod h1:7buI+3k5xO+9PdzBQJlogOAc5h+twDUxEpV6EuXWZ5A= -go.sia.tech/coreutils v0.10.1 h1:qs6JIUhzQGcWYdMoE0KURz8g+Wt+OI65KMmyc4or/DA= -go.sia.tech/coreutils v0.10.1/go.mod h1:99k+BlLKYsKHNdZAr5KqYIhoamPEbwhKZdq4FDV4HtU= +go.sia.tech/core v0.10.0 h1:EK/JtUqbATeZm+K+a7fKtCCtPYinu9p45pOFSTUHx6o= +go.sia.tech/core v0.10.0/go.mod h1:49Ti4JnaLjQXXjEjRnO5HyLKDue1GHuFyz3XECM2Mlw= +go.sia.tech/coreutils v0.11.0 h1:6qSStFdKFjnzDg34o+ohy5BXD1IGjEowXn/WV+r7eW8= +go.sia.tech/coreutils v0.11.0/go.mod h1:7sh8UsgV/YSd8njHQeZgWd+u5Twr+4iuHmDuOVwj9hI= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= -go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= -go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= +go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= +go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= go.sia.tech/web/walletd v0.27.0 h1:oTCqqZHvvWbcy/jKMN7urBEFYXQs1+yVzuKu17vgQtk= go.sia.tech/web/walletd v0.27.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= diff --git a/wallet/update.go b/wallet/update.go index af410c7..7001b8a 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -93,43 +93,36 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e } // add new siacoin elements to the store - cau.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { - if (created && spent) || se.SiacoinOutput.Value.IsZero() { - return - } - - relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) - if err != nil { + for _, sced := range cau.SiacoinElementDiffs() { + sce := sced.SiacoinElement + if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { + continue + } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { panic(err) } else if !relevant { - return + continue } - - if spent { - applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, se) + if sced.Spent { + applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, sce) } else { - applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, se) - } - }) - - cau.ForEachSiafundElement(func(se types.SiafundElement, created, spent bool) { - if (created && spent) || se.SiafundOutput.Value == 0 { - return + applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, sce) } - - relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) - if err != nil { + } + for _, sfed := range cau.SiafundElementDiffs() { + sfe := sfed.SiafundElement + if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { + continue + } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { panic(err) } else if !relevant { - return + continue } - - if spent { - applied.SpentSiafundElements = append(applied.SpentSiafundElements, se) + if sfed.Spent { + applied.SpentSiafundElements = append(applied.SpentSiafundElements, sfe) } else { - applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, se) + applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, sfe) } - }) + } // add events relevant := func(addr types.Address) bool { @@ -176,47 +169,38 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. } } - cru.ForEachSiacoinElement(func(se types.SiacoinElement, created, spent bool) { - if created && spent { - return - } - - relevant, err := tx.AddressRelevant(se.SiacoinOutput.Address) - if err != nil { + for _, sced := range cru.SiacoinElementDiffs() { + sce := sced.SiacoinElement + if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { + continue + } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { panic(err) } else if !relevant { - return + continue } - - if spent { + if sced.Spent { // re-add any spent siacoin elements - reverted.UnspentSiacoinElements = append(reverted.UnspentSiacoinElements, se) + reverted.UnspentSiacoinElements = append(reverted.UnspentSiacoinElements, sce) } else { // delete any created siacoin elements - reverted.DeletedSiacoinElements = append(reverted.DeletedSiacoinElements, se) - } - }) - - cru.ForEachSiafundElement(func(se types.SiafundElement, created, spent bool) { - if created && spent { - return + reverted.DeletedSiacoinElements = append(reverted.DeletedSiacoinElements, sce) } - - relevant, err := tx.AddressRelevant(se.SiafundOutput.Address) - if err != nil { + } + for _, sfed := range cru.SiafundElementDiffs() { + sfe := sfed.SiafundElement + if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { + continue + } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { panic(err) } else if !relevant { - return + continue } - - if spent { - // re-add any spent siafund elements - reverted.UnspentSiafundElements = append(reverted.UnspentSiafundElements, se) + if sfed.Spent { + reverted.UnspentSiafundElements = append(reverted.UnspentSiafundElements, sfe) } else { - // delete any created siafund elements - reverted.DeletedSiafundElements = append(reverted.DeletedSiafundElements, se) + reverted.DeletedSiafundElements = append(reverted.DeletedSiafundElements, sfe) } - }) + } if err := tx.RevertIndex(revertedIndex, reverted); err != nil { return fmt.Errorf("failed to revert index: %w", err) diff --git a/wallet/wallet.go b/wallet/wallet.go index e28e5a0..1430582 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -79,10 +79,10 @@ type ( // A ChainUpdate is a set of changes to the consensus state. ChainUpdate interface { - ForEachSiacoinElement(func(sce types.SiacoinElement, created, spent bool)) - ForEachSiafundElement(func(sfe types.SiafundElement, created, spent bool)) - ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool)) - ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) + SiacoinElementDiffs() []consensus.SiacoinElementDiff + SiafundElementDiffs() []consensus.SiafundElementDiff + FileContractElementDiffs() []consensus.FileContractElementDiff + V2FileContractElementDiffs() []consensus.V2FileContractElementDiff } ) @@ -153,18 +153,18 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f }) } - anythingRelevant := func() (ok bool) { - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { - if ok || relevant(sce.SiacoinOutput.Address) { - ok = true + anythingRelevant := func() bool { + for _, sced := range cu.SiacoinElementDiffs() { + if relevant(sced.SiacoinElement.SiacoinOutput.Address) { + return true } - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, _, _ bool) { - if ok || relevant(sfe.SiafundOutput.Address) { - ok = true + } + for _, sfed := range cu.SiafundElementDiffs() { + if relevant(sfed.SiafundElement.SiafundOutput.Address) { + return true } - }) - return + } + return false }() if !anythingRelevant { return nil @@ -173,14 +173,16 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f // collect all elements sces := make(map[types.SiacoinOutputID]types.SiacoinElement) sfes := make(map[types.SiafundOutputID]types.SiafundElement) - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, _, _ bool) { + for _, sced := range cu.SiacoinElementDiffs() { + sce := sced.SiacoinElement sce.StateElement.MerkleProof = nil - sces[types.SiacoinOutputID(sce.ID)] = sce - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, _, _ bool) { + sces[sce.ID] = sce + } + for _, sfed := range cu.SiafundElementDiffs() { + sfe := sfed.SiafundElement sfe.StateElement.MerkleProof = nil - sfes[types.SiafundOutputID(sfe.ID)] = sfe - }) + sfes[sfe.ID] = sfe + } // handle v1 transactions for _, txn := range b.Transactions { @@ -294,14 +296,15 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f } // handle contracts - cu.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { - if !resolved { - return + for _, fced := range cu.FileContractElementDiffs() { + if !fced.Resolved { + continue } + fce := fced.FileContractElement fce.StateElement.MerkleProof = nil - if valid { + if fced.Valid { for i := range fce.FileContract.ValidProofOutputs { address := fce.FileContract.ValidProofOutputs[i].Address if !relevant(address) { @@ -330,13 +333,14 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f }, []types.Address{address}) } } - }) + } - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + for _, fced := range cu.V2FileContractElementDiffs() { + fce := fced.V2FileContractElement + res := fced.Resolution if res == nil { - return + continue } - fce.StateElement.MerkleProof = nil var missed bool @@ -367,7 +371,7 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f Missed: missed, }, []types.Address{fce.V2FileContract.RenterOutput.Address}) } - }) + } // handle block rewards for i := range b.MinerPayouts { diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index a5ad9e4..d9d3c18 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -3566,11 +3566,8 @@ func TestEventTypes(t *testing.T) { } // get the confirmed file contract element - var fce types.V2FileContractElement - applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { - fce = ele - }) - for _, cau := range applied { + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { cau.UpdateElementProof(&fce.StateElement) } @@ -3657,12 +3654,8 @@ func TestEventTypes(t *testing.T) { } // get the confirmed file contract element - var fce types.V2FileContractElement - applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { - fce = ele - }) - // update its proof - for _, cau := range applied { + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { cau.UpdateElementProof(&fce.StateElement) } // get the proof index element @@ -3754,11 +3747,8 @@ func TestEventTypes(t *testing.T) { } // get the confirmed file contract element - var fce types.V2FileContractElement - applied[0].ForEachV2FileContractElement(func(ele types.V2FileContractElement, _ bool, _ *types.V2FileContractElement, _ types.V2FileContractResolutionType) { - fce = ele - }) - for _, cau := range applied { + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { cau.UpdateElementProof(&fce.StateElement) } @@ -4445,11 +4435,11 @@ func TestReset(t *testing.T) { var siacoinElements []types.SiacoinElement for _, cau := range applied { - cau.ForEachSiacoinElement(func(sce types.SiacoinElement, created, spent bool) { - if created && sce.SiacoinOutput.Address == addr { - siacoinElements = append(siacoinElements, sce) + for _, sced := range cau.SiacoinElementDiffs() { + if sced.Created && sced.SiacoinElement.SiacoinOutput.Address == addr { + siacoinElements = append(siacoinElements, sced.SiacoinElement) } - }) + } } var expectedSiacoins, expectedImmature types.Currency From f9c1d01ee87bf16f35c67bbc7cad15ee26c69736 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 7 Feb 2025 10:39:46 -0800 Subject: [PATCH 342/630] document change --- ..._response_of_consensus_updates_endpoint.md | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 .changeset/simplified_response_of_consensus_updates_endpoint.md diff --git a/.changeset/simplified_response_of_consensus_updates_endpoint.md b/.changeset/simplified_response_of_consensus_updates_endpoint.md new file mode 100644 index 0000000..5eab007 --- /dev/null +++ b/.changeset/simplified_response_of_consensus_updates_endpoint.md @@ -0,0 +1,243 @@ +--- +default: major +--- + +# Simplified response of consensus updates endpoint + +The response of `/api/consensus/updates/:index` has been simplified to make it easier for developers to index chain state. + +```json +{ + "applied": [ + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "35b81e41f594d7faeb88bd8eaac2eaa68ce99fe1c8fe5f0cba8fafa65ab3a70e", + "stateElement": { + "leafIndex": 0, + "merkleProof": [ + "88052fa2d1e22e4a5542fed9686cdad3fbeccbc60d15d4fd36a7691d61add1e1" + ] + }, + "siacoinOutput": { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + }, + "maturityHeight": 0 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": [ + { + "siafundElement": { + "id": "69ad26a0fbd1a6985d2053246650bb3ba5f3491d818748b6c8562db1ddb2c45b", + "stateElement": { + "leafIndex": 1, + "merkleProof": [ + "837482a39d5bf66f07bae3b89191e4375b82c9f341ce6a17e22e14e0333ab9f6" + ] + }, + "siafundOutput": { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + }, + "claimStart": "0" + }, + "created": true, + "spent": false + } + ], + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "stateElement": { + "leafIndex": 2 + }, + "chainIndex": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + } + }, + "updatedLeaves": {}, + "treeGrowth": {}, + "oldNumLeaves": 0, + "numLeaves": 3 + }, + "state": { + "index": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + }, + "prevTimestamps": [ + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 0, + "oakTarget": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "1", + "difficulty": "4294967295", + "oakWork": "4294967297", + "elements": { + "numLeaves": 3, + "trees": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "2023-01-13T00:53:20-08:00", + "minerPayouts": [], + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] + } + }, + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "ca02d6807c92f61af94e626604615fbcdb471f38fcd8f3add6c6e6e0485ce090", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "siacoinOutput": { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + }, + "maturityHeight": 145 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": null, + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006", + "stateElement": { + "leafIndex": 4 + }, + "chainIndex": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + } + }, + "updatedLeaves": {}, + "treeGrowth": { + "0": [ + "190d98a7d8ff464e57f89dc916b155455ecf927f4c74b9edf5e80c103f052bfa", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ], + "1": [ + "2b082bec52801c1e61e5b0d0c1f5fc3925bd24e16d2f490afeb70374828586f1" + ] + }, + "oldNumLeaves": 3, + "numLeaves": 5 + }, + "state": { + "index": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + }, + "prevTimestamps": [ + "2023-01-13T08:18:19-08:00", + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 26699000000000, + "oakTarget": "000000008052201448053c59f99803e7a8165929036cd574d91425423191387c", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "4294967297", + "difficulty": "4294967295", + "oakWork": "8568459756", + "elements": { + "numLeaves": 5, + "trees": [ + "589fb425faa23be357492394813dc575505899d42d0b23a7162e1c68f7eeb227", + "750cc671d80aef6ee5c73344ba4e74eccda77d9f0cf51ed6237952b1d84bc336" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "nonce": 10689346, + "timestamp": "2023-01-13T08:18:19-08:00", + "minerPayouts": [ + { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + } + ], + "transactions": [ + { + "id": "1148417ad8fa6546646da6922618358210bc7a668ef7cb25f6a8a3605851bc7b", + "arbitraryData": [ + "Tm9uU2lhAAAAAAAAAAAAAClvJjNhfcbxtEfP2yfbBM4=" + ] + } + ] + } + } + ], + "reverted": null +} +``` \ No newline at end of file From 3c70a71d495c1d7e8aefa74303c5d1dae7f0831b Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 7 Feb 2025 13:27:54 -0800 Subject: [PATCH 343/630] sqlite: improve events query performance --- persist/sqlite/addresses.go | 19 ++++++-------- persist/sqlite/events.go | 27 ++++++++++---------- persist/sqlite/store.go | 24 ------------------ persist/sqlite/wallet.go | 50 +++++++++---------------------------- 4 files changed, 34 insertions(+), 86 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index e3e1ebe..642d52f 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -27,28 +27,25 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e // AddressEvents returns the events of a single address. func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - const query = ` -WITH last_chain_index AS ( - SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 -) -SELECT + var scanHeight uint64 + err := tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return fmt.Errorf("failed to get last indexed height: %w", err) + } + + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, - CASE - WHEN last_chain_index.height < ci.height THEN 0 - ELSE last_chain_index.height - ci.height - END AS confirmations, ev.event_type, ev.event_data FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) -CROSS JOIN last_chain_index WHERE sa.sia_address = $1 ORDER BY ev.maturity_height DESC, ev.id DESC LIMIT $2 OFFSET $3` @@ -60,7 +57,7 @@ LIMIT $2 OFFSET $3` defer rows.Close() for rows.Next() { - event, _, err := scanEvent(rows) + event, _, err := scanEvent(rows, scanHeight) if err != nil { return fmt.Errorf("failed to scan event: %w", err) } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index 1ecd94f..d0b4784 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -13,31 +13,28 @@ import ( // it is skipped. func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { + var scanHeight uint64 + err := tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return fmt.Errorf("failed to get last indexed height: %w", err) + } + // sqlite doesn't have easy support for IN clauses, use a statement since // the number of event IDs is likely to be small instead of dynamically // building the query - const query = ` -WITH last_chain_index AS ( - SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 -) -SELECT + const query = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, - CASE - WHEN last_chain_index.height < ci.height THEN 0 - ELSE last_chain_index.height - ci.height - END AS confirmations, ev.event_type, ev.event_data FROM events ev INNER JOIN event_addresses ea ON (ev.id = ea.event_id) INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) -CROSS JOIN last_chain_index WHERE ev.event_id = $1` stmt, err := tx.Prepare(query) @@ -48,7 +45,7 @@ WHERE ev.event_id = $1` events = make([]wallet.Event, 0, len(eventIDs)) for _, id := range eventIDs { - event, _, err := scanEvent(stmt.QueryRow(encode(id))) + event, _, err := scanEvent(stmt.QueryRow(encode(id)), scanHeight) if errors.Is(err, sql.ErrNoRows) { continue } else if err != nil { @@ -74,13 +71,17 @@ func decodeEventData[T wallet.EventPayout | return *v } -func scanEvent(s scanner) (ev wallet.Event, eventID int64, err error) { +func scanEvent(s scanner, scanHeight uint64) (ev wallet.Event, eventID int64, err error) { var eventBuf []byte - err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Confirmations, &ev.Type, &eventBuf) + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) if err != nil { return } + if scanHeight >= ev.Index.Height { + ev.Confirmations = 1 + scanHeight - ev.Index.Height + } + dec := types.NewBufDecoder(eventBuf) switch ev.Type { case wallet.EventTypeV1Transaction: diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index e64b512..1929df5 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -106,30 +106,6 @@ func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { return nil } -func integrityCheck(db *sql.DB, log *zap.Logger) error { - rows, err := db.Query("PRAGMA integrity_check") - if err != nil { - return fmt.Errorf("failed to run integrity check: %w", err) - } - defer rows.Close() - var hasErrors bool - for rows.Next() { - var result string - if err := rows.Scan(&result); err != nil { - return fmt.Errorf("failed to scan integrity check result: %w", err) - } else if result != "ok" { - log.Error("integrity check failed", zap.String("result", result)) - hasErrors = true - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("failed to iterate integrity check results: %w", err) - } else if hasErrors { - return errors.New("integrity check failed") - } - return nil -} - // OpenDatabase creates a new SQLite store and initializes the database. If the // database does not exist, it is created. func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 6176e96..47c6b08 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -699,54 +699,28 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e } func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - // the events query can be slow in full index mode for wallets with no - // events. Check if the wallet has events first. - const hasEventsQuery = `SELECT EXISTS ( - SELECT 1 - FROM event_addresses ea - INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id - WHERE wa.wallet_id=$1 -) AS has_events;` - var hasEvents bool - if err := tx.QueryRow(hasEventsQuery, id).Scan(&hasEvents); err != nil { - return nil, nil, err - } else if !hasEvents { - return nil, nil, nil + var scanHeight uint64 + err = tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return nil, nil, fmt.Errorf("failed to get last indexed height: %w", err) } - const eventsQuery = ` -WITH last_chain_index AS ( - SELECT last_indexed_height+1 AS height FROM global_settings LIMIT 1 -), -event_ids AS ( - SELECT - ev.id - FROM events ev - INNER JOIN event_addresses ea ON ev.id = ea.event_id - INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id - WHERE wa.wallet_id = $1 - GROUP BY ev.id - ORDER BY ev.maturity_height DESC, ev.id DESC - LIMIT $2 OFFSET $3 -) -SELECT + const eventsQuery = `SELECT ev.id, ev.event_id, ev.maturity_height, ev.date_created, ci.height, ci.block_id, - CASE - WHEN last_chain_index.height < ci.height THEN 0 - ELSE last_chain_index.height - ci.height - END AS confirmations, ev.event_type, ev.event_data -FROM events ev -INNER JOIN event_ids ei ON ev.id = ei.id +FROM events ev INDEXED BY events_maturity_height_id_idx -- force index to prevent temp-btree sorts +INNER JOIN event_addresses ea ON ev.id = ea.event_id +INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id INNER JOIN chain_indices ci ON ev.chain_index_id = ci.id -CROSS JOIN last_chain_index -ORDER BY ev.maturity_height DESC, ev.id DESC;` +WHERE wa.wallet_id = $1 +ORDER BY ev.maturity_height DESC, ev.id DESC +LIMIT $2 OFFSET $3;` rows, err := tx.Query(eventsQuery, id, limit, offset) if err != nil { @@ -755,7 +729,7 @@ ORDER BY ev.maturity_height DESC, ev.id DESC;` defer rows.Close() for rows.Next() { - event, eventID, err := scanEvent(rows) + event, eventID, err := scanEvent(rows, scanHeight) if err != nil { return nil, nil, fmt.Errorf("failed to scan event: %w", err) } From 7eb4367116fb80aaf6331b3a90b3a0cd85153b64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 16:50:49 +0000 Subject: [PATCH 344/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.28.0 to 0.29.0 - [Commits](https://github.com/golang/term/compare/v0.28.0...v0.29.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index c8a5ab9..51adebd 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 - golang.org/x/term v0.28.0 + golang.org/x/term v0.29.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -36,7 +36,7 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index ad9e4da..12beda1 100644 --- a/go.sum +++ b/go.sum @@ -71,10 +71,10 @@ golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= From b8955f4e5fab64ef423e907774b718acbc12f81b Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Feb 2025 08:48:06 -0800 Subject: [PATCH 345/630] update core and coreutils --- go.mod | 12 ++++++------ go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 51adebd..5517dd4 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.0 - go.sia.tech/coreutils v0.11.0 + go.sia.tech/core v0.10.1 + go.sia.tech/coreutils v0.11.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.27.0 go.uber.org/zap v1.27.0 @@ -26,17 +26,17 @@ require ( github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.49.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect - go.etcd.io/bbolt v1.3.11 // indirect + go.etcd.io/bbolt v1.4.0 // indirect go.sia.tech/mux v1.3.0 // indirect go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.32.0 // indirect + golang.org/x/crypto v0.33.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/sync v0.10.0 // indirect + golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 12beda1..3a95118 100644 --- a/go.sum +++ b/go.sum @@ -37,14 +37,14 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= -go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.10.0 h1:EK/JtUqbATeZm+K+a7fKtCCtPYinu9p45pOFSTUHx6o= -go.sia.tech/core v0.10.0/go.mod h1:49Ti4JnaLjQXXjEjRnO5HyLKDue1GHuFyz3XECM2Mlw= -go.sia.tech/coreutils v0.11.0 h1:6qSStFdKFjnzDg34o+ohy5BXD1IGjEowXn/WV+r7eW8= -go.sia.tech/coreutils v0.11.0/go.mod h1:7sh8UsgV/YSd8njHQeZgWd+u5Twr+4iuHmDuOVwj9hI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.sia.tech/core v0.10.1 h1:96lmgO50oKPiQU46H14Ga+6NYo6IB++VQ4DI3QCc6/o= +go.sia.tech/core v0.10.1/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk= +go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo= +go.sia.tech/coreutils v0.11.1/go.mod h1:vnY0haOx1InIQR0Pc5YAXDe4WnF6po8dv5bNP73CAnE= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= @@ -61,22 +61,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= From cf53b9d248feac98f328ab4d4039389169ce14c7 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 8 Feb 2025 11:18:02 -0800 Subject: [PATCH 346/630] sqlite: flatten event lookups --- persist/sqlite/addresses.go | 68 ++++++++++++++++++------------------ persist/sqlite/consensus.go | 4 +-- persist/sqlite/events.go | 39 +++++++++++++++++++++ persist/sqlite/init.sql | 3 +- persist/sqlite/migrations.go | 24 ++++++++++++- persist/sqlite/wallet.go | 47 +++++++++---------------- 6 files changed, 117 insertions(+), 68 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 642d52f..b5c9056 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -24,47 +24,47 @@ func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, e return } +func getAddressEvents(tx *txn, address types.Address, offset, limit int) (eventIDs []int64, err error) { + const query = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +WHERE sa.sia_address = $1 +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT $2 OFFSET $3;` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + eventIDs = append(eventIDs, id) + } + return eventIDs, rows.Err() +} + // AddressEvents returns the events of a single address. func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - var scanHeight uint64 - err := tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + dbIDs, err := getAddressEvents(tx, address, offset, limit) if err != nil { - return fmt.Errorf("failed to get last indexed height: %w", err) + return err } - const query = `SELECT - ev.id, - ev.event_id, - ev.maturity_height, - ev.date_created, - ci.height, - ci.block_id, - ev.event_type, - ev.event_data -FROM events ev INDEXED BY events_maturity_height_id_idx -- force the index to prevent temp-btree sorts -INNER JOIN event_addresses ea ON (ev.id = ea.event_id) -INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) -INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) -WHERE sa.sia_address = $1 -ORDER BY ev.maturity_height DESC, ev.id DESC -LIMIT $2 OFFSET $3` - - rows, err := tx.Query(query, encode(address), limit, offset) + events, err = getEventsByID(tx, dbIDs) if err != nil { - return err + return fmt.Errorf("failed to get events by ID: %w", err) } - defer rows.Close() - for rows.Next() { - event, _, err := scanEvent(rows, scanHeight) - if err != nil { - return fmt.Errorf("failed to scan event: %w", err) - } - event.Relevant = []types.Address{address} - events = append(events, event) + for i := range events { + events[i].Relevant = []types.Address{address} } - return rows.Err() + return nil }) return } @@ -72,7 +72,7 @@ LIMIT $2 OFFSET $3` // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE sa.sia_address=$1 AND se.maturity_height <= $2 AND se.spent_index_id IS NULL @@ -123,7 +123,7 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIn // AddressSiafundOutputs returns the unspent siafund outputs for an address. func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE sa.sia_address = $1 AND se.spent_index_id IS NULL @@ -197,7 +197,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 return se, nil } - siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 25973d6..8004259 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1111,7 +1111,7 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { } defer addrStmt.Close() - relevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id) VALUES ($1, $2) ON CONFLICT (event_id, address_id) DO NOTHING`) + relevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id, event_maturity_height) VALUES ($1, $2, $3) ON CONFLICT (event_id, address_id) DO NOTHING`) if err != nil { return fmt.Errorf("failed to prepare relevant address statement: %w", err) } @@ -1148,7 +1148,7 @@ func addEvents(tx *txn, events []wallet.Event, indexID int64) error { return fmt.Errorf("failed to get address: %w", err) } - _, err = relevantAddrStmt.Exec(eventID, addressID) + _, err = relevantAddrStmt.Exec(eventID, addressID, event.MaturityHeight) if err != nil { return fmt.Errorf("failed to add relevant address: %w", err) } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index d0b4784..a6b3754 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -71,6 +71,45 @@ func decodeEventData[T wallet.EventPayout | return *v } +func getEventsByID(tx *txn, eventIDs []int64) (events []wallet.Event, err error) { + var scanHeight uint64 + err = tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return nil, fmt.Errorf("failed to get last indexed height: %w", err) + } + + stmt, err := tx.Prepare(`SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + ev.event_type, + ev.event_data +FROM events ev +INNER JOIN event_addresses ea ON (ev.id = ea.event_id) +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) +WHERE ev.id=$1`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]wallet.Event, 0, len(eventIDs)) + for i, id := range eventIDs { + event, _, err := scanEvent(stmt.QueryRow(id), scanHeight) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return nil, fmt.Errorf("failed to query event %d: %w", i, err) + } + events = append(events, event) + } + return +} + func scanEvent(s scanner, scanHeight uint64) (ev wallet.Event, eventID int64, err error) { var eventBuf []byte err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index cf6a94b..4bc860f 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -67,11 +67,12 @@ CREATE INDEX events_maturity_height_id_idx ON events (maturity_height DESC, id D CREATE TABLE event_addresses ( event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + event_maturity_height INTEGER NOT NULL, -- flattened from events to improve query performance PRIMARY KEY (event_id, address_id) ); CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); -CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (event_id, address_id); +CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); CREATE TABLE wallets ( id INTEGER PRIMARY KEY, diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index c00bbd9..0aa0a8d 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,6 +7,27 @@ import ( "go.uber.org/zap" ) +func migrateVersion6(tx *txn, _ *zap.Logger) error { + const query = ` +CREATE TABLE event_addresses_new ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + event_maturity_height INTEGER NOT NULL, -- flattened from events to improve query performance + PRIMARY KEY (event_id, address_id) +); +INSERT INTO event_addresses_new (event_id, address_id, event_maturity_height) SELECT ea.event_id, ea.address_id, ev.maturity_height FROM event_addresses ea INNER JOIN events ev ON ea.event_id = ev.id; + +DROP TABLE event_addresses; + +ALTER TABLE event_addresses_new RENAME TO event_addresses; +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); +` + _, err := tx.Exec(query) + return err +} + // migrateVersion5 resets the database to trigger a full resync to switch // events from JSON to Sia encoding func migrateVersion5(tx *txn, _ *zap.Logger) error { @@ -66,7 +87,7 @@ CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements( siafund_value INTEGER NOT NULL, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), - spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ ); CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); @@ -163,4 +184,5 @@ var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion3, migrateVersion4, migrateVersion5, + migrateVersion6, } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 47c6b08..06e2733 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -53,12 +53,16 @@ WHERE wa.wallet_id=? AND ea.event_id=?`) // WalletEvents returns the events relevant to a wallet, sorted by height descending. func (s *Store) WalletEvents(id wallet.ID, offset, limit int) (events []wallet.Event, err error) { err = s.transaction(func(tx *txn) error { - var dbIDs []int64 - events, dbIDs, err = getWalletEvents(tx, id, offset, limit) + dbIDs, err := getWalletEvents(tx, id, offset, limit) if err != nil { return fmt.Errorf("failed to get wallet events: %w", err) } + events, err = getEventsByID(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } + eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, id, dbIDs) if err != nil { return fmt.Errorf("failed to get relevant addresses: %w", err) @@ -698,47 +702,30 @@ func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ e return } -func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (events []wallet.Event, eventIDs []int64, err error) { - var scanHeight uint64 - err = tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) - if err != nil { - return nil, nil, fmt.Errorf("failed to get last indexed height: %w", err) - } - - const eventsQuery = `SELECT - ev.id, - ev.event_id, - ev.maturity_height, - ev.date_created, - ci.height, - ci.block_id, - ev.event_type, - ev.event_data -FROM events ev INDEXED BY events_maturity_height_id_idx -- force index to prevent temp-btree sorts -INNER JOIN event_addresses ea ON ev.id = ea.event_id -INNER JOIN wallet_addresses wa ON ea.address_id = wa.address_id -INNER JOIN chain_indices ci ON ev.chain_index_id = ci.id +func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (eventIDs []int64, err error) { + const eventsQuery = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +INNER JOIN wallet_addresses wa ON sa.id = wa.address_id WHERE wa.wallet_id = $1 -ORDER BY ev.maturity_height DESC, ev.id DESC +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC LIMIT $2 OFFSET $3;` rows, err := tx.Query(eventsQuery, id, limit, offset) if err != nil { - return nil, nil, err + return nil, err } defer rows.Close() for rows.Next() { - event, eventID, err := scanEvent(rows, scanHeight) - if err != nil { - return nil, nil, fmt.Errorf("failed to scan event: %w", err) + var eventID int64 + if err := rows.Scan(&eventID); err != nil { + return nil, fmt.Errorf("failed to scan event ID: %w", err) } - - events = append(events, event) eventIDs = append(eventIDs, eventID) } if err := rows.Err(); err != nil { - return nil, nil, err + return nil, err } return } From 8773d1fe6455ee8ddc2895fba7c96d779e8819ac Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 8 Feb 2025 12:46:59 -0800 Subject: [PATCH 347/630] add wallet events benchmark --- persist/sqlite/events_test.go | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 persist/sqlite/events_test.go diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go new file mode 100644 index 0000000..1ef067b --- /dev/null +++ b/persist/sqlite/events_test.go @@ -0,0 +1,102 @@ +package sqlite + +import ( + "fmt" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +func runBenchmarkWalletEvents(b *testing.B, name string, addresses, eventsPerAddress int) { + b.Run(name, func(b *testing.B) { + db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3"), zap.NewNop()) + if err != nil { + b.Fatal(err) + } + defer db.Close() + + w, err := db.AddWallet(wallet.Wallet{ + Name: "test", + }) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < addresses; i++ { + addr := types.Address(frand.Entropy256()) + if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + b.Fatal(err) + } + + err := db.transaction(func(tx *txn) error { + utx := &updateTx{ + indexMode: wallet.IndexModeFull, + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } + + events := make([]wallet.Event, eventsPerAddress) + for i := range events { + events[i] = wallet.Event{ + ID: types.Hash256(frand.Entropy256()), + MaturityHeight: uint64(i + 1), + Relevant: []types.Address{addr}, + Type: wallet.EventTypeV1Transaction, + Data: wallet.EventV1Transaction{}, + } + } + + return utx.ApplyIndex(types.ChainIndex{ + Height: uint64(i + 1), + ID: types.BlockID(frand.Entropy256()), + }, wallet.AppliedState{ + Events: events, + }) + }) + if err != nil { + b.Fatal(err) + } + } + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + expectedEvents := eventsPerAddress * addresses + if expectedEvents > 100 { + expectedEvents = 100 + } + + events, err := db.WalletEvents(w.ID, 0, 100) + if err != nil { + b.Fatal(err) + } else if len(events) != expectedEvents { + b.Fatalf("expected %d events, got %d", expectedEvents, len(events)) + } + } + }) +} + +func BenchmarkWalletEvents(b *testing.B) { + benchmarks := []struct { + addresses int + eventsPerAddress int + }{ + {1, 1}, + {1, 10}, + {1, 1000}, + {10, 1}, + {10, 1000}, + {10, 100000}, + {1000000, 0}, + {1000000, 1}, + {1000000, 10}, + } + for _, bm := range benchmarks { + totalTransactions := bm.addresses * bm.eventsPerAddress + runBenchmarkWalletEvents(b, fmt.Sprintf("wallet with %d addresses and %d transactions", bm.addresses, totalTransactions), bm.addresses, bm.eventsPerAddress) + } +} From dd18a0400b4c4d4e3b799f66e7dec1bb4ea95ef7 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Feb 2025 08:39:35 -0800 Subject: [PATCH 348/630] rename index --- persist/sqlite/init.sql | 2 +- persist/sqlite/migrations.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 4bc860f..2b404e1 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -72,7 +72,7 @@ CREATE TABLE event_addresses ( ); CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); -CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); +CREATE INDEX event_addresses_event_id_address_id_event_maturity_height_event_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); CREATE TABLE wallets ( id INTEGER PRIMARY KEY, diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 0aa0a8d..84dbed5 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -22,7 +22,7 @@ DROP TABLE event_addresses; ALTER TABLE event_addresses_new RENAME TO event_addresses; CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); -CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); +CREATE INDEX event_addresses_event_id_address_id_event_maturity_height_event_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); ` _, err := tx.Exec(query) return err From 924e29bcee56ec61915d033680b34fe6325f74be Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Feb 2025 08:56:33 -0800 Subject: [PATCH 349/630] wallet: don't panic when threadgroup is closed --- wallet/manager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wallet/manager.go b/wallet/manager.go index 43ad162..31765f9 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -629,7 +629,9 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) go func() { ctx, cancel, err := m.tg.AddWithContext(context.Background()) - if err != nil { + if errors.Is(err, threadgroup.ErrClosed) { + return + } else if err != nil { log.Panic("failed to add to threadgroup", zap.Error(err)) } defer cancel() From df40c64cc67a4b8fd0d0221f0532945ff7994d58 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 14 Feb 2025 13:14:47 -0800 Subject: [PATCH 350/630] deps: update web --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5517dd4..fa11dc7 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.10.1 go.sia.tech/coreutils v0.11.1 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.27.0 + go.sia.tech/web/walletd v0.28.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.29.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 3a95118..8889bdf 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.27.0 h1:oTCqqZHvvWbcy/jKMN7urBEFYXQs1+yVzuKu17vgQtk= -go.sia.tech/web/walletd v0.27.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.28.0 h1:gFSN4Z/lH++c1wuFQeEZaXDh+UL6OpHqmMJPQFsMVPs= +go.sia.tech/web/walletd v0.28.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From fecf6342b118d554de6df053953db98fcae197f6 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Feb 2025 16:39:28 -0800 Subject: [PATCH 351/630] api: return error from construction API when spend policy is unset --- api/api_test.go | 46 +++++++++++++++++++++++--- api/server.go | 86 +++++++++++++++++++++---------------------------- 2 files changed, 77 insertions(+), 55 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 1c29341..48c77c8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1361,9 +1361,9 @@ func TestConstructSiacoins(t *testing.T) { } wc := c.Wallet(w.ID) + // add an address with no spend policy err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - SpendPolicy: &senderPolicy, + Address: senderAddr, }) if err != nil { t.Fatal(err) @@ -1376,6 +1376,25 @@ func TestConstructSiacoins(t *testing.T) { testutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, ws) + // try to construct a valid transaction with no spend policy + _, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if !strings.Contains(err.Error(), "no spend policy") { + t.Fatalf("expected error to contain %q, got %q", "no spend policy", err) + } + + // add the spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey())), + }, + }) + if err != nil { + t.Fatal(err) + } + // try to construct a transaction with more siafunds than the wallet holds. // this will lock all of the wallet's siacoins resp, err := wc.Construct([]types.SiacoinOutput{ @@ -1684,9 +1703,9 @@ func TestConstructV2Siacoins(t *testing.T) { } wc := c.Wallet(w.ID) + // add an address without a spend policy err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - SpendPolicy: &senderPolicy, + Address: senderAddr, }) if err != nil { t.Fatal(err) @@ -1699,9 +1718,26 @@ func TestConstructV2Siacoins(t *testing.T) { testutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, ws) + // try to construct a transaction + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if !strings.Contains(err.Error(), "no spend policy") { + t.Fatalf("expected spend policy error, got %q", err) + } + + // add a spend policy to the address + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + // try to construct a transaction with more siafunds than the wallet holds. // this will lock all of the wallet's Siacoin UTXOs - resp, err := wc.ConstructV2([]types.SiacoinOutput{ + resp, err = wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(1), Address: receiverAddr}, }, []types.SiafundOutput{ {Value: 100000, Address: senderAddr}, diff --git a/api/server.go b/api/server.go index 48afef9..98ff2d1 100644 --- a/api/server.go +++ b/api/server.go @@ -802,17 +802,23 @@ func (s *server) walletsConstructHandler(jc jape.Context) { }) } - knownAddresses := make(map[types.Address]wallet.Address) - getAddress := func(addr types.Address) (wallet.Address, error) { + knownAddresses := make(map[types.Address]types.UnlockConditions) + getAddressUnlockConditions := func(addr types.Address) (types.UnlockConditions, error) { if a, ok := knownAddresses[addr]; ok { return a, nil } a, err := s.wm.WalletAddress(walletID, addr) if err != nil { - return wallet.Address{}, err + return types.UnlockConditions{}, err + } else if a.SpendPolicy == nil { + return types.UnlockConditions{}, fmt.Errorf("address %q has no spend policy", addr) } - knownAddresses[addr] = a - return a, nil + uc, ok := a.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) + if !ok { + return types.UnlockConditions{}, fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr) + } + knownAddresses[addr] = types.UnlockConditions(uc) + return knownAddresses[addr], nil } resp := WalletConstructResponse{ @@ -829,24 +835,15 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } for _, sce := range sces { - addr, err := getAddress(sce.SiacoinOutput.Address) + uc, err := getAddressUnlockConditions(sce.SiacoinOutput.Address) if err != nil { jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) return } sci := types.SiacoinInput{ - ParentID: sce.ID, - } - - if addr.SpendPolicy != nil { - // best effort to fill unlock conditions - uc, ok := addr.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) - if !ok { - jc.Error(fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr.Address), http.StatusBadRequest) - return - } - sci.UnlockConditions = types.UnlockConditions(uc) + ParentID: sce.ID, + UnlockConditions: uc, } txn.SiacoinInputs = append(txn.SiacoinInputs, sci) @@ -859,24 +856,16 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } for _, sfe := range sfes { - addr, err := getAddress(sfe.SiafundOutput.Address) + uc, err := getAddressUnlockConditions(sfe.SiafundOutput.Address) if err != nil { jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) return } sfi := types.SiafundInput{ - ParentID: sfe.ID, - ClaimAddress: wcr.ChangeAddress, - } - if addr.SpendPolicy != nil { - // best effort to fill unlock conditions - uc, ok := addr.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) - if !ok { - jc.Error(fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr.Address), http.StatusBadRequest) - return - } - sfi.UnlockConditions = types.UnlockConditions(uc) + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: wcr.ChangeAddress, } txn.SiafundInputs = append(txn.SiafundInputs, sfi) txn.Signatures = append(txn.Signatures, types.TransactionSignature{ @@ -995,17 +984,22 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { }) } - knownAddresses := make(map[types.Address]wallet.Address) - getAddress := func(addr types.Address) (wallet.Address, error) { + knownAddresses := make(map[types.Address]types.SpendPolicy) + getAddressSpendPolicy := func(addr types.Address) (types.SpendPolicy, error) { if a, ok := knownAddresses[addr]; ok { return a, nil } a, err := s.wm.WalletAddress(walletID, addr) if err != nil { - return wallet.Address{}, err + return types.SpendPolicy{}, err } - knownAddresses[addr] = a - return a, nil + + if a.SpendPolicy == nil { + return types.SpendPolicy{}, fmt.Errorf("address %q has no spend policy", addr) + } else { + knownAddresses[addr] = *a.SpendPolicy + } + return knownAddresses[addr], nil } resp := WalletConstructV2Response{ @@ -1026,7 +1020,7 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { // guaranteed to have a non-zero Siacoin basis while the Siafund basis will be zero when not // sending Siafunds. for _, sfe := range sfes { - addr, err := getAddress(sfe.SiafundOutput.Address) + sp, err := getAddressSpendPolicy(sfe.SiafundOutput.Address) if err != nil { jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) return @@ -1035,13 +1029,9 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { sfi := types.V2SiafundInput{ Parent: sfe, ClaimAddress: wcr.ChangeAddress, - } - - if addr.SpendPolicy != nil { - // best effort to fill spend policy - sfi.SatisfiedPolicy = types.SatisfiedPolicy{ - Policy: *addr.SpendPolicy, - } + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, } txn.SiafundInputs = append(txn.SiafundInputs, sfi) } @@ -1056,7 +1046,7 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } for _, sce := range sces { - addr, err := getAddress(sce.SiacoinOutput.Address) + sp, err := getAddressSpendPolicy(sce.SiacoinOutput.Address) if err != nil { jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) return @@ -1064,13 +1054,9 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { sci := types.V2SiacoinInput{ Parent: sce, - } - - if addr.SpendPolicy != nil { - // best effort to fill spend policy - sci.SatisfiedPolicy = types.SatisfiedPolicy{ - Policy: *addr.SpendPolicy, - } + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, } txn.SiacoinInputs = append(txn.SiacoinInputs, sci) } From 1b0eb38f700d8531d28826047bd7e08dbaee514a Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 13 Feb 2025 16:19:48 -0800 Subject: [PATCH 352/630] change error text --- api/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server.go b/api/server.go index 98ff2d1..7bdcab9 100644 --- a/api/server.go +++ b/api/server.go @@ -815,7 +815,7 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } uc, ok := a.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) if !ok { - return types.UnlockConditions{}, fmt.Errorf("address %q only unlock conditions are suppored in v1 transactions", addr) + return types.UnlockConditions{}, fmt.Errorf("address %q has v2-only spend policy", addr) } knownAddresses[addr] = types.UnlockConditions(uc) return knownAddresses[addr], nil From 65021ca02581c55ea640ef0b6f05495990fe21d2 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 14 Feb 2025 14:54:27 -0800 Subject: [PATCH 353/630] api: return better status codes --- api/server.go | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/api/server.go b/api/server.go index 7bdcab9..576fc9e 100644 --- a/api/server.go +++ b/api/server.go @@ -803,22 +803,25 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } knownAddresses := make(map[types.Address]types.UnlockConditions) - getAddressUnlockConditions := func(addr types.Address) (types.UnlockConditions, error) { + getAddressUnlockConditions := func(jc jape.Context, addr types.Address) (types.UnlockConditions, bool) { if a, ok := knownAddresses[addr]; ok { - return a, nil + return a, true } a, err := s.wm.WalletAddress(walletID, addr) if err != nil { - return types.UnlockConditions{}, err + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return types.UnlockConditions{}, false } else if a.SpendPolicy == nil { - return types.UnlockConditions{}, fmt.Errorf("address %q has no spend policy", addr) + jc.Error(fmt.Errorf("address %q has no spend policy", addr), http.StatusBadRequest) + return types.UnlockConditions{}, false } uc, ok := a.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) if !ok { - return types.UnlockConditions{}, fmt.Errorf("address %q has v2-only spend policy", addr) + jc.Error(fmt.Errorf("address %q has v2-only spend policy", addr), http.StatusBadRequest) + return types.UnlockConditions{}, false } knownAddresses[addr] = types.UnlockConditions(uc) - return knownAddresses[addr], nil + return knownAddresses[addr], true } resp := WalletConstructResponse{ @@ -835,9 +838,8 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } for _, sce := range sces { - uc, err := getAddressUnlockConditions(sce.SiacoinOutput.Address) - if err != nil { - jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + uc, ok := getAddressUnlockConditions(jc, sce.SiacoinOutput.Address) + if !ok { return } @@ -856,9 +858,8 @@ func (s *server) walletsConstructHandler(jc jape.Context) { } for _, sfe := range sfes { - uc, err := getAddressUnlockConditions(sfe.SiafundOutput.Address) - if err != nil { - jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + uc, ok := getAddressUnlockConditions(jc, sfe.SiafundOutput.Address) + if !ok { return } @@ -985,21 +986,22 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } knownAddresses := make(map[types.Address]types.SpendPolicy) - getAddressSpendPolicy := func(addr types.Address) (types.SpendPolicy, error) { + getAddressSpendPolicy := func(jc jape.Context, addr types.Address) (types.SpendPolicy, bool) { if a, ok := knownAddresses[addr]; ok { - return a, nil + return a, true } a, err := s.wm.WalletAddress(walletID, addr) if err != nil { - return types.SpendPolicy{}, err + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return types.SpendPolicy{}, false } if a.SpendPolicy == nil { - return types.SpendPolicy{}, fmt.Errorf("address %q has no spend policy", addr) - } else { - knownAddresses[addr] = *a.SpendPolicy + jc.Error(fmt.Errorf("address %q has no spend policy", addr), http.StatusBadRequest) + return types.SpendPolicy{}, false } - return knownAddresses[addr], nil + knownAddresses[addr] = *a.SpendPolicy + return knownAddresses[addr], true } resp := WalletConstructV2Response{ @@ -1020,9 +1022,8 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { // guaranteed to have a non-zero Siacoin basis while the Siafund basis will be zero when not // sending Siafunds. for _, sfe := range sfes { - sp, err := getAddressSpendPolicy(sfe.SiafundOutput.Address) - if err != nil { - jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + sp, ok := getAddressSpendPolicy(jc, sfe.SiafundOutput.Address) + if !ok { return } @@ -1046,9 +1047,8 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } for _, sce := range sces { - sp, err := getAddressSpendPolicy(sce.SiacoinOutput.Address) - if err != nil { - jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + sp, ok := getAddressSpendPolicy(jc, sce.SiacoinOutput.Address) + if !ok { return } From b8c6d652f55c6616455f4b619691a1f9486c0e16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 16:58:23 +0000 Subject: [PATCH 354/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.28.0 to 0.29.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.28.0...hostd@0.29.0) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fa11dc7..e6b76d7 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.10.1 go.sia.tech/coreutils v0.11.1 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.28.0 + go.sia.tech/web/walletd v0.29.0 go.uber.org/zap v1.27.0 golang.org/x/term v0.29.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 8889bdf..5a18494 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= -go.sia.tech/web/walletd v0.28.0 h1:gFSN4Z/lH++c1wuFQeEZaXDh+UL6OpHqmMJPQFsMVPs= -go.sia.tech/web/walletd v0.28.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.29.0 h1:JHJj6TlQozKGcUqUyL0YXR0I+Poe1kjgcGA5v1/9tjA= +go.sia.tech/web/walletd v0.29.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From 1f745eb57304c1618578ef903f7e5cdfc4a15e0b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 20 Feb 2025 17:44:48 +0100 Subject: [PATCH 355/630] readme: fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 093f2eb..d55a7ba 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Flags: -debug enable debug mode with additional profiling and mining endpoints -dir string - directory to store node state in (default "/Users/n8maninger/Downloads/walletd-tmp") + directory to store node state in (default "/Users/username/Library/Application Support/walletd") -http string address to serve API on (default "localhost:9980") -http.public From 908876463e7f6af31342f51d8ea8f9b522f73c55 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 20 Feb 2025 18:11:52 +0100 Subject: [PATCH 356/630] add explanation for walletd config --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d55a7ba..f990367 100644 --- a/README.md +++ b/README.md @@ -100,8 +100,15 @@ Flags: ``` ### YAML -All configuration settings can be set in a YAML file. The file should be named -`walletd.yml` in the working directory. All fields are optional. +All configuration settings can be set in a YAML file. The default location of that file is +- `/etc/walletd/walletd.yml` on Linux +- `~/Library/Application Support/walletd/walletd.yml` on macOS +- `%APPDATA%\SiaFoundation\walletd.yml` on Windows +- `/data/walletd.yml` in the Docker container + +It can be generated using the `walletd config` command. Alternatively a local +configuration can be created manually by creating a file name `walletd.yml` in +the working directory. All fields are optional. ```yaml directory: /etc/walletd autoOpenWebUI: true From 770d2977a538aedbd6813bf02b34e82d53881f96 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Feb 2025 16:17:16 -0800 Subject: [PATCH 357/630] fix hardfork dates --- .changeset/support_v2_hardfork.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/support_v2_hardfork.md b/.changeset/support_v2_hardfork.md index a2c2045..c17af17 100644 --- a/.changeset/support_v2_hardfork.md +++ b/.changeset/support_v2_hardfork.md @@ -13,14 +13,14 @@ The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has be - Improved transfer speeds - enables hot storage #### Phase 1 - Allow Height -- **Activation Height:** `513400` (March 10th, 2025) +- **Activation Height:** `52600` (June 6th, 2025) - **New Features:** V2 transactions, contracts, and RHP4 - **V1 Support:** Both V1 and V2 will be supported during this phase - **Purpose:** This period gives time for integrators to transition from V1 to V2 - **Requirements:** Users will need to update to support the hardfork before this block height #### Phase 2 - Require Height -- **Activation Height:** `526000` (June 6th, 2025) +- **Activation Height:** `530000` (July 6th, 2025) - **New Features:** The consensus database can be trimmed to only store the Merkle proofs - **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted - **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height From 0586aa1c5f836e66cf284aca2081354c6ead5882 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Feb 2025 16:17:25 -0800 Subject: [PATCH 358/630] chore: prepare release 2.0.0 --- .../add_basis_to_wallet_fund_endpoints.md | 5 - ...erkle_proof_basis_to_utxo_api_responses.md | 39 -- .../add_transaction_construction_api.md | 9 - ...re_consistency_between_database_schemas.md | 5 - .changeset/log_startup_errors_to_stderr.md | 5 - ..._response_of_consensus_updates_endpoint.md | 243 ------------- .changeset/support_v2_hardfork.md | 26 -- ...standard_locations_for_application_data.md | 23 -- CHANGELOG.md | 338 ++++++++++++++++++ go.mod | 2 +- 10 files changed, 339 insertions(+), 356 deletions(-) delete mode 100644 .changeset/add_basis_to_wallet_fund_endpoints.md delete mode 100644 .changeset/add_merkle_proof_basis_to_utxo_api_responses.md delete mode 100644 .changeset/add_transaction_construction_api.md delete mode 100644 .changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md delete mode 100644 .changeset/log_startup_errors_to_stderr.md delete mode 100644 .changeset/simplified_response_of_consensus_updates_endpoint.md delete mode 100644 .changeset/support_v2_hardfork.md delete mode 100644 .changeset/use_standard_locations_for_application_data.md diff --git a/.changeset/add_basis_to_wallet_fund_endpoints.md b/.changeset/add_basis_to_wallet_fund_endpoints.md deleted file mode 100644 index b42fb16..0000000 --- a/.changeset/add_basis_to_wallet_fund_endpoints.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Add basis to wallet fund endpoints diff --git a/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md b/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md deleted file mode 100644 index 94ef47f..0000000 --- a/.changeset/add_merkle_proof_basis_to_utxo_api_responses.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -default: major ---- - -# Add Merkle Proof Basis to UTXO API Responses - -Changes the response to include the Merkle proof basis for the following endpoints: -- `[GET] /addresses/:address/outputs/siacoin` -- `[GET] /addresses/:address/outputs/siafund` -- `[GET] /wallets/:id/outputs/siacoin` -- `[GET] /wallets/:id/outputs/siafund` - - -```json -{ - "basis": { - "height": 1, - "id": "f362385eea61f81627f283a31af9faf6417fbb88d53b794639a34e18515996e9" - }, - "outputs": [ - { - "id": "ed556177482e70822a5dcad9343efb51998425884788415349bef8eba7e063ae", - "stateElement": { - "leafIndex": 3, - "merkleProof": [ - "01048fc792904f156844a5524671304d3a020861da144afa4acc6553db63c1fd", - "33efdfaf9bb212842292ab6f298c454e1b3d412aa7beb7efdccdfccf09f5b4ee", - "102345919e408540d240460b0d84aa2f6da9a3d8f74765fd7c6daae6e46dd7f3" - ] - }, - "siacoinOutput": { - "value": "500000000000000000000000", - "address": "fbfc3d034b1eb45f63e0087571ec1f3028a9a2f8c180381d47713e6112467d91f474059476f2" - }, - "maturityHeight": 0 - } - ] -} -``` \ No newline at end of file diff --git a/.changeset/add_transaction_construction_api.md b/.changeset/add_transaction_construction_api.md deleted file mode 100644 index 053b46e..0000000 --- a/.changeset/add_transaction_construction_api.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -default: minor ---- - -# Add transaction construction API - -Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for simple send transactions. - -See API docs for request and response bodies \ No newline at end of file diff --git a/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md b/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md deleted file mode 100644 index 87d2e57..0000000 --- a/.changeset/added_a_test_for_migrations_to_ensure_consistency_between_database_schemas.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Added a test for migrations to ensure consistency between database schemas diff --git a/.changeset/log_startup_errors_to_stderr.md b/.changeset/log_startup_errors_to_stderr.md deleted file mode 100644 index 347e743..0000000 --- a/.changeset/log_startup_errors_to_stderr.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Log startup errors to stderr diff --git a/.changeset/simplified_response_of_consensus_updates_endpoint.md b/.changeset/simplified_response_of_consensus_updates_endpoint.md deleted file mode 100644 index 5eab007..0000000 --- a/.changeset/simplified_response_of_consensus_updates_endpoint.md +++ /dev/null @@ -1,243 +0,0 @@ ---- -default: major ---- - -# Simplified response of consensus updates endpoint - -The response of `/api/consensus/updates/:index` has been simplified to make it easier for developers to index chain state. - -```json -{ - "applied": [ - { - "update": { - "siacoinElements": [ - { - "siacoinElement": { - "id": "35b81e41f594d7faeb88bd8eaac2eaa68ce99fe1c8fe5f0cba8fafa65ab3a70e", - "stateElement": { - "leafIndex": 0, - "merkleProof": [ - "88052fa2d1e22e4a5542fed9686cdad3fbeccbc60d15d4fd36a7691d61add1e1" - ] - }, - "siacoinOutput": { - "value": "1000000000000000000000000000000000000", - "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" - }, - "maturityHeight": 0 - }, - "created": true, - "spent": false - } - ], - "siafundElementDiffs": [ - { - "siafundElement": { - "id": "69ad26a0fbd1a6985d2053246650bb3ba5f3491d818748b6c8562db1ddb2c45b", - "stateElement": { - "leafIndex": 1, - "merkleProof": [ - "837482a39d5bf66f07bae3b89191e4375b82c9f341ce6a17e22e14e0333ab9f6" - ] - }, - "siafundOutput": { - "value": 10000, - "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" - }, - "claimStart": "0" - }, - "created": true, - "spent": false - } - ], - "fileContractElementDiffs": null, - "v2FileContractElementDiffs": null, - "attestationElements": null, - "chainIndexElement": { - "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", - "stateElement": { - "leafIndex": 2 - }, - "chainIndex": { - "height": 0, - "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" - } - }, - "updatedLeaves": {}, - "treeGrowth": {}, - "oldNumLeaves": 0, - "numLeaves": 3 - }, - "state": { - "index": { - "height": 0, - "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" - }, - "prevTimestamps": [ - "2023-01-13T00:53:20-08:00", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z" - ], - "depth": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", - "siafundTaxRevenue": "0", - "oakTime": 0, - "oakTarget": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", - "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", - "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", - "totalWork": "1", - "difficulty": "4294967295", - "oakWork": "4294967297", - "elements": { - "numLeaves": 3, - "trees": [ - "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", - "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" - ] - }, - "attestations": 0 - }, - "block": { - "parentID": "0000000000000000000000000000000000000000000000000000000000000000", - "nonce": 0, - "timestamp": "2023-01-13T00:53:20-08:00", - "minerPayouts": [], - "transactions": [ - { - "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", - "siacoinOutputs": [ - { - "value": "1000000000000000000000000000000000000", - "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" - } - ], - "siafundOutputs": [ - { - "value": 10000, - "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" - } - ] - } - ] - } - }, - { - "update": { - "siacoinElements": [ - { - "siacoinElement": { - "id": "ca02d6807c92f61af94e626604615fbcdb471f38fcd8f3add6c6e6e0485ce090", - "stateElement": { - "leafIndex": 3, - "merkleProof": [ - "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", - "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" - ] - }, - "siacoinOutput": { - "value": "300000000000000000000000000000", - "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" - }, - "maturityHeight": 145 - }, - "created": true, - "spent": false - } - ], - "siafundElementDiffs": null, - "fileContractElementDiffs": null, - "v2FileContractElementDiffs": null, - "attestationElements": null, - "chainIndexElement": { - "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006", - "stateElement": { - "leafIndex": 4 - }, - "chainIndex": { - "height": 1, - "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" - } - }, - "updatedLeaves": {}, - "treeGrowth": { - "0": [ - "190d98a7d8ff464e57f89dc916b155455ecf927f4c74b9edf5e80c103f052bfa", - "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" - ], - "1": [ - "2b082bec52801c1e61e5b0d0c1f5fc3925bd24e16d2f490afeb70374828586f1" - ] - }, - "oldNumLeaves": 3, - "numLeaves": 5 - }, - "state": { - "index": { - "height": 1, - "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" - }, - "prevTimestamps": [ - "2023-01-13T08:18:19-08:00", - "2023-01-13T00:53:20-08:00", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z", - "0001-01-01T00:00:00Z" - ], - "depth": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", - "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", - "siafundTaxRevenue": "0", - "oakTime": 26699000000000, - "oakTarget": "000000008052201448053c59f99803e7a8165929036cd574d91425423191387c", - "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", - "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", - "totalWork": "4294967297", - "difficulty": "4294967295", - "oakWork": "8568459756", - "elements": { - "numLeaves": 5, - "trees": [ - "589fb425faa23be357492394813dc575505899d42d0b23a7162e1c68f7eeb227", - "750cc671d80aef6ee5c73344ba4e74eccda77d9f0cf51ed6237952b1d84bc336" - ] - }, - "attestations": 0 - }, - "block": { - "parentID": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", - "nonce": 10689346, - "timestamp": "2023-01-13T08:18:19-08:00", - "minerPayouts": [ - { - "value": "300000000000000000000000000000", - "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" - } - ], - "transactions": [ - { - "id": "1148417ad8fa6546646da6922618358210bc7a668ef7cb25f6a8a3605851bc7b", - "arbitraryData": [ - "Tm9uU2lhAAAAAAAAAAAAAClvJjNhfcbxtEfP2yfbBM4=" - ] - } - ] - } - } - ], - "reverted": null -} -``` \ No newline at end of file diff --git a/.changeset/support_v2_hardfork.md b/.changeset/support_v2_hardfork.md deleted file mode 100644 index c17af17..0000000 --- a/.changeset/support_v2_hardfork.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -default: major ---- - -# Support V2 Hardfork - -The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. - -#### V2 Highlights -- Drastically reduces blockchain size on disk -- Improves UTXO spend policies - including HTLC support for Atomic Swaps -- More efficient contract renewals - reducing lock up requirements for hosts and renters -- Improved transfer speeds - enables hot storage - -#### Phase 1 - Allow Height -- **Activation Height:** `52600` (June 6th, 2025) -- **New Features:** V2 transactions, contracts, and RHP4 -- **V1 Support:** Both V1 and V2 will be supported during this phase -- **Purpose:** This period gives time for integrators to transition from V1 to V2 -- **Requirements:** Users will need to update to support the hardfork before this block height - -#### Phase 2 - Require Height -- **Activation Height:** `530000` (July 6th, 2025) -- **New Features:** The consensus database can be trimmed to only store the Merkle proofs -- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted -- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height diff --git a/.changeset/use_standard_locations_for_application_data.md b/.changeset/use_standard_locations_for_application_data.md deleted file mode 100644 index d53f435..0000000 --- a/.changeset/use_standard_locations_for_application_data.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -default: major ---- - -# Use standard locations for application data - -Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. - -#### Linux, FreeBSD, OpenBSD -- Configuration: `/etc/walletd/walletd.yml` -- Data directory: `/var/lib/walletd` - -#### macOS -- Configuration: `~/Library/Application Support/walletd.yml` -- Data directory: `~/Library/Application Support/walletd` - -#### Windows -- Configuration: `%APPDATA%\SiaFoundation\walletd.yml` -- Data directory: `%APPDATA%\SiaFoundation\walletd` - -#### Docker -- Configuration: `/data/walletd.yml` -- Data directory: `/data` diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dddb0e..1281049 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,341 @@ +## 2.0.0 (2025-02-21) + +### Breaking Changes + +#### Add Merkle Proof Basis to UTXO API Responses + +Changes the response to include the Merkle proof basis for the following endpoints: +- `[GET] /addresses/:address/outputs/siacoin` +- `[GET] /addresses/:address/outputs/siafund` +- `[GET] /wallets/:id/outputs/siacoin` +- `[GET] /wallets/:id/outputs/siafund` + + +```json +{ + "basis": { + "height": 1, + "id": "f362385eea61f81627f283a31af9faf6417fbb88d53b794639a34e18515996e9" + }, + "outputs": [ + { + "id": "ed556177482e70822a5dcad9343efb51998425884788415349bef8eba7e063ae", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "01048fc792904f156844a5524671304d3a020861da144afa4acc6553db63c1fd", + "33efdfaf9bb212842292ab6f298c454e1b3d412aa7beb7efdccdfccf09f5b4ee", + "102345919e408540d240460b0d84aa2f6da9a3d8f74765fd7c6daae6e46dd7f3" + ] + }, + "siacoinOutput": { + "value": "500000000000000000000000", + "address": "fbfc3d034b1eb45f63e0087571ec1f3028a9a2f8c180381d47713e6112467d91f474059476f2" + }, + "maturityHeight": 0 + } + ] +} +``` + +#### Simplified response of consensus updates endpoint + +The response of `/api/consensus/updates/:index` has been simplified to make it easier for developers to index chain state. + +```json +{ + "applied": [ + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "35b81e41f594d7faeb88bd8eaac2eaa68ce99fe1c8fe5f0cba8fafa65ab3a70e", + "stateElement": { + "leafIndex": 0, + "merkleProof": [ + "88052fa2d1e22e4a5542fed9686cdad3fbeccbc60d15d4fd36a7691d61add1e1" + ] + }, + "siacoinOutput": { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + }, + "maturityHeight": 0 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": [ + { + "siafundElement": { + "id": "69ad26a0fbd1a6985d2053246650bb3ba5f3491d818748b6c8562db1ddb2c45b", + "stateElement": { + "leafIndex": 1, + "merkleProof": [ + "837482a39d5bf66f07bae3b89191e4375b82c9f341ce6a17e22e14e0333ab9f6" + ] + }, + "siafundOutput": { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + }, + "claimStart": "0" + }, + "created": true, + "spent": false + } + ], + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "stateElement": { + "leafIndex": 2 + }, + "chainIndex": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + } + }, + "updatedLeaves": {}, + "treeGrowth": {}, + "oldNumLeaves": 0, + "numLeaves": 3 + }, + "state": { + "index": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + }, + "prevTimestamps": [ + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 0, + "oakTarget": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "1", + "difficulty": "4294967295", + "oakWork": "4294967297", + "elements": { + "numLeaves": 3, + "trees": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "2023-01-13T00:53:20-08:00", + "minerPayouts": [], + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] + } + }, + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "ca02d6807c92f61af94e626604615fbcdb471f38fcd8f3add6c6e6e0485ce090", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "siacoinOutput": { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + }, + "maturityHeight": 145 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": null, + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006", + "stateElement": { + "leafIndex": 4 + }, + "chainIndex": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + } + }, + "updatedLeaves": {}, + "treeGrowth": { + "0": [ + "190d98a7d8ff464e57f89dc916b155455ecf927f4c74b9edf5e80c103f052bfa", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ], + "1": [ + "2b082bec52801c1e61e5b0d0c1f5fc3925bd24e16d2f490afeb70374828586f1" + ] + }, + "oldNumLeaves": 3, + "numLeaves": 5 + }, + "state": { + "index": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + }, + "prevTimestamps": [ + "2023-01-13T08:18:19-08:00", + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 26699000000000, + "oakTarget": "000000008052201448053c59f99803e7a8165929036cd574d91425423191387c", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "4294967297", + "difficulty": "4294967295", + "oakWork": "8568459756", + "elements": { + "numLeaves": 5, + "trees": [ + "589fb425faa23be357492394813dc575505899d42d0b23a7162e1c68f7eeb227", + "750cc671d80aef6ee5c73344ba4e74eccda77d9f0cf51ed6237952b1d84bc336" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "nonce": 10689346, + "timestamp": "2023-01-13T08:18:19-08:00", + "minerPayouts": [ + { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + } + ], + "transactions": [ + { + "id": "1148417ad8fa6546646da6922618358210bc7a668ef7cb25f6a8a3605851bc7b", + "arbitraryData": [ + "Tm9uU2lhAAAAAAAAAAAAAClvJjNhfcbxtEfP2yfbBM4=" + ] + } + ] + } + } + ], + "reverted": null +} +``` + +#### Support V2 Hardfork + +The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. + +##### V2 Highlights +- Drastically reduces blockchain size on disk +- Improves UTXO spend policies - including HTLC support for Atomic Swaps +- More efficient contract renewals - reducing lock up requirements for hosts and renters +- Improved transfer speeds - enables hot storage + +##### Phase 1 - Allow Height +- **Activation Height:** `52600` (June 6th, 2025) +- **New Features:** V2 transactions, contracts, and RHP4 +- **V1 Support:** Both V1 and V2 will be supported during this phase +- **Purpose:** This period gives time for integrators to transition from V1 to V2 +- **Requirements:** Users will need to update to support the hardfork before this block height + +##### Phase 2 - Require Height +- **Activation Height:** `530000` (July 6th, 2025) +- **New Features:** The consensus database can be trimmed to only store the Merkle proofs +- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted +- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height + +#### Use standard locations for application data + +Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. + +##### Linux, FreeBSD, OpenBSD +- Configuration: `/etc/walletd/walletd.yml` +- Data directory: `/var/lib/walletd` + +##### macOS +- Configuration: `~/Library/Application Support/walletd.yml` +- Data directory: `~/Library/Application Support/walletd` + +##### Windows +- Configuration: `%APPDATA%\SiaFoundation\walletd.yml` +- Data directory: `%APPDATA%\SiaFoundation\walletd` + +##### Docker +- Configuration: `/data/walletd.yml` +- Data directory: `/data` + +### Features + +- Add basis to wallet fund endpoints +- Log startup errors to stderr + +#### Add transaction construction API + +Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for simple send transactions. + +See API docs for request and response bodies + +### Fixes + +- Added a test for migrations to ensure consistency between database schemas + ## 0.8.0 This is the first stable release for the walletd app -- the new reference wallet for users and exchanges diff --git a/go.mod b/go.mod index e6b76d7..4c07468 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd +module go.sia.tech/walletd/v2 // v2.0.0 go 1.23.1 From 42659f7056fd70044c8bd60a6ada48e2a4b73afb Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Feb 2025 16:22:13 -0800 Subject: [PATCH 359/630] fix package --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 4c07468..a08e0c2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.0.0 +module go.sia.tech/walletd // v2.0.0 go 1.23.1 @@ -28,7 +28,7 @@ require ( github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.4.0 // indirect go.sia.tech/mux v1.3.0 // indirect - go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef // indirect + go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.33.0 // indirect diff --git a/go.sum b/go.sum index 5a18494..a9184db 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= -go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef h1:X0Xm9AQYHhdd85yi9gqkkCZMb9/WtLwC0nDgv65N90Y= -go.sia.tech/web v0.0.0-20240422221546-c1709d16b6ef/go.mod h1:nGEhGmI8zV/BcC3LOCC5JLVYpidNYJIvLGIqVRWQBCg= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.29.0 h1:JHJj6TlQozKGcUqUyL0YXR0I+Poe1kjgcGA5v1/9tjA= go.sia.tech/web/walletd v0.29.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From a1fd68b361130c8cfc376b32487929a70272510e Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 19 Feb 2025 12:39:05 -0800 Subject: [PATCH 360/630] sqlite, wallet: track spent event id --- persist/sqlite/consensus.go | 65 +++++++++++++++++++++++------------- persist/sqlite/init.sql | 12 ++++--- persist/sqlite/migrations.go | 14 ++++++++ wallet/manager.go | 2 +- wallet/update.go | 52 ++++++++++++++++++++++++++--- 5 files changed, 113 insertions(+), 32 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 8004259..46fae99 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -122,6 +122,10 @@ func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState return fmt.Errorf("failed to insert chain index: %w", err) } + if err := addEvents(tx, state.Events, indexID); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { return fmt.Errorf("failed to spend siacoin elements: %w", err) } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, ut.indexMode, log.Named("addSiacoinElements")); err != nil { @@ -134,9 +138,6 @@ func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState return fmt.Errorf("failed to add siafund elements: %w", err) } - if err := addEvents(tx, state.Events, indexID); err != nil { - return fmt.Errorf("failed to add events: %w", err) - } return nil } @@ -719,7 +720,7 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error } defer done() - stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -734,10 +735,9 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error balanceChanges[addrRef.ID] = addrRef.Balance } - var dummy types.Hash256 - if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + if res, err := stmt.Exec(encode(se.ID)); err != nil { return err - } else if errors.Is(err, sql.ErrNoRows) { + } else if n, _ := res.RowsAffected(); n == 0 { continue // skip if the element does not exist } @@ -769,7 +769,7 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error return nil } -func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64) error { +func spendSiacoinElements(tx *txn, elements []wallet.SpentSiacoinElement, indexID int64) error { if len(elements) == 0 { return nil } @@ -780,7 +780,13 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 } defer done() - stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) + getEventIDStmt, err := tx.Prepare(`SELECT id FROM events WHERE event_id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getEventIDStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -795,10 +801,14 @@ func spendSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int6 balanceChanges[addrRef.ID] = addrRef.Balance } - var dummy types.Hash256 - if err := stmt.QueryRow(indexID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + var eventDBID int64 + if err := getEventIDStmt.QueryRow(encode(se.EventID)).Scan(&eventDBID); err != nil { + return fmt.Errorf("failed to get event ID: %w", err) + } + + if res, err := stmt.Exec(indexID, eventDBID, encode(se.ID)); err != nil { return err - } else if errors.Is(err, sql.ErrNoRows) { + } else if n, _ := res.RowsAffected(); n == 0 { continue // skip if the element does not exist } @@ -968,7 +978,7 @@ func removeSiafundElements(tx *txn, elements []types.SiafundElement) error { return nil } -func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64) error { +func spendSiafundElements(tx *txn, elements []wallet.SpentSiafundElement, indexID int64) error { if len(elements) == 0 { return nil } @@ -979,7 +989,13 @@ func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int6 } defer done() - stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=$1 WHERE id=$2 AND spent_index_id IS NULL RETURNING id`) + getEventIDStmt, err := tx.Prepare(`SELECT id FROM events WHERE event_id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getEventIDStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -994,10 +1010,14 @@ func spendSiafundElements(tx *txn, elements []types.SiafundElement, indexID int6 balanceChanges[addrRef.ID] = addrRef.Balance } - var dummy types.Hash256 - if err := stmt.QueryRow(indexID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + var eventDBID int64 + if err := getEventIDStmt.QueryRow(encode(se.EventID)).Scan(&eventDBID); err != nil { + return fmt.Errorf("failed to get event ID: %w", err) + } + + if res, err := stmt.Exec(indexID, eventDBID, encode(se.ID)); err != nil { return err - } else if errors.Is(err, sql.ErrNoRows) { + } else if n, _ := res.RowsAffected(); n == 0 { continue // skip if the element does not exist } @@ -1044,7 +1064,7 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error } defer done() - stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -1059,10 +1079,9 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error balanceChanges[addrRef.ID] = addrRef.Balance } - var dummy types.Hash256 - if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + if res, err := stmt.Exec(encode(se.ID)); err != nil { return err - } else if errors.Is(err, sql.ErrNoRows) { + } else if n, _ := res.RowsAffected(); n == 0 { continue // skip if the element does not exist } @@ -1167,7 +1186,7 @@ func revertEvents(tx *txn, index types.ChainIndex) error { } func revertSpentOrphanedSiacoinElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]wallet.Balance, error) { - rows, err := tx.Query(`UPDATE siacoin_elements SET spent_index_id=NULL WHERE id IN (SELECT se.id FROM siacoin_elements se + rows, err := tx.Query(`UPDATE siacoin_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id IN (SELECT se.id FROM siacoin_elements se INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) WHERE ci.height=$1 AND ci.block_id<>$2) RETURNING address_id, siacoin_value`, index.Height, encode(index.ID)) @@ -1228,7 +1247,7 @@ RETURNING id, address_id, siacoin_value, matured, spent_index_id IS NOT NULL`, i } func revertSpentOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { - rows, err := tx.Query(`UPDATE siafund_elements SET spent_index_id=NULL WHERE id IN (SELECT se.id FROM siafund_elements se + rows, err := tx.Query(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id IN (SELECT se.id FROM siafund_elements se INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) WHERE ci.height=$1 AND ci.block_id<>$2) RETURNING id, address_id, siafund_value`, index.Height, encode(index.ID)) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 2b404e1..9584437 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -18,16 +18,18 @@ CREATE TABLE siacoin_elements ( siacoin_value BLOB NOT NULL, merkle_proof BLOB NOT NULL, leaf_index INTEGER UNIQUE NOT NULL, - maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + maturity_height INTEGER NOT NULL, -- stored as int64 for easier querying address_id INTEGER NOT NULL REFERENCES sia_addresses (id), - matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + matured BOOLEAN NOT NULL, -- tracks whether the value has been added to the address balance chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), - spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ + spent_index_id INTEGER REFERENCES chain_indices (id), -- soft delete + spent_event_id INTEGER REFERENCES events (id) -- atomic swap tracking ); CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); CREATE INDEX siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); CREATE INDEX siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_spent_event_id_idx ON siacoin_elements (spent_event_id); CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id); CREATE TABLE siafund_elements ( @@ -38,11 +40,13 @@ CREATE TABLE siafund_elements ( siafund_value INTEGER NOT NULL, address_id INTEGER NOT NULL REFERENCES sia_addresses (id), chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), - spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ + spent_index_id INTEGER REFERENCES chain_indices (id), -- soft delete + spent_event_id INTEGER REFERENCES events (id) -- atomic swap tracking ); CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); CREATE INDEX siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_spent_event_id_idx ON siafund_elements (spent_event_id); CREATE INDEX siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id); CREATE TABLE state_tree ( diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 84dbed5..1358a7f 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,6 +7,19 @@ import ( "go.uber.org/zap" ) +// migrateVersion7 adds spent_event_id columns to siacoin_elements and +// siafund_elements to track the event that spent the element. +func migrateVersion7(tx *txn, _ *zap.Logger) error { + const query = `ALTER TABLE siacoin_elements ADD COLUMN spent_event_id INTEGER REFERENCES events (id); +CREATE INDEX siacoin_elements_spent_event_id_idx ON siacoin_elements (spent_event_id); +ALTER TABLE siafund_elements ADD COLUMN spent_event_id INTEGER REFERENCES events (id); +CREATE INDEX siafund_elements_spent_event_id_idx ON siafund_elements (spent_event_id);` + _, err := tx.Exec(query) + return err +} + +// migrateVersion6 flattens the maturity height from events into event_addresses +// to improve query performance. func migrateVersion6(tx *txn, _ *zap.Logger) error { const query = ` CREATE TABLE event_addresses_new ( @@ -185,4 +198,5 @@ var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion4, migrateVersion5, migrateVersion6, + migrateVersion7, } diff --git a/wallet/manager.go b/wallet/manager.go index 31765f9..154744f 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -695,7 +695,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) default: } default: - log.Panic("failed to sync store", zap.Error(err)) + panic("failed to sync store: " + err.Error()) } } m.mu.Unlock() diff --git a/wallet/update.go b/wallet/update.go index 7001b8a..1f3c4a3 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -27,15 +27,25 @@ type ( Balance } + SpentSiacoinElement struct { + types.SiacoinElement + EventID types.TransactionID + } + + SpentSiafundElement struct { + types.SiafundElement + EventID types.TransactionID + } + // AppliedState contains all state changes made to a store after applying a chain // update. AppliedState struct { NumLeaves uint64 Events []Event CreatedSiacoinElements []types.SiacoinElement - SpentSiacoinElements []types.SiacoinElement + SpentSiacoinElements []SpentSiacoinElement CreatedSiafundElements []types.SiafundElement - SpentSiafundElements []types.SiafundElement + SpentSiafundElements []SpentSiafundElement } // RevertedState contains all state changes made to a store after reverting @@ -92,6 +102,26 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e NumLeaves: cau.State.Elements.NumLeaves, } + spentEventIDs := make(map[types.Hash256]types.TransactionID) + for _, txn := range cau.Block.Transactions { + txnID := txn.ID() + for _, input := range txn.SiacoinInputs { + spentEventIDs[types.Hash256(input.ParentID)] = txnID + } + for _, input := range txn.SiafundInputs { + spentEventIDs[types.Hash256(input.ParentID)] = txnID + } + } + for _, txn := range cau.Block.V2Transactions() { + txnID := txn.ID() + for _, input := range txn.SiacoinInputs { + spentEventIDs[types.Hash256(input.Parent.ID)] = txnID + } + for _, input := range txn.SiafundInputs { + spentEventIDs[types.Hash256(input.Parent.ID)] = txnID + } + } + // add new siacoin elements to the store for _, sced := range cau.SiacoinElementDiffs() { sce := sced.SiacoinElement @@ -103,7 +133,14 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e continue } if sced.Spent { - applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, sce) + spentTxnID, ok := spentEventIDs[types.Hash256(sce.ID)] + if !ok { + panic(fmt.Errorf("missing transaction ID for spent siacoin element %v", sce.ID)) + } + applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, SpentSiacoinElement{ + SiacoinElement: sce, + EventID: spentTxnID, + }) } else { applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, sce) } @@ -118,7 +155,14 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e continue } if sfed.Spent { - applied.SpentSiafundElements = append(applied.SpentSiafundElements, sfe) + spentTxnID, ok := spentEventIDs[types.Hash256(sfe.ID)] + if !ok { + panic(fmt.Errorf("missing transaction ID for spent siafund element %v", sfe.ID)) + } + applied.SpentSiafundElements = append(applied.SpentSiafundElements, SpentSiafundElement{ + SiafundElement: sfe, + EventID: spentTxnID, + }) } else { applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, sfe) } From 4c810a6879f6a97c283406f1e2a2ee3c8f5ce0b5 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 19 Feb 2025 13:36:53 -0800 Subject: [PATCH 361/630] wallet, sqlite: add spent event methods --- persist/sqlite/utxo.go | 57 ++++++++++++++++++++++++++++++++++++++++++ wallet/manager.go | 26 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go index 881d700..3a8fc56 100644 --- a/persist/sqlite/utxo.go +++ b/persist/sqlite/utxo.go @@ -70,3 +70,60 @@ WHERE se.id=$1 AND spent_index_id IS NULL` } return } + +// SiacoinElementSpentEvent returns the event that spent a Siacoin UTXO. +func (s *Store) SiacoinElementSpentEvent(id types.SiacoinOutputID) (ev wallet.Event, spent bool, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT spent_event_id FROM siacoin_elements WHERE id=$1` + + var spentEventID sql.NullInt64 + err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } else if err != nil { + return fmt.Errorf("failed to query spent event ID: %w", err) + } else if !spentEventID.Valid { + return nil + } + + spent = true + events, err := getEventsByID(tx, []int64{spentEventID.Int64}) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } else if len(events) != 1 { + panic("expected exactly one event") // should never happen + } + ev = events[0] + return nil + }) + return +} + +// SiafundElementSpentEvent returns the event that spent a Siafund UTXO. +func (s *Store) SiafundElementSpentEvent(id types.SiafundOutputID) (ev wallet.Event, spent bool, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT spent_event_id FROM siafund_elements WHERE id=$1` + + var spentEventID sql.NullInt64 + err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } else if err != nil { + return fmt.Errorf("failed to query spent event ID: %w", err) + } else if !spentEventID.Valid { + return nil + } + + spent = true + events, err := getEventsByID(tx, []int64{spentEventID.Int64}) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } else if len(events) != 1 { + panic("expected exactly one event") // should never happen + } + ev = events[0] + return nil + }) + + return +} diff --git a/wallet/manager.go b/wallet/manager.go index 154744f..182e577 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -92,6 +92,16 @@ type ( SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + // SiacoinElementSpentEvent returns the event of a spent siacoin element. + // If the element is not spent, the return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiacoinElementSpentEvent(types.SiacoinOutputID) (Event, bool, error) + // SiafundElementSpentEvent returns the event of a spent siafund element. + // If the element is not spent, the second return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiafundElementSpentEvent(types.SiafundOutputID) (Event, bool, error) SetIndexMode(IndexMode) error LastCommittedIndex() (types.ChainIndex, error) @@ -560,6 +570,22 @@ func (m *Manager) SiafundElement(id types.SiafundOutputID) (types.SiafundElement return m.store.SiafundElement(id) } +// SiacoinElementSpentEvent returns the event of a spent siacoin element. +// If the element is not spent, the return value will be (Event{}, false, nil). +// If the element is not found, the error will be ErrNotFound. An element +// is only tracked for 144 blocks after it is spent. +func (m *Manager) SiacoinElementSpentEvent(id types.SiacoinOutputID) (Event, bool, error) { + return m.store.SiacoinElementSpentEvent(id) +} + +// SiafundElementSpentEvent returns the event of a spent siafund element. +// If the element is not spent, the second return value will be (Event{}, false, nil). +// If the element is not found, the error will be ErrNotFound. An element +// is only tracked for 144 blocks after it is spent. +func (m *Manager) SiafundElementSpentEvent(id types.SiafundOutputID) (Event, bool, error) { + return m.store.SiafundElementSpentEvent(id) +} + // Close closes the wallet manager. func (m *Manager) Close() error { m.tg.Stop() From 0d356b693a7138c3892df376cb98041ff1ae77b1 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 19 Feb 2025 13:37:03 -0800 Subject: [PATCH 362/630] api: add spent event tests --- api/api.go | 7 ++ api/api_test.go | 203 ++++++++++++++++++++++++++++++++++++++++++++++++ api/client.go | 14 ++++ api/server.go | 62 ++++++++++++++- 4 files changed, 284 insertions(+), 2 deletions(-) diff --git a/api/api.go b/api/api.go index e974bbb..4603c9b 100644 --- a/api/api.go +++ b/api/api.go @@ -187,3 +187,10 @@ type SiafundElementsResponse struct { Basis types.ChainIndex `json:"basis"` Outputs []types.SiafundElement `json:"outputs"` } + +// ElementSpentResponse is the response type for /outputs/siacoin/:id/spent and +// /outputs/siafund/:id/spent. +type ElementSpentResponse struct { + Spent bool `json:"spent"` + Event *wallet.Event `json:"event,omitempty"` +} diff --git a/api/api_test.go b/api/api_test.go index 48c77c8..b23b0b5 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1819,6 +1819,209 @@ func TestConstructV2Siacoins(t *testing.T) { } } +func TestSpentElement(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + // create wallets + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + peerStore, err := sqlite.NewPeerStore(ws) + if err != nil { + t.Fatal(err) + } + + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create the syncer + s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerListener.Addr().String(), + }) + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, s, wm) + + // trigger initial scan + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + sce, basis, err := c.AddressSiacoinOutputs(senderAddr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sce) != 1 { + t.Fatalf("expected 1 siacoin element, got %v", len(sce)) + } + + // check if the element is spent + spent, err := c.SpentSiacoinElement(sce[0].ID) + if err != nil { + t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siacoin element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siacoin element to have no event, got %v", spent.Event) + } + + // spend the element + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Value: sce[0].SiacoinOutput.Value, + Address: receiverAddr, + }, + }, + } + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + // check if the element is spent + spent, err = c.SpentSiacoinElement(sce[0].ID) + if err != nil { + t.Fatal(err) + } else if !spent.Spent { + t.Fatal("expected siacoin element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siacoin element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siacoin element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) + } + + // mine until the utxo is pruned + testutil.MineBlocks(t, cm, types.VoidAddress, 144) + waitForBlock(t, cm, ws) + + _, err = c.SpentSiacoinElement(sce[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) + } + + sfe, basis, err := c.AddressSiafundOutputs(senderAddr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfe) != 1 { + t.Fatalf("expected 1 siafund element, got %v", len(sfe)) + } + + // check if the siafund element is spent + // check if the element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) + if err != nil { + t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siafund element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siafund element to have no event, got %v", spent.Event) + } + + // spend the element + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfe[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + ClaimAddress: senderAddr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + { + Address: receiverAddr, + Value: sfe[0].SiafundOutput.Value, + }, + }, + } + cs, err = c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, ws) + + // check if the element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) + if err != nil { + t.Fatal(err) + } else if !spent.Spent { + t.Fatal("expected siafund element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siafund element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siafund element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) + } + + // mine until the utxo is pruned + testutil.MineBlocks(t, cm, types.VoidAddress, 144) + waitForBlock(t, cm, ws) + + _, err = c.SpentSiafundElement(sfe[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) + } +} + func TestConstructV2Siafunds(t *testing.T) { log := zaptest.NewLogger(t) diff --git a/api/client.go b/api/client.go index 456dbe1..236b772 100644 --- a/api/client.go +++ b/api/client.go @@ -257,6 +257,20 @@ func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { return } +// SpentSiacoinElement returns whether a siacoin output has been spent and the +// event that spent it. +func (c *Client) SpentSiacoinElement(id types.SiacoinOutputID) (resp ElementSpentResponse, err error) { + err = c.c.GET(fmt.Sprintf("/outputs/siacoin/%v/spent", id), &resp) + return +} + +// SpentSiafundElement returns whether a siafund output has been spent and the +// event that spent it. +func (c *Client) SpentSiafundElement(id types.SiafundOutputID) (resp ElementSpentResponse, err error) { + err = c.c.GET(fmt.Sprintf("/outputs/siafund/%v/spent", id), &resp) + return +} + // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { diff --git a/api/server.go b/api/server.go index 576fc9e..d49c0f2 100644 --- a/api/server.go +++ b/api/server.go @@ -117,6 +117,16 @@ type ( SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + // SiacoinElementSpentEvent returns the event of a spent siacoin element. + // If the element is not spent, the return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiacoinElementSpentEvent(types.SiacoinOutputID) (wallet.Event, bool, error) + // SiafundElementSpentEvent returns the event of a spent siafund element. + // If the element is not spent, the second return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiafundElementSpentEvent(types.SiafundOutputID) (wallet.Event, bool, error) Reserve([]types.Hash256) error Release([]types.Hash256) @@ -580,6 +590,52 @@ func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { }) } +func (s *server) outputsSiacoinSpentHandlerGET(jc jape.Context) { + var id types.SiacoinOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + event, spent, err := s.wm.SiacoinElementSpentEvent(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't load siacoin element", err) != nil { + return + } + + resp := ElementSpentResponse{ + Spent: spent, + } + if spent { + resp.Event = &event + } + + jc.Encode(resp) +} + +func (s *server) outputsSiafundSpentHandlerGET(jc jape.Context) { + var id types.SiafundOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + event, spent, err := s.wm.SiafundElementSpentEvent(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't load siafund element", err) != nil { + return + } + + resp := ElementSpentResponse{ + Spent: spent, + } + if spent { + resp.Event = &event + } + + jc.Encode(resp) +} + func (s *server) walletsReserveHandler(jc jape.Context) { var wrr WalletReserveRequest if jc.Decode(&wrr) != nil { @@ -1324,8 +1380,10 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /addresses/:addr/outputs/siacoin": wrapPublicAuthHandler(srv.addressesAddrOutputsSCHandler), "GET /addresses/:addr/outputs/siafund": wrapPublicAuthHandler(srv.addressesAddrOutputsSFHandler), - "GET /outputs/siacoin/:id": wrapPublicAuthHandler(srv.outputsSiacoinHandlerGET), - "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), + "GET /outputs/siacoin/:id": wrapPublicAuthHandler(srv.outputsSiacoinHandlerGET), + "GET /outputs/siacoin/:id/spent": wrapPublicAuthHandler(srv.outputsSiacoinSpentHandlerGET), + "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), + "GET /outputs/siafund/:id/spent": wrapPublicAuthHandler(srv.outputsSiafundSpentHandlerGET), "GET /events/:id": wrapPublicAuthHandler(srv.eventsHandlerGET), From 741ac74ff8b0379e700295ecf0ec876eddd152b1 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 19 Feb 2025 15:30:24 -0800 Subject: [PATCH 363/630] fix lint --- wallet/update.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wallet/update.go b/wallet/update.go index 1f3c4a3..da9544d 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -27,11 +27,15 @@ type ( Balance } + // SpentSiacoinElement pairs a spent siacoin element with the ID of the + // transaction that spent it. SpentSiacoinElement struct { types.SiacoinElement EventID types.TransactionID } + // SpentSiafundElement pairs a spent siafund element with the ID of the + // transaction that spent it. SpentSiafundElement struct { types.SiafundElement EventID types.TransactionID From 99a00baaef233de3e2709361b76aad0b3f7e8784 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 19 Feb 2025 15:30:41 -0800 Subject: [PATCH 364/630] add changeset --- .changeset/add_spent_element_endpoints.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .changeset/add_spent_element_endpoints.md diff --git a/.changeset/add_spent_element_endpoints.md b/.changeset/add_spent_element_endpoints.md new file mode 100644 index 0000000..f8148b0 --- /dev/null +++ b/.changeset/add_spent_element_endpoints.md @@ -0,0 +1,17 @@ +--- +default: minor +--- + +# Added Spent Element Endpoints + +Added two new endpoints `[GET] /outputs/siacoin/:id/spent` and `[GET] /outputs/siafund/:id/spent`. These endpoints will return a boolean, indicating whether the UTXO was spent, and the transaction it was spent in. These endpoints are designed to make verifying Atomic swaps easier. + +#### Example Usage + +```` +$ curl http://localhost:9980/api/outputs/siacoin/9b89152bb967130326702c9bfb51109e9f80274ec314ba58d9ef49b881340f2f/spent +{ + spent: true, + transaction: {} +} +``` From e8a7066ce378f470cbe1e72484df33eb2687bcf0 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Feb 2025 07:30:46 -0800 Subject: [PATCH 365/630] address review comments --- .changeset/add_spent_element_endpoints.md | 2 +- api/server.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/add_spent_element_endpoints.md b/.changeset/add_spent_element_endpoints.md index f8148b0..a117027 100644 --- a/.changeset/add_spent_element_endpoints.md +++ b/.changeset/add_spent_element_endpoints.md @@ -12,6 +12,6 @@ Added two new endpoints `[GET] /outputs/siacoin/:id/spent` and `[GET] /outputs/s $ curl http://localhost:9980/api/outputs/siacoin/9b89152bb967130326702c9bfb51109e9f80274ec314ba58d9ef49b881340f2f/spent { spent: true, - transaction: {} + event: {} } ``` diff --git a/api/server.go b/api/server.go index d49c0f2..29c61a3 100644 --- a/api/server.go +++ b/api/server.go @@ -599,6 +599,7 @@ func (s *server) outputsSiacoinSpentHandlerGET(jc jape.Context) { event, spent, err := s.wm.SiacoinElementSpentEvent(id) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) + return } else if jc.Check("couldn't load siacoin element", err) != nil { return } @@ -622,6 +623,7 @@ func (s *server) outputsSiafundSpentHandlerGET(jc jape.Context) { event, spent, err := s.wm.SiafundElementSpentEvent(id) if errors.Is(err, wallet.ErrNotFound) { jc.Error(err, http.StatusNotFound) + return } else if jc.Check("couldn't load siafund element", err) != nil { return } From 5c5c1b4e48611793466faa4f3aeeec1b87afaf40 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Feb 2025 09:26:55 -0800 Subject: [PATCH 366/630] revert out of scope changes --- persist/sqlite/consensus.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 46fae99..cf1dcce 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -735,9 +735,10 @@ func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error balanceChanges[addrRef.ID] = addrRef.Balance } - if res, err := stmt.Exec(encode(se.ID)); err != nil { + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { return err - } else if n, _ := res.RowsAffected(); n == 0 { + } else if errors.Is(err, sql.ErrNoRows) { continue // skip if the element does not exist } @@ -786,7 +787,7 @@ func spendSiacoinElements(tx *txn, elements []wallet.SpentSiacoinElement, indexI } defer getEventIDStmt.Close() - stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL`) + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -806,9 +807,10 @@ func spendSiacoinElements(tx *txn, elements []wallet.SpentSiacoinElement, indexI return fmt.Errorf("failed to get event ID: %w", err) } - if res, err := stmt.Exec(indexID, eventDBID, encode(se.ID)); err != nil { + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, eventDBID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { return err - } else if n, _ := res.RowsAffected(); n == 0 { + } else if errors.Is(err, sql.ErrNoRows) { continue // skip if the element does not exist } @@ -1015,9 +1017,10 @@ func spendSiafundElements(tx *txn, elements []wallet.SpentSiafundElement, indexI return fmt.Errorf("failed to get event ID: %w", err) } - if res, err := stmt.Exec(indexID, eventDBID, encode(se.ID)); err != nil { + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, eventDBID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { return err - } else if n, _ := res.RowsAffected(); n == 0 { + } else if errors.Is(err, sql.ErrNoRows) { continue // skip if the element does not exist } @@ -1064,7 +1067,7 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error } defer done() - stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL`) + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -1079,9 +1082,10 @@ func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error balanceChanges[addrRef.ID] = addrRef.Balance } - if res, err := stmt.Exec(encode(se.ID)); err != nil { + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { return err - } else if n, _ := res.RowsAffected(); n == 0 { + } else if errors.Is(err, sql.ErrNoRows) { continue // skip if the element does not exist } From 7b81d59f161bd6851aea795689c645b829f149bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 18:21:39 +0000 Subject: [PATCH 367/630] build(deps): bump go.sia.tech/core in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/core](https://github.com/SiaFoundation/core). Updates `go.sia.tech/core` from 0.10.1 to 0.10.2 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.10.1...v0.10.2) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a08e0c2..f63b11c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.1 + go.sia.tech/core v0.10.2 go.sia.tech/coreutils v0.11.1 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.0 diff --git a/go.sum b/go.sum index a9184db..116007e 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.1 h1:96lmgO50oKPiQU46H14Ga+6NYo6IB++VQ4DI3QCc6/o= -go.sia.tech/core v0.10.1/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk= +go.sia.tech/core v0.10.2 h1:flwT7DUFZ/CQKBWJaoy3etvy2EWVtrwXmYITjihkYQ4= +go.sia.tech/core v0.10.2/go.mod h1:7kpJAs7Ju4Ho72Y/rK/Q3lgKXShn8A7KPn+QpM/3SXQ= go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo= go.sia.tech/coreutils v0.11.1/go.mod h1:vnY0haOx1InIQR0Pc5YAXDe4WnF6po8dv5bNP73CAnE= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= From 04329c05fd72b90bf0e0cd2ffa4d95278d8e4f17 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 26 Feb 2025 16:05:58 -0800 Subject: [PATCH 368/630] add ed25519 key store --- .changeset/add_signing_key_store.md | 25 + api/api.go | 21 + api/api_test.go | 1949 +++++++-------------------- api/client.go | 31 + api/server.go | 75 +- cmd/walletd/node.go | 6 +- internal/testutil/testutil.go | 102 ++ keys/manager.go | 80 ++ keys/manager_test.go | 57 + persist/sqlite/init.sql | 5 + persist/sqlite/keys.go | 48 + persist/sqlite/keys_test.go | 44 + persist/sqlite/migrations.go | 9 + wallet/wallet.go | 14 + 14 files changed, 981 insertions(+), 1485 deletions(-) create mode 100644 .changeset/add_signing_key_store.md create mode 100644 internal/testutil/testutil.go create mode 100644 keys/manager.go create mode 100644 keys/manager_test.go create mode 100644 persist/sqlite/keys.go create mode 100644 persist/sqlite/keys_test.go diff --git a/.changeset/add_signing_key_store.md b/.changeset/add_signing_key_store.md new file mode 100644 index 0000000..9436d6f --- /dev/null +++ b/.changeset/add_signing_key_store.md @@ -0,0 +1,25 @@ +--- +default: minor +--- + +# Add ed25519 key store + +Adds an optional ed25519 signing key store for integrators to store arbitrary private keys for signing transactions. It allows for both generating private keys on the server and importing private keys. + + +*The endpoint will return 404 if the `--public` CLI flag is set. It is only recommended for use on localhost. It is not used by the UI.* + +```go + +client := api.NewClient(walletAddr, walletdPassword) + +pubKey, err := client.GenerateSigningKey() +if err != nil { + panic(err) +} + +sig, err := client.SignHash(pubKey, hash) +if err != nil { + panic(err) +} +``` diff --git a/api/api.go b/api/api.go index 4603c9b..cbcf818 100644 --- a/api/api.go +++ b/api/api.go @@ -194,3 +194,24 @@ type ElementSpentResponse struct { Spent bool `json:"spent"` Event *wallet.Event `json:"event,omitempty"` } + +// An AddSigningKeyRequest is a request to add an ed25519 signing key to +// key store. +type AddSigningKeyRequest struct { + PrivateKey types.PrivateKey `json:"privateKey"` +} + +// An AddSigningKeyResponse is the response to an AddSigningKeyRequest. +type AddSigningKeyResponse struct { + PublicKey types.PublicKey `json:"publicKey"` +} + +// A SignHashRequest is a request to sign a hash with a key. +type SignHashRequest struct { + Hash types.Hash256 `json:"hash"` +} + +// A SignHashResponse is the response to a SignHashRequest. +type SignHashResponse struct { + Signature types.Signature `json:"signature"` +} diff --git a/api/api_test.go b/api/api_test.go index b23b0b5..48670ff 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -2,110 +2,68 @@ package api_test import ( "bytes" - "context" "encoding/hex" "encoding/json" "fmt" "net" "net/http" - "path/filepath" "reflect" "strings" "testing" "time" - "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" "go.sia.tech/core/types" - "go.sia.tech/coreutils" - "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/syncer" - "go.sia.tech/coreutils/testutil" "go.sia.tech/jape" "go.sia.tech/walletd/api" - "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/internal/testutil" + "go.sia.tech/walletd/keys" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) -func testNetwork() (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 5 - n.HardforkV2.RequireHeight = 10 - return n, genesisBlock -} - -func runServer(t *testing.T, cm api.ChainManager, s api.Syncer, wm api.WalletManager) *api.Client { - t.Helper() +func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logger, walletOpts ...wallet.Option) *api.Client { + tb.Helper() l, err := net.Listen("tcp", ":0") if err != nil { - t.Fatal("failed to listen:", err) + tb.Fatal("failed to listen:", err) + } + tb.Cleanup(func() { l.Close() }) + + wm, err := wallet.NewManager(cn.Chain, cn.Store, append([]wallet.Option{wallet.WithLogger(log.Named("wallet"))}, walletOpts...)...) + if err != nil { + tb.Fatal("failed to create wallet manager:", err) } - t.Cleanup(func() { l.Close() }) + tb.Cleanup(func() { wm.Close() }) + + km := keys.NewManager(cn.Store) + tb.Cleanup(func() { km.Close() }) server := &http.Server{ - Handler: api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t))), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, km, api.WithDebug(), api.WithLogger(log)), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } - t.Cleanup(func() { server.Close() }) + tb.Cleanup(func() { server.Close() }) go server.Serve(l) return api.NewClient("http://"+l.Addr().String(), "password") } -func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { - for i := 0; i < 1000; i++ { - time.Sleep(10 * time.Millisecond) - tip, _ := ws.LastCommittedIndex() - if tip == cm.Tip() { - return - } - } - tb.Fatal("timed out waiting for block") -} - func TestWalletAdd(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() + n, genesisBlock := testutil.V1Network() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ Value: types.Siacoins(1), Address: giftAddress, } - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - c := runServer(t, cm, nil, wm) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) checkWalletResponse := func(wr api.WalletUpdateRequest, w wallet.Wallet, isUpdate bool) error { // check wallet @@ -252,51 +210,16 @@ func TestWallet(t *testing.T) { defer syncerListener.Close() // create chain manager - n, genesisBlock := testNetwork() + n, genesisBlock := testutil.V1Network() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ Value: types.Siacoins(1), Address: giftAddress, } + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - // create the sqlite store - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) - - // create the wallet manager - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - // create seed address vault - sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) - - // run server - c := runServer(t, cm, s, wm) w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) @@ -307,7 +230,7 @@ func TestWallet(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - waitForBlock(t, cm, ws) + cn.WaitForSync(t) balance, err := wc.Balance() if err != nil { @@ -333,8 +256,12 @@ func TestWallet(t *testing.T) { } // create and add an address - addr := sav.NewAddress("primary") - if err := wc.AddAddress(addr); err != nil { + sk2 := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(sk2.PublicKey()) + err = wc.AddAddress(wallet.Address{ + Address: addr, + }) + if err != nil { t.Fatal(err) } @@ -344,7 +271,7 @@ func TestWallet(t *testing.T) { t.Fatal(err) } else if len(addresses) != 1 { t.Fatal("address list should have one address") - } else if addresses[0].Address != addr.Address { + } else if addresses[0].Address != addr { t.Fatalf("address should be %v, got %v", addr, addresses[0]) } @@ -356,19 +283,25 @@ func TestWallet(t *testing.T) { UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), }}, SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, - {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, + {Address: addr, Value: types.Siacoins(1).Div64(2)}, + {Address: addr, Value: types.Siacoins(1).Div64(2)}, }, Signatures: []types.TransactionSignature{{ ParentID: types.Hash256(giftSCOID), CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } - sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if err := c.TxpoolBroadcast(cm.Tip(), []types.Transaction{txn}, nil); err != nil { + if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } @@ -386,22 +319,8 @@ func TestWallet(t *testing.T) { } else if len(unconfirmed) != 1 { t.Fatal("txpool should have one transaction") } - - cs := cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: []types.Transaction{txn}, - } - - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, ws) + // confirm the transaction + cn.MineBlocks(t, types.VoidAddress, 1) // get new balance balance, err = wc.Balance() @@ -426,24 +345,13 @@ func TestWallet(t *testing.T) { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) - } else if basis != cm.Tip() { - t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) } // mine a block to add an immature balance - cs = cm.TipState() - b = types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: addr.Address, Value: cs.BlockReward()}}, - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, ws) + expectedPayout := cn.Chain.TipState().BlockReward() + cn.MineBlocks(t, addr, 1) // get new balance balance, err = wc.Balance() @@ -451,28 +359,13 @@ func TestWallet(t *testing.T) { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { t.Fatal("balance should be 1 SC, got", balance.Siacoins) - } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { - t.Fatalf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("immature balance should be %d SC, got %d SC", expectedPayout, balance.ImmatureSiacoins) } // mine enough blocks for the miner payout to mature - expectedBalance := types.Siacoins(1).Add(b.MinerPayouts[0].Value) - target := cs.MaturityHeight() - for cs.Index.Height < target { - cs = cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm, ws) + expectedBalance := types.Siacoins(1).Add(expectedPayout) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) // get new balance balance, err = wc.Balance() @@ -488,7 +381,7 @@ func TestWallet(t *testing.T) { func TestAddresses(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() + n, genesisBlock := testutil.V1Network() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ @@ -496,76 +389,21 @@ func TestAddresses(t *testing.T) { Address: giftAddress, } - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + sk2 := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(sk2.PublicKey()) - sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) - c := runServer(t, cm, nil, wm) + // personal index mode requires a wallet for indexing w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { t.Fatal(err) - } else if w.Name != "primary" { - t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) } wc := c.Wallet(w.ID) - if err := c.Rescan(0); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, ws) - - balance, err := wc.Balance() - if err != nil { - t.Fatal(err) - } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { - t.Fatal("balance should be 0") - } - - // shouldn't have any events yet - events, err := wc.Events(0, -1) + err = wc.AddAddress(wallet.Address{Address: addr}) if err != nil { t.Fatal(err) - } else if len(events) != 0 { - t.Fatal("event history should be empty") - } - - // shouldn't have any addresses yet - addresses, err := wc.Addresses() - if err != nil { - t.Fatal(err) - } else if len(addresses) != 0 { - t.Fatal("address list should be empty") - } - - // create and add an address - addr := sav.NewAddress("primary") - if err := wc.AddAddress(addr); err != nil { - t.Fatal(err) - } - - // should have an address now - addresses, err = wc.Addresses() - if err != nil { - t.Fatal(err) - } else if len(addresses) != 1 { - t.Fatal("address list should have one address") - } else if addresses[0].Address != addr.Address { - t.Fatalf("address should be %v, got %v", addr, addresses[0]) } // send gift to wallet @@ -576,34 +414,31 @@ func TestAddresses(t *testing.T) { UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), }}, SiacoinOutputs: []types.SiacoinOutput{ - {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, - {Address: addr.Address, Value: types.Siacoins(1).Div64(2)}, + {Address: addr, Value: types.Siacoins(1).Div64(2)}, + {Address: addr, Value: types.Siacoins(1).Div64(2)}, }, Signatures: []types.TransactionSignature{{ ParentID: types.Hash256(giftSCOID), CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } - sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) - txn.Signatures[0].Signature = sig[:] - cs := cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: []types.Transaction{txn}, - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) } - if err := cm.AddBlocks([]types.Block{b}); err != nil { + + sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction to the transaction pool + if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } - waitForBlock(t, cm, ws) + cn.MineBlocks(t, types.VoidAddress, 1) // get new balance - balance, err = c.AddressBalance(addr.Address) + balance, err := c.AddressBalance(addr) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { @@ -613,68 +448,42 @@ func TestAddresses(t *testing.T) { } // transaction should appear in history - events, err = c.AddressEvents(addr.Address, 0, 100) + events, err := c.AddressEvents(addr, 0, 100) if err != nil { t.Fatal(err) } else if len(events) == 0 { t.Fatal("transaction should appear in history") } - outputs, basis, err := c.AddressSiacoinOutputs(addr.Address, 0, 100) + outputs, basis, err := c.AddressSiacoinOutputs(addr, 0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) - } else if basis != cm.Tip() { - t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) } // mine a block to add an immature balance - cs = cm.TipState() - b = types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: addr.Address, Value: cs.BlockReward()}}, - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm, ws) + expectedPayout := cn.Chain.TipState().BlockReward() + cn.MineBlocks(t, addr, 1) // get new balance - balance, err = c.AddressBalance(addr.Address) + balance, err = c.AddressBalance(addr) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { t.Fatal("balance should be 1 SC, got", balance.Siacoins) - } else if !balance.ImmatureSiacoins.Equals(b.MinerPayouts[0].Value) { - t.Fatalf("immature balance should be %d SC, got %d SC", b.MinerPayouts[0].Value, balance.ImmatureSiacoins) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("immature balance should be %d SC, got %d SC", expectedPayout, balance.ImmatureSiacoins) } // mine enough blocks for the miner payout to mature - expectedBalance := types.Siacoins(1).Add(b.MinerPayouts[0].Value) - target := cs.MaturityHeight() - for cs.Index.Height < target { - cs = cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm, ws) + expectedBalance := types.Siacoins(1).Add(expectedPayout) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) // get new balance - balance, err = c.AddressBalance(addr.Address) + balance, err = c.AddressBalance(addr) if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(expectedBalance) { @@ -684,734 +493,120 @@ func TestAddresses(t *testing.T) { } } -func TestV2(t *testing.T) { +func TestConsensusUpdates(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() - // gift primary wallet some coins - primaryPrivateKey := types.GeneratePrivateKey() - primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress - // secondary wallet starts with nothing - secondaryPrivateKey := types.GeneratePrivateKey() - secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) + n, genesisBlock := testutil.V1Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + cn.MineBlocks(t, types.VoidAddress, 10) - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + reverted, applied, err := c.ConsensusUpdates(types.ChainIndex{}, 10) if err != nil { t.Fatal(err) + } else if len(reverted) != 0 { + t.Fatal("expected no reverted blocks") + } else if len(applied) != 11 { // genesis + 10 mined blocks (chain manager off-by-one) + t.Fatalf("expected 11 applied blocks, got %v", len(applied)) } - cm := chain.NewManager(dbstore, tipState) - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) + + for i, cau := range applied { + // using i for height since we're testing the update contents + expected, ok := cn.Chain.BestIndex(uint64(i)) + if !ok { + t.Fatalf("failed to get expected index for block %v", i) + } else if cau.State.Index != expected { + t.Fatalf("expected index %v, got %v", expected, cau.State.Index) + } else if cau.State.Network.Name != n.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. + t.Fatalf("expected network to be %q, got %q", n.Name, cau.State.Network.Name) + } } - defer ws.Close() - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) +} + +func TestConstructSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V1Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, } - defer wm.Close() - c := runServer(t, cm, nil, wm) - primaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) if err != nil { t.Fatal(err) } - primary := c.Wallet(primaryWallet.ID) - if err := primary.AddAddress(wallet.Address{Address: primaryAddress}); err != nil { - t.Fatal(err) - } - secondaryWallet, err := c.AddWallet(api.WalletUpdateRequest{Name: "secondary"}) + + wc := c.Wallet(w.ID) + // add an address with no spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + }) if err != nil { t.Fatal(err) } - secondary := c.Wallet(secondaryWallet.ID) - if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { - t.Fatal(err) - } if err := c.Rescan(0); err != nil { t.Fatal(err) } - waitForBlock(t, cm, ws) + cn.MineBlocks(t, types.VoidAddress, 1) - // define some helper functions - addBlock := func(txns []types.Transaction, v2txns []types.V2Transaction) error { - cs := cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: txns, - } - if v2txns != nil { - b.V2 = &types.V2BlockData{ - Height: cs.Index.Height + 1, - Transactions: v2txns, - } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - return cm.AddBlocks([]types.Block{b}) - } - checkBalances := func(p, s types.Currency) { - t.Helper() - waitForBlock(t, cm, ws) - if primaryBalance, err := primary.Balance(); err != nil { - t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) - } - if secondaryBalance, err := secondary.Balance(); err != nil { - t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) - } + // try to construct a valid transaction with no spend policy + _, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if !strings.Contains(err.Error(), "no spend policy") { + t.Fatalf("expected error to contain %q, got %q", "no spend policy", err) } - sendV1 := func() error { - t.Helper() - waitForBlock(t, cm, ws) - // which wallet is sending? - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, basis, err := primary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } else if basis != cm.Tip() { - t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) - } - if len(sces) == 0 { - sces, basis, err = secondary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } else if basis != cm.Tip() { - t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) - } - key = secondaryPrivateKey - dest = primaryAddress - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - - txn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - Signatures: []types.TransactionSignature{{ - ParentID: types.Hash256(sce.ID), - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }}, - } - sig := key.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(sce.ID), 0, 0, nil)) - txn.Signatures[0].Signature = sig[:] - if err := addBlock([]types.Transaction{txn}, nil); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil + // add the spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey())), + }, + }) + if err != nil { + t.Fatal(err) } - sendV2 := func() error { - t.Helper() - waitForBlock(t, cm, ws) - - // which wallet is sending? - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, basis, err := primary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } else if basis != cm.Tip() { - t.Fatalf("basis should be %v, got %v", cm.Tip(), basis) - } - if len(sces) == 0 { - sces, _, err = secondary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } - key = secondaryPrivateKey - dest = primaryAddress - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cm.TipState().InputSigHash(txn))} - if err := addBlock(nil, []types.V2Transaction{txn}); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil + // try to construct a transaction with more siafunds than the wallet holds. + // this will lock all of the wallet's siacoins + resp, err := wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, []types.SiafundOutput{ + {Value: 100000, Address: senderAddr}, + }, senderAddr) + if !strings.Contains(err.Error(), "insufficient funds") { + t.Fatal(err) } - // attempt to send primary->secondary with a v2 txn; should fail - if err := sendV2(); err == nil { - t.Fatal("expected v2 txn to be rejected") - } - // use a v1 transaction instead - if err := sendV1(); err != nil { - t.Fatal(err) - } - - // mine past v2 allow height - for cm.Tip().Height <= n.HardforkV2.AllowHeight { - if err := addBlock(nil, nil); err != nil { - t.Fatal(err) - } - } - // now send coins back with a v2 transaction - if err := sendV2(); err != nil { - t.Fatal(err) - } - // v1 transactions should also still work - if err := sendV1(); err != nil { - t.Fatal(err) - } - - // mine past v2 require height - for cm.Tip().Height <= n.HardforkV2.RequireHeight { - if err := addBlock(nil, nil); err != nil { - t.Fatal(err) - } - } - // v1 transactions should no longer work - if err := sendV1(); err == nil { - t.Fatal("expected v1 txn to be rejected") - } - // use a v2 transaction instead - if err := sendV2(); err != nil { - t.Fatal(err) - } -} - -func TestP2P(t *testing.T) { - t.Skip("flaky test") // TODO refactor - - logger := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() - // gift primary wallet some coins - primaryPrivateKey := types.GeneratePrivateKey() - primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress - // secondary wallet starts with nothing - secondaryPrivateKey := types.GeneratePrivateKey() - secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) - - // create wallets - dbstore1, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - log1 := logger.Named("one") - cm1 := chain.NewManager(dbstore1, tipState) - store1, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log1.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer store1.Close() - - peerStore, err := sqlite.NewPeerStore(store1) - if err != nil { - t.Fatal(err) - } - - wm1, err := wallet.NewManager(cm1, store1, wallet.WithLogger(log1.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm1.Close() - - l1, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer l1.Close() - s1 := syncer.New(l1, cm1, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l1.Addr().String(), - }) - go s1.Run() - defer s1.Close() - c1 := runServer(t, cm1, s1, wm1) - w1, err := c1.AddWallet(api.WalletUpdateRequest{Name: "primary"}) - if err != nil { - t.Fatal(err) - } - primary := c1.Wallet(w1.ID) - if err := primary.AddAddress(wallet.Address{Address: primaryAddress}); err != nil { - t.Fatal(err) - } - if err := c1.Rescan(0); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm1, store1) - - dbstore2, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - log2 := logger.Named("two") - cm2 := chain.NewManager(dbstore2, tipState) - store2, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log2.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer store2.Close() - wm2, err := wallet.NewManager(cm2, store2, wallet.WithLogger(log2.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm2.Close() - - l2, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer l2.Close() - s2 := syncer.New(l2, cm2, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l2.Addr().String(), - }, syncer.WithLogger(zaptest.NewLogger(t))) - go s2.Run() - defer s2.Close() - c2 := runServer(t, cm2, s2, wm2) - - w2, err := c2.AddWallet(api.WalletUpdateRequest{Name: "secondary"}) - if err != nil { - t.Fatal(err) - } - secondary := c2.Wallet(w2.ID) - if err := secondary.AddAddress(wallet.Address{Address: secondaryAddress}); err != nil { - t.Fatal(err) - } - if err := c2.Rescan(0); err != nil { - t.Fatal(err) - } - waitForBlock(t, cm2, store2) - - // define some helper functions - addBlock := func() error { - // choose a client at random - c := c1 - if frand.Intn(2) == 0 { - c = c2 - } - - cs, err := c.ConsensusTipState() - if err != nil { - return err - } - - _, txns, v2txns, err := c.TxpoolTransactions() - if err != nil { - return err - } - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: txns, - } - if len(v2txns) > 0 { - b.V2 = &types.V2BlockData{ - Height: cs.Index.Height + 1, - Transactions: v2txns, - } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := c.SyncerBroadcastBlock(b); err != nil { - return err - } - // wait for tips to update - again: - time.Sleep(10 * time.Millisecond) - if tip1, err := c1.ConsensusTip(); err != nil { - return err - } else if tip2, err := c2.ConsensusTip(); err != nil { - return err - } else if tip1 == cs.Index || tip2 == cs.Index { - goto again - } - return nil - } - checkBalances := func(p, s types.Currency) { - t.Helper() - waitForBlock(t, cm1, store1) - waitForBlock(t, cm2, store2) - if primaryBalance, err := primary.Balance(); err != nil { - t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) - } - if secondaryBalance, err := secondary.Balance(); err != nil { - t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) - } - } - sendV1 := func() error { - t.Helper() - - // which wallet is sending? - c := c1 - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } - if len(sces) == 0 { - c = c2 - key = secondaryPrivateKey - dest = primaryAddress - sces, _, err = secondary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - - txn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - Signatures: []types.TransactionSignature{{ - ParentID: types.Hash256(sce.ID), - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }}, - } - cs, err := c.ConsensusTipState() - if err != nil { - return err - } - sig := key.SignHash(cs.WholeSigHash(txn, types.Hash256(sce.ID), 0, 0, nil)) - txn.Signatures[0].Signature = sig[:] - if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { - return err - } else if err := addBlock(); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil - } - sendV2 := func() error { - t.Helper() - - // which wallet is sending? - c := c1 - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } - if len(sces) == 0 { - c = c2 - key = secondaryPrivateKey - dest = primaryAddress - sces, _, err = secondary.SiacoinOutputs(0, 100) - if err != nil { - t.Fatal(err) - } - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - } - cs, err := c.ConsensusTipState() - if err != nil { - return err - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cs.InputSigHash(txn))} - if err := c.TxpoolBroadcast(cs.Index, nil, []types.V2Transaction{txn}); err != nil { - return err - } else if err := addBlock(); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil - } - - // connect the syncers - if _, err := s1.Connect(context.Background(), s2.Addr()); err != nil { - t.Fatal(err) - } - - // attempt to send primary->secondary with a v2 txn; should fail - if err := sendV2(); err == nil { - t.Fatal("expected v2 txn to be rejected") - } - // use a v1 transaction instead - if err := sendV1(); err != nil { - t.Fatal(err) - } - - // mine past v2 allow height - for cm1.Tip().Height <= n.HardforkV2.AllowHeight { - if err := addBlock(); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm1, store1) - // now send coins back with a v2 transaction - if err := sendV2(); err != nil { - t.Fatal(err) - } - // v1 transactions should also still work - if err := sendV1(); err != nil { - t.Fatal(err) - } - - // mine past v2 require height - for cm1.Tip().Height <= n.HardforkV2.RequireHeight { - if err := addBlock(); err != nil { - t.Fatal(err) - } - } - waitForBlock(t, cm1, store1) - // v1 transactions should no longer work - if err := sendV1(); err == nil { - t.Fatal("expected v1 txn to be rejected") - } - // use a v2 transaction instead - if err := sendV2(); err != nil { - t.Fatal(err) - } -} - -func TestConsensusUpdates(t *testing.T) { - log := zaptest.NewLogger(t) - - n, genesisBlock := testNetwork() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - c := runServer(t, cm, nil, wm) - - for i := 0; i < 10; i++ { - b, ok := coreutils.MineBlock(cm, types.VoidAddress, time.Second) - if !ok { - t.Fatal("failed to mine block") - } else if err := cm.AddBlocks([]types.Block{b}); err != nil { - t.Fatal(err) - } - } - - waitForBlock(t, cm, ws) - - reverted, applied, err := c.ConsensusUpdates(types.ChainIndex{}, 10) - if err != nil { - t.Fatal(err) - } else if len(reverted) != 0 { - t.Fatal("expected no reverted blocks") - } else if len(applied) != 11 { // genesis + 10 mined blocks (chain manager off-by-one) - t.Fatalf("expected 11 applied blocks, got %v", len(applied)) - } - - for i, cau := range applied { - // using i for height since we're testing the update contents - expected, ok := cm.BestIndex(uint64(i)) - if !ok { - t.Fatalf("failed to get expected index for block %v", i) - } else if cau.State.Index != expected { - t.Fatalf("expected index %v, got %v", expected, cau.State.Index) - } else if cau.State.Network.Name != n.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. - t.Fatalf("expected network to be %q, got %q", n.Name, cau.State.Network.Name) - } - } -} - -func TestConstructSiacoins(t *testing.T) { - log := zaptest.NewLogger(t) - - n, genesisBlock := testNetwork() - senderPrivateKey := types.GeneratePrivateKey() - senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - c := runServer(t, cm, s, wm) - - w, err := c.AddWallet(api.WalletUpdateRequest{ - Name: "primary", - }) - if err != nil { - t.Fatal(err) - } - - wc := c.Wallet(w.ID) - // add an address with no spend policy - err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - }) - if err != nil { - t.Fatal(err) - } - - if err := c.Rescan(0); err != nil { - t.Fatal(err) - } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) - - // try to construct a valid transaction with no spend policy - _, err = wc.Construct([]types.SiacoinOutput{ - {Value: types.Siacoins(1), Address: receiverAddr}, - }, nil, senderAddr) - if !strings.Contains(err.Error(), "no spend policy") { - t.Fatalf("expected error to contain %q, got %q", "no spend policy", err) - } - - // add the spend policy - err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - SpendPolicy: &types.SpendPolicy{ - Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey())), - }, - }) - if err != nil { - t.Fatal(err) - } - - // try to construct a transaction with more siafunds than the wallet holds. - // this will lock all of the wallet's siacoins - resp, err := wc.Construct([]types.SiacoinOutput{ - {Value: types.Siacoins(1), Address: receiverAddr}, - }, []types.SiafundOutput{ - {Value: 100000, Address: senderAddr}, - }, senderAddr) - if !strings.Contains(err.Error(), "insufficient funds") { - t.Fatal(err) - } - - // construct a transaction with a single siacoin output - // this will fail if the utxos were not unlocked - resp, err = wc.Construct([]types.SiacoinOutput{ - {Value: types.Siacoins(1), Address: receiverAddr}, - }, nil, senderAddr) - if err != nil { + // construct a transaction with a single siacoin output + // this will fail if the utxos were not unlocked + resp, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if err != nil { t.Fatal(err) } @@ -1458,9 +653,7 @@ func TestConstructSiacoins(t *testing.T) { case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + cn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1482,7 +675,7 @@ func TestConstructSiacoins(t *testing.T) { func TestConstructSiafunds(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() + n, genesisBlock := testutil.V1Network() senderPrivateKey := types.GeneratePrivateKey() senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} senderAddr := senderPolicy.Address() @@ -1497,44 +690,8 @@ func TestConstructSiafunds(t *testing.T) { } genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - c := runServer(t, cm, s, wm) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -1555,9 +712,7 @@ func TestConstructSiafunds(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + cn.MineBlocks(t, types.VoidAddress, 1) resp, err := wc.Construct(nil, []types.SiafundOutput{ {Value: 1, Address: receiverAddr}, @@ -1616,9 +771,7 @@ func TestConstructSiafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + cn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1642,58 +795,22 @@ func TestConstructSiafunds(t *testing.T) { func TestConstructV2Siacoins(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() - senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, } - defer wm.Close() - c := runServer(t, cm, s, wm) + cm := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cm, log) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -1714,9 +831,7 @@ func TestConstructV2Siacoins(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + cm.MineBlocks(t, types.VoidAddress, 1) // try to construct a transaction resp, err := wc.ConstructV2([]types.SiacoinOutput{ @@ -1798,9 +913,7 @@ func TestConstructV2Siacoins(t *testing.T) { case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + cm.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1819,7 +932,7 @@ func TestConstructV2Siacoins(t *testing.T) { } } -func TestSpentElement(t *testing.T) { +func TestConstructV2Siafunds(t *testing.T) { log := zaptest.NewLogger(t) n, genesisBlock := testutil.V2Network() @@ -1837,192 +950,96 @@ func TestSpentElement(t *testing.T) { } genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) - syncerListener, err := net.Listen("tcp", ":0") + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) if err != nil { t.Fatal(err) } - defer syncerListener.Close() - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, }) - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) if err != nil { t.Fatal(err) } - defer wm.Close() - c := runServer(t, cm, s, wm) - - // trigger initial scan - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) - - sce, basis, err := c.AddressSiacoinOutputs(senderAddr, 0, 100) - if err != nil { + if err := c.Rescan(0); err != nil { t.Fatal(err) - } else if len(sce) != 1 { - t.Fatalf("expected 1 siacoin element, got %v", len(sce)) } + cn.MineBlocks(t, types.VoidAddress, 1) - // check if the element is spent - spent, err := c.SpentSiacoinElement(sce[0].ID) + resp, err := wc.ConstructV2(nil, []types.SiafundOutput{ + {Value: 1, Address: receiverAddr}, + }, senderAddr) if err != nil { t.Fatal(err) - } else if spent.Spent { - t.Fatal("expected siacoin element to be unspent") - } else if spent.Event != nil { - t.Fatalf("expected siacoin element to have no event, got %v", spent.Event) } - // spend the element - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{ - { - Parent: sce[0], - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: senderPolicy, - }, - }, - }, - SiacoinOutputs: []types.SiacoinOutput{ - { - Value: sce[0].SiacoinOutput.Value, - Address: receiverAddr, - }, - }, - } cs, err := c.ConsensusTipState() if err != nil { t.Fatal(err) } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ - senderPrivateKey.SignHash(cs.InputSigHash(txn)), - } - - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { - t.Fatal(err) - } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) - - // check if the element is spent - spent, err = c.SpentSiacoinElement(sce[0].ID) - if err != nil { - t.Fatal(err) - } else if !spent.Spent { - t.Fatal("expected siacoin element to be spent") - } else if types.TransactionID(spent.Event.ID) != txn.ID() { - t.Fatalf("expected siacoin element to have event %q, got %q", txn.ID(), spent.Event.ID) - } else if spent.Event.Type != wallet.EventTypeV2Transaction { - t.Fatalf("expected siacoin element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) - } - - // mine until the utxo is pruned - testutil.MineBlocks(t, cm, types.VoidAddress, 144) - waitForBlock(t, cm, ws) - _, err = c.SpentSiacoinElement(sce[0].ID) - if !strings.Contains(err.Error(), "not found") { - t.Fatalf("expected error to contain %q, got %q", "not found", err) + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + sig := senderPrivateKey.SignHash(sigHash) + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - - sfe, basis, err := c.AddressSiafundOutputs(senderAddr, 0, 100) - if err != nil { - t.Fatal(err) - } else if len(sfe) != 1 { - t.Fatalf("expected 1 siafund element, got %v", len(sfe)) + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - // check if the siafund element is spent - // check if the element is spent - spent, err = c.SpentSiafundElement(sfe[0].ID) - if err != nil { + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) - } else if spent.Spent { - t.Fatal("expected siafund element to be unspent") - } else if spent.Event != nil { - t.Fatalf("expected siafund element to have no event, got %v", spent.Event) } - // spend the element - txn = types.V2Transaction{ - SiafundInputs: []types.V2SiafundInput{ - { - Parent: sfe[0], - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: senderPolicy, - }, - ClaimAddress: senderAddr, - }, - }, - SiafundOutputs: []types.SiafundOutput{ - { - Address: receiverAddr, - Value: sfe[0].SiafundOutput.Value, - }, - }, - } - cs, err = c.ConsensusTipState() + unconfirmed, err := wc.UnconfirmedEvents() if err != nil { t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) } - txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ - senderPrivateKey.SignHash(cs.InputSigHash(txn)), - } - - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { - t.Fatal(err) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } + cn.MineBlocks(t, types.VoidAddress, 1) - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) - - // check if the element is spent - spent, err = c.SpentSiafundElement(sfe[0].ID) + confirmed, err := wc.Events(0, 5) if err != nil { t.Fatal(err) - } else if !spent.Spent { - t.Fatal("expected siafund element to be spent") - } else if types.TransactionID(spent.Event.ID) != txn.ID() { - t.Fatalf("expected siafund element to have event %q, got %q", txn.ID(), spent.Event.ID) - } else if spent.Event.Type != wallet.EventTypeV2Transaction { - t.Fatalf("expected siafund element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction } - // mine until the utxo is pruned - testutil.MineBlocks(t, cm, types.VoidAddress, 144) - waitForBlock(t, cm, ws) - - _, err = c.SpentSiafundElement(sfe[0].ID) - if !strings.Contains(err.Error(), "not found") { - t.Fatalf("expected error to contain %q, got %q", "not found", err) + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } } -func TestConstructV2Siafunds(t *testing.T) { +func TestSpentElement(t *testing.T) { log := zaptest.NewLogger(t) n, genesisBlock := testutil.V2Network() @@ -2040,238 +1057,191 @@ func TestConstructV2Siafunds(t *testing.T) { } genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - peerStore, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - // create the syncer - s := syncer.New(syncerListener, cm, peerStore, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) - c := runServer(t, cm, s, wm) + // trigger initial scan + cn.MineBlocks(t, types.VoidAddress, 1) - w, err := c.AddWallet(api.WalletUpdateRequest{ - Name: "primary", - }) + sce, basis, err := c.AddressSiacoinOutputs(senderAddr, 0, 100) if err != nil { t.Fatal(err) + } else if len(sce) != 1 { + t.Fatalf("expected 1 siacoin element, got %v", len(sce)) } - wc := c.Wallet(w.ID) - err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - SpendPolicy: &senderPolicy, - }) + // check if the element is spent + spent, err := c.SpentSiacoinElement(sce[0].ID) if err != nil { t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siacoin element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siacoin element to have no event, got %v", spent.Event) } - if err := c.Rescan(0); err != nil { - t.Fatal(err) - } - - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) - - resp, err := wc.ConstructV2(nil, []types.SiafundOutput{ - {Value: 1, Address: receiverAddr}, - }, senderAddr) - if err != nil { - t.Fatal(err) + // spend the element + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Value: sce[0].SiacoinOutput.Value, + Address: receiverAddr, + }, + }, } - cs, err := c.ConsensusTipState() if err != nil { t.Fatal(err) } - - // sign the transaction - sigHash := cs.InputSigHash(resp.Transaction) - sig := senderPrivateKey.SignHash(sigHash) - for i := range resp.Transaction.SiafundInputs { - resp.Transaction.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} - } - for i := range resp.Transaction.SiafundInputs { - resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), } - if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } + cn.MineBlocks(t, types.VoidAddress, 1) - unconfirmed, err := wc.UnconfirmedEvents() + // check if the element is spent + spent, err = c.SpentSiacoinElement(sce[0].ID) if err != nil { t.Fatal(err) - } else if len(unconfirmed) != 1 { - t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) - } - sent := unconfirmed[0] - switch { - case types.TransactionID(sent.ID) != resp.ID: - t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) - case sent.Type != wallet.EventTypeV2Transaction: - t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) - case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): - t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) - case sent.SiafundOutflow()-sent.SiafundInflow() != 1: - t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } else if !spent.Spent { + t.Fatal("expected siacoin element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siacoin element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siacoin element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) - waitForBlock(t, cm, ws) + // mine until the utxo is pruned + cn.MineBlocks(t, types.VoidAddress, 144) - confirmed, err := wc.Events(0, 5) - if err != nil { - t.Fatal(err) - } else if len(confirmed) != 2 { - t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + _, err = c.SpentSiacoinElement(sce[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) } - sent = confirmed[0] - switch { - case types.TransactionID(sent.ID) != resp.ID: - t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) - case sent.Type != wallet.EventTypeV2Transaction: - t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) - case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): - t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) - case sent.SiafundOutflow()-sent.SiafundInflow() != 1: - t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + sfe, basis, err := c.AddressSiafundOutputs(senderAddr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfe) != 1 { + t.Fatalf("expected 1 siafund element, got %v", len(sfe)) } -} -func TestDebugMine(t *testing.T) { - log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + // check if the siafund element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) if err != nil { t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siafund element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siafund element to have no event, got %v", spent.Event) } - cm := chain.NewManager(dbstore, tipState) - l, err := net.Listen("tcp", ":0") + // spend the element + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfe[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + ClaimAddress: senderAddr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + { + Address: receiverAddr, + Value: sfe[0].SiafundOutput.Value, + }, + }, + } + cs, err = c.ConsensusTipState() if err != nil { t.Fatal(err) } - defer l.Close() + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), + } - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - defer ws.Close() + cn.MineBlocks(t, types.VoidAddress, 1) - ps, err := sqlite.NewPeerStore(ws) + // check if the element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) if err != nil { t.Fatal(err) + } else if !spent.Spent { + t.Fatal("expected siafund element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siafund element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siafund element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) } - s := syncer.New(l, cm, ps, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l.Addr().String(), - }) - defer s.Close() - go s.Run() + // mine until the utxo is pruned + cn.MineBlocks(t, types.VoidAddress, 144) - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) + _, err = c.SpentSiafundElement(sfe[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) } - defer wm.Close() +} + +func TestDebugMine(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V1Network() - c := runServer(t, cm, s, wm) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) jc := jape.Client{ BaseURL: c.BaseURL(), Password: "password", } - err = jc.POST("/debug/mine", api.DebugMineRequest{ + err := jc.POST("/debug/mine", api.DebugMineRequest{ Blocks: 5, Address: types.VoidAddress, }, nil) if err != nil { t.Fatal(err) } + cn.WaitForSync(t) - if cm.Tip().Height != 5 { - t.Fatalf("expected tip height to be 5, got %v", cm.Tip().Height) + tip, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } else if tip.Height != 5 { + t.Fatalf("expected tip height to be 5, got %v", tip.Height) } } func TestAPISecurity(t *testing.T) { - n, genesisBlock := testutil.Network() + n, genesisBlock := testutil.V1Network() log := zaptest.NewLogger(t) - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - ps, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - s := syncer.New(syncerListener, cm, ps, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerListener.Addr().String(), - }) - defer s.Close() - go s.Run() - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + wm, err := wallet.NewManager(cn.Chain, cn.Store, wallet.WithLogger(log.Named("wallet"))) if err != nil { t.Fatal(err) } defer wm.Close() + km := keys.NewManager(cn.Store) + defer km.Close() + httpListener, err := net.Listen("tcp", ":0") if err != nil { t.Fatal("failed to listen:", err) @@ -2279,14 +1249,17 @@ func TestAPISecurity(t *testing.T) { t.Cleanup(func() { httpListener.Close() }) server := &http.Server{ - Handler: api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, km, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } t.Cleanup(func() { server.Close() }) - go server.Serve(httpListener) + replaceHandler := func(apiOpts ...api.ServerOption) { + server.Handler = api.NewServer(cn.Chain, cn.Syncer, wm, km, apiOpts...) + } + // create a client with correct credentials c := api.NewClient("http://"+httpListener.Addr().String(), "test") if _, err := c.ConsensusTip(); err != nil { @@ -2302,7 +1275,7 @@ func TestAPISecurity(t *testing.T) { } // replace the handler with a new one that doesn't require auth - server.Handler = api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t))) + replaceHandler() // create a client without credentials c = api.NewClient("http://"+httpListener.Addr().String(), "") @@ -2317,7 +1290,7 @@ func TestAPISecurity(t *testing.T) { } // replace the handler with one that requires auth and has public endpoints - server.Handler = api.NewServer(cm, s, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test"), api.WithPublicEndpoints(true)) + replaceHandler(api.WithBasicAuth("test"), api.WithPublicEndpoints(true)) // create a client without credentials c = api.NewClient("http://"+httpListener.Addr().String(), "") @@ -2327,6 +1300,13 @@ func TestAPISecurity(t *testing.T) { t.Fatal(err) } + // check that the signing endpoint returns 404 when public mode is enabled + if _, err := c.SignHash(frand.Entropy256(), frand.Entropy256()); err == nil { + t.Fatal("expected 404 error") + } else if !strings.Contains(err.Error(), "404") { + t.Fatal("expected 404 error, got", err) + } + // check that a private endpoint is still protected if _, err := c.Wallets(); err == nil { t.Fatal("expected auth error") @@ -2347,47 +1327,10 @@ func TestAPISecurity(t *testing.T) { func TestAPINoContent(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testNetwork() - - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - l, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer l.Close() - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - ps, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - s := syncer.New(l, cm, ps, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l.Addr().String(), - }) - defer s.Close() - go s.Run() - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + n, genesisBlock := testutil.V1Network() - c := runServer(t, cm, s, wm) + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) buf, err := json.Marshal(api.TxpoolBroadcastRequest{ Transactions: []types.Transaction{}, @@ -2416,50 +1359,8 @@ func TestV2TransactionUpdateBasis(t *testing.T) { log := zaptest.NewLogger(t) n, genesisBlock := testutil.V2Network() - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(dbstore, tipState) - - l, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer l.Close() - - ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer ws.Close() - - ps, err := sqlite.NewPeerStore(ws) - if err != nil { - t.Fatal(err) - } - - s := syncer.New(l, cm, ps, gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l.Addr().String(), - }) - defer s.Close() - go s.Run() - - wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() - - c := runServer(t, cm, s, wm) - - mineAndSync := func(addr types.Address, n int) { - testutil.MineBlocks(t, cm, addr, n) - waitForBlock(t, cm, ws) - } + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) // create a wallet w, err := c.AddWallet(api.WalletUpdateRequest{ @@ -2484,8 +1385,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } // fund the wallet - mineAndSync(addr, 5) - mineAndSync(types.VoidAddress, int(n.MaturityDelay)) + cn.MineBlocks(t, addr, 5+int(n.MaturityDelay)) resp, err := wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(100), Address: addr}, @@ -2510,8 +1410,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { t.Fatal(err) } - - mineAndSync(addr, 1) + cn.MineBlocks(t, types.VoidAddress, 1) // create a child transaction sce := parentTxn.EphemeralSiacoinOutput(0) @@ -2533,20 +1432,104 @@ func TestV2TransactionUpdateBasis(t *testing.T) { txnset := []types.V2Transaction{parentTxn, childTxn} - basis, txnset, err = c.V2UpdateTransactionSetBasis(txnset, basis, cm.Tip()) + tip, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } + + basis, txnset, err = c.V2UpdateTransactionSetBasis(txnset, basis, tip) if err != nil { t.Fatal(err) } else if len(txnset) != 1 { t.Fatalf("expected 1 transactions, got %v", len(txnset)) } else if txnset[0].ID() != childTxn.ID() { t.Fatalf("expected parent transaction to be removed") - } else if basis != cm.Tip() { - t.Fatalf("expected basis to be %v, got %v", cm.Tip(), basis) + } else if basis != tip { + t.Fatalf("expected basis to be %v, got %v", tip, basis) } if err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { t.Fatal(err) } + cn.MineBlocks(t, types.VoidAddress, 1) +} + +func TestSigning(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V2Network() + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + pk, err := c.GenerateSigningKey() + if err != nil { + t.Fatal(err) + } + + // create a wallet + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + + policy := types.SpendPolicy{Type: types.PolicyTypePublicKey(pk)} + addr := policy.Address() + + err = wc.AddAddress(wallet.Address{ + Address: addr, + SpendPolicy: &policy, + }) + if err != nil { + t.Fatal(err) + } + + // fund the wallet + cn.MineBlocks(t, addr, 1) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(100), Address: addr}, + }, nil, addr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } - mineAndSync(addr, 1) + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i, si := range resp.Transaction.SiacoinInputs { + pk := types.PublicKey(si.SatisfiedPolicy.Policy.Type.(types.PolicyTypePublicKey)) + + sig, err := c.SignHash(pk, sigHash) + if err != nil { + t.Fatal(err) + } else if !pk.VerifyHash(sigHash, sig) { + t.Fatal("signature verification failed") + } + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + events, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected event type %q, got %q", wallet.EventTypeV2Transaction, events[0].Type) + } else if types.TransactionID(events[0].ID) != resp.ID { + t.Fatalf("expected event ID %q, got %q", resp.ID, events[0].ID) + } } diff --git a/api/client.go b/api/client.go index 236b772..bf8e7f7 100644 --- a/api/client.go +++ b/api/client.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "net/url" "sync" "time" @@ -271,6 +272,36 @@ func (c *Client) SpentSiafundElement(id types.SiafundOutputID) (resp ElementSpen return } +// GenerateSigningKey generates a new ed25519 private key +// on the server and adds it to the key store. Returns the +// public key. +func (c *Client) GenerateSigningKey() (types.PublicKey, error) { + var resp AddSigningKeyResponse + err := c.c.POST("/keys/generate/ed25519", nil, &resp) + return resp.PublicKey, err +} + +// ImportSigningKey imports an ed25519 signing key into the key store. +// Returns the public key. +func (c *Client) ImportSigningKey(sk types.PrivateKey) (types.PublicKey, error) { + var resp AddSigningKeyResponse + err := c.c.POST("/keys/ed25519", AddSigningKeyRequest{PrivateKey: sk}, &resp) + return resp.PublicKey, err +} + +// DeleteSigningKey deletes an ed25519 signing key from the key store. +func (c *Client) DeleteSigningKey(pk types.PublicKey) error { + return c.c.DELETE(fmt.Sprintf("/keys/ed25519/%s", pk)) +} + +// SignHash signs a hash with the specified key. If the key is not found, it +// returns 404 and [keys.ErrNotFound]. +func (c *Client) SignHash(key types.PublicKey, hash types.Hash256) (types.Signature, error) { + var resp SignHashResponse + err := c.c.POST(fmt.Sprintf("/keys/ed25519/%s/sign", url.PathEscape(key.String())), SignHashRequest{hash}, &resp) + return resp.Signature, err +} + // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { diff --git a/api/server.go b/api/server.go index 29c61a3..612bd5c 100644 --- a/api/server.go +++ b/api/server.go @@ -19,6 +19,7 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/build" + "go.sia.tech/walletd/keys" "go.sia.tech/walletd/wallet" ) @@ -131,6 +132,13 @@ type ( Reserve([]types.Hash256) error Release([]types.Hash256) } + + // A SigningKeyManager manages ed25519 signing keys. + SigningKeyManager interface { + Add(types.PrivateKey) error + Delete(types.PublicKey) error + Sign(types.PublicKey, types.Hash256) (types.Signature, error) + } ) type server struct { @@ -143,6 +151,7 @@ type server struct { cm ChainManager s Syncer wm WalletManager + km SigningKeyManager scanMu sync.Mutex // for resubscribe scanInProgress bool @@ -1251,6 +1260,61 @@ func (s *server) outputsSiafundHandlerGET(jc jape.Context) { jc.Encode(output) } +func (s *server) keysEd25519GenerateHandlerPOST(jc jape.Context) { + sk := types.GeneratePrivateKey() + if jc.Check("failed to add key", s.km.Add(sk)) != nil { + return + } + jc.Encode(AddSigningKeyResponse{ + PublicKey: sk.PublicKey(), + }) +} + +func (s *server) keysEd25519HandlerPUT(jc jape.Context) { + var req AddSigningKeyRequest + if jc.Decode(&req) != nil { + return + } else if jc.Check("failed to add key", s.km.Add(req.PrivateKey)) != nil { + return + } + + jc.Encode(AddSigningKeyResponse{ + PublicKey: req.PrivateKey.PublicKey(), + }) +} + +func (s *server) keysEd25519HandlerDELETE(jc jape.Context) { + var pk types.PublicKey + if jc.DecodeParam("pub", &pk) != nil { + return + } else if jc.Check("failed to remove key", s.km.Delete(pk)) != nil { + return + } + jc.EmptyResonse() +} + +func (s *server) keysEd25519SignHandlerPOST(jc jape.Context) { + var pub types.PublicKey + if jc.DecodeParam("pub", &pub) != nil { + return + } + var req SignHashRequest + if jc.Decode(&req) != nil { + return + } + + sig, err := s.km.Sign(pub, req.Hash) + if errors.Is(err, keys.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to sign message", err) != nil { + return + } + jc.Encode(SignHashResponse{ + Signature: sig, + }) +} + func (s *server) debugMineHandler(jc jape.Context) { var req DebugMineRequest if jc.Decode(&req) != nil { @@ -1304,7 +1368,7 @@ func (s *server) pprofHandler(jc jape.Context) { } // NewServer returns an HTTP handler that serves the walletd API. -func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { +func NewServer(cm ChainManager, s Syncer, wm WalletManager, km SigningKeyManager, opts ...ServerOption) http.Handler { srv := server{ log: zap.NewNop(), debugEnabled: false, @@ -1314,6 +1378,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption cm: cm, s: s, wm: wm, + km: km, } for _, opt := range opts { opt(&srv) @@ -1412,6 +1477,14 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), } + if !srv.publicEndpoints { + // key management endpoints are disabled on public nodes + handlers["POST /keys/generate/ed25519"] = wrapAuthHandler(srv.keysEd25519GenerateHandlerPOST) + handlers["PUT /keys/ed25519"] = wrapAuthHandler(srv.keysEd25519HandlerPUT) + handlers["DELETE /keys/ed25519/:pub"] = wrapAuthHandler(srv.keysEd25519HandlerDELETE) + handlers["POST /keys/ed25519/:pub/sign"] = wrapAuthHandler(srv.keysEd25519SignHandlerPOST) + } + if srv.debugEnabled { handlers["POST /debug/mine"] = wrapAuthHandler(srv.debugMineHandler) handlers["GET /debug/pprof/:handler"] = wrapAuthHandler(srv.pprofHandler) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index c0ec30c..007f1ad 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -22,6 +22,7 @@ import ( "go.sia.tech/walletd/api" "go.sia.tech/walletd/build" "go.sia.tech/walletd/config" + "go.sia.tech/walletd/keys" "go.sia.tech/walletd/persist/sqlite" "go.sia.tech/walletd/wallet" "go.sia.tech/web/walletd" @@ -201,6 +202,9 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu } defer wm.Close() + km := keys.NewManager(store) + defer km.Close() + apiOpts := []api.ServerOption{ api.WithLogger(log.Named("api")), api.WithPublicEndpoints(cfg.HTTP.PublicEndpoints), @@ -209,7 +213,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu if enableDebug { apiOpts = append(apiOpts, api.WithDebug()) } - api := api.NewServer(cm, s, wm, apiOpts...) + api := api.NewServer(cm, s, wm, km, apiOpts...) web := walletd.Handler() server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 0000000..839c154 --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,102 @@ +package testutil + +import ( + "net" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/persist/sqlite" + "go.uber.org/zap" +) + +type ( + // A ConsensusNode is a test harness for starting a bare-bones consensus node. + ConsensusNode struct { + Store *sqlite.Store + Chain *chain.Manager + Syncer *syncer.Syncer + } +) + +// WaitForSync waits for the store to sync to the current tip of the chain manager. +func (cn *ConsensusNode) WaitForSync(tb testing.TB) { + tb.Helper() + + for i := 0; i < 1000; i++ { + index, err := cn.Store.LastCommittedIndex() + if err != nil { + tb.Fatal(err) + } else if index == cn.Chain.Tip() { + return + } + time.Sleep(10 * time.Millisecond) + } + tb.Fatal("timeout waiting for sync") +} + +// MineBlocks mines n blocks, sending the rewards to addr. +func (cn *ConsensusNode) MineBlocks(tb testing.TB, addr types.Address, n int) { + tb.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(tb, cn.Chain, addr, 1) + cn.WaitForSync(tb) + } +} + +// NewConsensusNode creates a new ConsensusNode. +func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, log *zap.Logger) *ConsensusNode { + l, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { l.Close() }) + + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + if err != nil { + tb.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + store, err := sqlite.OpenDatabase(filepath.Join(tb.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { store.Close() }) + + peerStore, err := sqlite.NewPeerStore(store) + if err != nil { + tb.Fatal(err) + } + + s := syncer.New(l, cm, peerStore, gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l.Addr().String(), + }) + tb.Cleanup(func() { s.Close() }) + go s.Run() + + return &ConsensusNode{ + Store: store, + Chain: cm, + Syncer: s, + } +} + +// V1Network returns a test network and genesis block. +func V1Network() (*consensus.Network, types.Block) { + return testutil.Network() +} + +// V2Network returns a test network and genesis block with early V2 hardforks +func V2Network() (*consensus.Network, types.Block) { + return testutil.V2Network() +} diff --git a/keys/manager.go b/keys/manager.go new file mode 100644 index 0000000..9c3a310 --- /dev/null +++ b/keys/manager.go @@ -0,0 +1,80 @@ +package keys + +import ( + "errors" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/internal/threadgroup" +) + +var ( + // ErrInvalidSize is returned when a key has an invalid size. + ErrInvalidSize = errors.New("invalid key size") + // ErrNotFound is returned when a signing key is not found. + ErrNotFound = errors.New("not found") +) + +type ( + // A Store saves and loads ed25519 signing keys. + Store interface { + GetSigningKey(types.PublicKey) (types.PrivateKey, error) + AddSigningKey(types.PrivateKey) error + DeleteSigningKey(types.PublicKey) error + } + + // A Manager is a key-value store for ed25519 signing keys. + Manager struct { + tg *threadgroup.ThreadGroup + store Store + } +) + +// Add adds a key to the manager. +func (m *Manager) Add(key types.PrivateKey) error { + done, err := m.tg.Add() + if err != nil { + return err + } + defer done() + return m.store.AddSigningKey(key) +} + +// Sign returns the signature for a hash. If the key is not found, it returns +// [ErrNotFound]. +func (m *Manager) Sign(key types.PublicKey, hash types.Hash256) (types.Signature, error) { + done, err := m.tg.Add() + if err != nil { + return types.Signature{}, err + } + defer done() + + sk, err := m.store.GetSigningKey(key) + if err != nil { + return types.Signature{}, err + } + return sk.SignHash(hash), nil +} + +// Delete removes a key from the manager. +func (m *Manager) Delete(key types.PublicKey) error { + done, err := m.tg.Add() + if err != nil { + return err + } + defer done() + return m.store.DeleteSigningKey(key) +} + +// Close closes the manager. +func (m *Manager) Close() error { + m.tg.Stop() + return nil +} + +// NewManager creates a new key manager. +func NewManager(store Store) *Manager { + return &Manager{ + store: store, + tg: threadgroup.New(), + } +} diff --git a/keys/manager_test.go b/keys/manager_test.go new file mode 100644 index 0000000..1b15b8f --- /dev/null +++ b/keys/manager_test.go @@ -0,0 +1,57 @@ +package keys_test + +import ( + "errors" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" + "go.sia.tech/walletd/persist/sqlite" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +func TestKeyManager(t *testing.T) { + store, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zap.NewNop()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + m := keys.NewManager(store) + defer m.Close() + + sk := types.GeneratePrivateKey() + + if err := m.Add(sk); err != nil { + t.Fatal(err) + } + + // try to add it again + if err := m.Add(sk); err != nil { + t.Fatal(err) + } + + hash := types.Hash256(frand.Entropy256()) + + sig, err := m.Sign(sk.PublicKey(), hash) + if err != nil { + t.Fatal(err) + } else if !sk.PublicKey().VerifyHash(hash, sig) { + t.Fatal("signature failed to verify") + } + + // try to sign with an unknown key + _, err = m.Sign(types.GeneratePrivateKey().PublicKey(), hash) + if !errors.Is(err, keys.ErrNotFound) { + t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) + } + + // delete the key + if err := m.Delete(sk.PublicKey()); err != nil { + t.Fatal(err) + } else if _, err := m.Sign(sk.PublicKey(), hash); !errors.Is(err, keys.ErrNotFound) { + t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) + } +} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 9584437..0add31c 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -111,6 +111,11 @@ CREATE TABLE syncer_bans ( ); CREATE INDEX syncer_bans_expiration_index_idx ON syncer_bans (expiration); +CREATE TABLE signing_keys ( + public_key BLOB PRIMARY KEY, + private_key BLOB UNIQUE NOT NULL +); + CREATE TABLE global_settings ( id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row db_version INTEGER NOT NULL, -- used for migrations diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go new file mode 100644 index 0000000..aa1426e --- /dev/null +++ b/persist/sqlite/keys.go @@ -0,0 +1,48 @@ +package sqlite + +import ( + "crypto/ed25519" + "database/sql" + "errors" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" +) + +// AddSigningKey adds a signing key to the store. If the key already exists, it +// is not added again. +func (s *Store) AddSigningKey(sk types.PrivateKey) error { + if len(sk) != ed25519.PrivateKeySize { + return keys.ErrInvalidSize + } + return s.transaction(func(tx *txn) error { + _, err := tx.Exec("INSERT INTO signing_keys (public_key, private_key) VALUES (?, ?) ON CONFLICT (public_key) DO NOTHING", encode(sk.PublicKey()), sk[:]) + return err + }) +} + +// GetSigningKey returns the private key corresponding to the given public key. +// If the key is not found, it returns [keys.ErrNotFound]. +func (s *Store) GetSigningKey(pk types.PublicKey) (sk types.PrivateKey, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT private_key FROM signing_keys WHERE public_key = ?", encode(pk)).Scan(&sk) + if errors.Is(err, sql.ErrNoRows) { + return keys.ErrNotFound + } else if err != nil { + return err + } else if len(sk) != ed25519.PrivateKeySize { + return keys.ErrInvalidSize + } + return nil + }) + return +} + +// DeleteSigningKey deletes the signing key with the given public key. If the key +// does not exist, it returns nil. +func (s *Store) DeleteSigningKey(pk types.PublicKey) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec("DELETE FROM signing_keys WHERE public_key = ?", encode(pk)) + return err + }) +} diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go new file mode 100644 index 0000000..ba1cf9e --- /dev/null +++ b/persist/sqlite/keys_test.go @@ -0,0 +1,44 @@ +package sqlite + +import ( + "errors" + "path/filepath" + "slices" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +func TestSigningKeys(t *testing.T) { + store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + sk := types.GeneratePrivateKey() + + err = store.AddSigningKey(types.PrivateKey(frand.Bytes(8))) + if !errors.Is(err, keys.ErrInvalidSize) { + t.Fatal(err) + } + + _, err = store.GetSigningKey(sk.PublicKey()) + if !errors.Is(err, keys.ErrNotFound) { + t.Fatal(err) + } + + if err = store.AddSigningKey(sk); err != nil { + t.Fatal(err) + } + + sk2, err := store.GetSigningKey(sk.PublicKey()) + if err != nil { + t.Fatal(err) + } else if !slices.Equal(sk, sk2) { + t.Fatal("keys don't match") + } +} diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 1358a7f..cf4d3f4 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,6 +7,14 @@ import ( "go.uber.org/zap" ) +func migrateVersion8(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`CREATE TABLE signing_keys ( + public_key BLOB PRIMARY KEY, + private_key BLOB UNIQUE NOT NULL +);`) + return err +} + // migrateVersion7 adds spent_event_id columns to siacoin_elements and // siafund_elements to track the event that spent the element. func migrateVersion7(tx *txn, _ *zap.Logger) error { @@ -199,4 +207,5 @@ var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion5, migrateVersion6, migrateVersion7, + migrateVersion8, } diff --git a/wallet/wallet.go b/wallet/wallet.go index 1430582..0174db4 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -77,6 +77,20 @@ type ( Metadata json.RawMessage `json:"metadata"` } + // An UnspentSiacoinElement is an unspent siacoin output paired + // with the number of confirmations. + UnspentSiacoinElement struct { + types.SiacoinElement + Confirmations uint64 `json:"confirmations"` + } + + // An UnspentSiafundElement is an unspent siafund output paired + // with the number of confirmations. + UnspentSiafundElement struct { + types.SiafundElement + Confirmations uint64 `json:"confirmations"` + } + // A ChainUpdate is a set of changes to the consensus state. ChainUpdate interface { SiacoinElementDiffs() []consensus.SiacoinElementDiff From f6844f1fe2e3d1e413517c3981ea0e6d1022ba8a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 28 Feb 2025 11:03:26 +0100 Subject: [PATCH 369/630] add endpoint to fetch block by id --- api/api_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ api/client.go | 6 ++++++ api/server.go | 15 +++++++++++++ 3 files changed, 77 insertions(+) diff --git a/api/api_test.go b/api/api_test.go index b23b0b5..79aaf66 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1232,6 +1232,62 @@ func TestP2P(t *testing.T) { } } +func TestConsensus(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testNetwork() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + ws, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "wallets.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer ws.Close() + + wm, err := wallet.NewManager(cm, ws, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + c := runServer(t, cm, nil, wm) + + // mine a block + minedBlock, ok := coreutils.MineBlock(cm, types.Address{}, time.Minute) + if !ok { + t.Fatal(err) + } else if err := cm.AddBlocks([]types.Block{minedBlock}); err != nil { + t.Fatal(err) + } + + // block should be tip now + ci, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } else if ci.ID != minedBlock.ID() { + t.Fatalf("expected consensus tip to be %v, got %v", minedBlock.ID(), ci.ID) + } + + // fetch block + b, err := c.ConsensusBlocksID(minedBlock.ID()) + if err != nil { + t.Fatal(err) + } else if b.ID() != minedBlock.ID() { + t.Fatal("mismatch") + } +} + func TestConsensusUpdates(t *testing.T) { log := zaptest.NewLogger(t) diff --git a/api/client.go b/api/client.go index 236b772..4adb6e2 100644 --- a/api/client.go +++ b/api/client.go @@ -95,6 +95,12 @@ func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { return } +// ConsensusBlocksID returns the block with the given id. +func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err error) { + err = c.c.GET(fmt.Sprintf("/consensus/blocks/%v", bid), &resp) + return +} + // ConsensusIndex returns the consensus index at the specified height. func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { err = c.c.GET(fmt.Sprintf("/consensus/index/%d", height), &resp) diff --git a/api/server.go b/api/server.go index 29c61a3..0cfc01f 100644 --- a/api/server.go +++ b/api/server.go @@ -61,6 +61,7 @@ type ( Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) + Block(id types.BlockID) (types.Block, bool) TipState() consensus.State AddBlocks([]types.Block) error RecommendedFee() types.Currency @@ -172,6 +173,19 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { jc.Encode(s.cm.TipState()) } +func (s *server) consensusBlocksIDHandler(jc jape.Context) { + var bid types.BlockID + if jc.DecodeParam("id", &bid) != nil { + return + } + block, found := s.cm.Block(bid) + if !found { + jc.Error(errors.New("couldn't find block"), http.StatusNotFound) + return + } + jc.Encode(block) +} + func (s *server) consensusIndexHeightHandler(jc jape.Context) { var height uint64 if jc.DecodeParam("height", &height) != nil { @@ -1363,6 +1377,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), "GET /consensus/tipstate": wrapPublicAuthHandler(srv.consensusTipStateHandler), + "GET /consensus/blocks/:id": wrapPublicAuthHandler(srv.consensusBlocksIDHandler), "GET /consensus/updates/:index": wrapPublicAuthHandler(srv.consensusUpdatesIndexHandler), "GET /consensus/index/:height": wrapPublicAuthHandler(srv.consensusIndexHeightHandler), From 4b8161db7ac48c32bbaff4eab9f8b61380bb3e39 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 28 Feb 2025 19:50:30 -0800 Subject: [PATCH 370/630] add encryption to persisted keys --- api/api_test.go | 32 +++++++--- api/server.go | 14 ++++- cmd/walletd/main.go | 18 ++++-- cmd/walletd/node.go | 14 +++-- config/config.go | 7 +++ go.mod | 2 +- keys/manager.go | 120 ++++++++++++++++++++++++++++++++---- keys/manager_test.go | 22 ++++++- persist/sqlite/init.sql | 3 +- persist/sqlite/keys.go | 55 ++++++++++++++--- persist/sqlite/keys_test.go | 14 ++--- 11 files changed, 248 insertions(+), 53 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 48670ff..6e5df00 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -38,11 +38,14 @@ func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logge } tb.Cleanup(func() { wm.Close() }) - km := keys.NewManager(cn.Store) + km, err := keys.NewManager(cn.Store, "foo") + if err != nil { + tb.Fatal("failed to create key manager:", err) + } tb.Cleanup(func() { km.Close() }) server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, km, api.WithDebug(), api.WithLogger(log)), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithKeyManager(km), api.WithDebug(), api.WithLogger(log)), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -1239,25 +1242,28 @@ func TestAPISecurity(t *testing.T) { } defer wm.Close() - km := keys.NewManager(cn.Store) + km, err := keys.NewManager(cn.Store, "foo") + if err != nil { + t.Fatal(err) + } defer km.Close() httpListener, err := net.Listen("tcp", ":0") if err != nil { t.Fatal("failed to listen:", err) } - t.Cleanup(func() { httpListener.Close() }) + defer httpListener.Close() server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, km, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithKeyManager(km), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } - t.Cleanup(func() { server.Close() }) + defer server.Close() go server.Serve(httpListener) replaceHandler := func(apiOpts ...api.ServerOption) { - server.Handler = api.NewServer(cn.Chain, cn.Syncer, wm, km, apiOpts...) + server.Handler = api.NewServer(cn.Chain, cn.Syncer, wm, apiOpts...) } // create a client with correct credentials @@ -1266,6 +1272,11 @@ func TestAPISecurity(t *testing.T) { t.Fatal(err) } + // check that the signing key endpoints are working + if _, err := c.GenerateSigningKey(); err != nil { + t.Fatal(err) + } + // create a client with incorrect credentials c = api.NewClient("http://"+httpListener.Addr().String(), "wrong") if _, err := c.ConsensusTip(); err == nil { @@ -1274,6 +1285,13 @@ func TestAPISecurity(t *testing.T) { t.Fatal("expected auth error, got", err) } + // check that the signing key endpoints are working + if _, err := c.GenerateSigningKey(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + // replace the handler with a new one that doesn't require auth replaceHandler() diff --git a/api/server.go b/api/server.go index 612bd5c..2063e3e 100644 --- a/api/server.go +++ b/api/server.go @@ -40,6 +40,13 @@ func WithDebug() ServerOption { } } +// WithKeyManager sets the key manager used by the server. +func WithKeyManager(ks SigningKeyManager) ServerOption { + return func(s *server) { + s.km = ks + } +} + // WithPublicEndpoints sets whether the server should disable authentication // on endpoints that are safe for use when running walletd as a service. func WithPublicEndpoints(public bool) ServerOption { @@ -1262,6 +1269,7 @@ func (s *server) outputsSiafundHandlerGET(jc jape.Context) { func (s *server) keysEd25519GenerateHandlerPOST(jc jape.Context) { sk := types.GeneratePrivateKey() + defer clear(sk) if jc.Check("failed to add key", s.km.Add(sk)) != nil { return } @@ -1272,6 +1280,7 @@ func (s *server) keysEd25519GenerateHandlerPOST(jc jape.Context) { func (s *server) keysEd25519HandlerPUT(jc jape.Context) { var req AddSigningKeyRequest + defer clear(req.PrivateKey) if jc.Decode(&req) != nil { return } else if jc.Check("failed to add key", s.km.Add(req.PrivateKey)) != nil { @@ -1368,7 +1377,7 @@ func (s *server) pprofHandler(jc jape.Context) { } // NewServer returns an HTTP handler that serves the walletd API. -func NewServer(cm ChainManager, s Syncer, wm WalletManager, km SigningKeyManager, opts ...ServerOption) http.Handler { +func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { srv := server{ log: zap.NewNop(), debugEnabled: false, @@ -1378,7 +1387,6 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, km SigningKeyManager cm: cm, s: s, wm: wm, - km: km, } for _, opt := range opts { opt(&srv) @@ -1477,7 +1485,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, km SigningKeyManager "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), } - if !srv.publicEndpoints { + if srv.km != nil && !srv.publicEndpoints { // key management endpoints are disabled on public nodes handlers["POST /keys/generate/ed25519"] = wrapAuthHandler(srv.keysEd25519GenerateHandlerPOST) handlers["PUT /keys/ed25519"] = wrapAuthHandler(srv.keysEd25519HandlerPUT) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 07da3d2..727ee72 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -22,10 +22,11 @@ import ( ) const ( - apiPasswordEnvVar = "WALLETD_API_PASSWORD" - configFileEnvVar = "WALLETD_CONFIG_FILE" - dataDirEnvVar = "WALLETD_DATA_DIR" - logFileEnvVar = "WALLETD_LOG_FILE_PATH" + apiPasswordEnvVar = "WALLETD_API_PASSWORD" + configFileEnvVar = "WALLETD_CONFIG_FILE" + dataDirEnvVar = "WALLETD_DATA_DIR" + logFileEnvVar = "WALLETD_LOG_FILE_PATH" + keystoreSecretEnvVar = "WALLETD_KEYSTORE_SECRET" ) const ( @@ -76,6 +77,10 @@ var cfg = config.Config{ Mode: wallet.IndexModePersonal, BatchSize: 1000, }, + KeyStore: config.KeyStore{ + Enabled: false, + Secret: os.Getenv(keystoreSecretEnvVar), + }, Log: config.Log{ Level: "info", File: config.LogFile{ @@ -211,6 +216,7 @@ func main() { rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") rootCmd.BoolVar(&cfg.HTTP.PublicEndpoints, "http.public", cfg.HTTP.PublicEndpoints, "disables auth on endpoints that should be publicly accessible when running walletd as a service") + rootCmd.BoolVar(&cfg.KeyStore.Enabled, "keystore", cfg.KeyStore.Enabled, "enables the keystore") rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") @@ -256,6 +262,10 @@ func main() { checkFatalError("failed to parse index mode", cfg.Index.Mode.UnmarshalText([]byte(indexModeStr))) + if cfg.KeyStore.Enabled && cfg.KeyStore.Secret == "" { + checkFatalError("keystore is enabled but no secret was provided", errors.New("missing keystore secret")) + } + var logCores []zapcore.Core if cfg.Log.StdOut.Enabled { // if no log level is set for stdout, use the global log level diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 007f1ad..38ca50b 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -202,9 +202,6 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu } defer wm.Close() - km := keys.NewManager(store) - defer km.Close() - apiOpts := []api.ServerOption{ api.WithLogger(log.Named("api")), api.WithPublicEndpoints(cfg.HTTP.PublicEndpoints), @@ -213,7 +210,16 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu if enableDebug { apiOpts = append(apiOpts, api.WithDebug()) } - api := api.NewServer(cm, s, wm, km, apiOpts...) + if cfg.KeyStore.Enabled { + km, err := keys.NewManager(store, cfg.KeyStore.Secret) + if err != nil { + return fmt.Errorf("failed to create key manager: %w", err) + } + defer km.Close() + + apiOpts = append(apiOpts, api.WithKeyManager(km)) + } + api := api.NewServer(cm, s, wm, apiOpts...) web := walletd.Handler() server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/config/config.go b/config/config.go index 90ba010..e3d5a75 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,12 @@ type ( Peers []string `yaml:"peers,omitempty"` } + // KeyStore contains the configuration for the key store. + KeyStore struct { + Enabled bool `yaml:"enabled,omitempty"` + Secret string `yaml:"secret,omitempty"` + } + // Consensus contains the configuration for the consensus set. Consensus struct { Network string `yaml:"network,omitempty"` @@ -71,6 +77,7 @@ type ( Syncer Syncer `yaml:"syncer,omitempty"` Log Log `yaml:"log,omitempty"` Index Index `yaml:"index,omitempty"` + KeyStore KeyStore `yaml:"keystore,omitempty"` } ) diff --git a/go.mod b/go.mod index f63b11c..20eb220 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.0 go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.33.0 golang.org/x/term v0.29.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 @@ -31,7 +32,6 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.33.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect diff --git a/keys/manager.go b/keys/manager.go index 9c3a310..fb36f6d 100644 --- a/keys/manager.go +++ b/keys/manager.go @@ -1,10 +1,16 @@ package keys import ( + "crypto/cipher" + "crypto/ed25519" "errors" + "fmt" "go.sia.tech/core/types" "go.sia.tech/walletd/internal/threadgroup" + "golang.org/x/crypto/argon2" + "golang.org/x/crypto/chacha20poly1305" + "lukechampine.com/frand" ) var ( @@ -12,35 +18,73 @@ var ( ErrInvalidSize = errors.New("invalid key size") // ErrNotFound is returned when a signing key is not found. ErrNotFound = errors.New("not found") + + // ErrKeySaltSet is returned when the key salt is already set. + ErrKeySaltSet = errors.New("key salt already set") + + // ErrIncorrectSecret is returned when the secret is incorrect. + ErrIncorrectSecret = errors.New("incorrect secret") ) type ( // A Store saves and loads ed25519 signing keys. Store interface { - GetSigningKey(types.PublicKey) (types.PrivateKey, error) - AddSigningKey(types.PrivateKey) error + // GetSigningKey returns the encrypted signing key with the given public key. + // If the key is not found, it returns [ErrNotFound]. The key must be + // decrypted before being used. + GetSigningKey(types.PublicKey) ([]byte, error) + // AddSigningKey adds a signing key to the store. If the key already + // exists, nil is returned. The key must be encrypted before being + // stored. + AddSigningKey(pk types.PublicKey, buf []byte) error + // DeleteSigningKey deletes the signing key with the given public key. + // If the key does not exist, it returns [ErrNotFound]. DeleteSigningKey(types.PublicKey) error + + // KeySalt returns the salt used to derive the key encryption + // key. If no salt has been set, KeySalt returns [keys.ErrNotFound]. + GetKeySalt() ([]byte, error) + + // SetKeySalt sets the salt used to derive the key encryption key. + // If a salt has already been set, [keys.ErrKeySaltSet] is returned. + SetKeySalt([]byte) error + + // GetBytesForVerify returns random encrypted bytes for verifying + // the encryption key. + GetBytesForVerify() ([]byte, error) } // A Manager is a key-value store for ed25519 signing keys. Manager struct { - tg *threadgroup.ThreadGroup + tg *threadgroup.ThreadGroup + + aead cipher.AEAD store Store } ) -// Add adds a key to the manager. -func (m *Manager) Add(key types.PrivateKey) error { +// Add adds a key to the manager. If the key is not the correct +// size, it returns [ErrInvalidSize]. +func (m *Manager) Add(sk types.PrivateKey) error { + if len(sk) != ed25519.PrivateKeySize { + return ErrInvalidSize + } + done, err := m.tg.Add() if err != nil { return err } defer done() - return m.store.AddSigningKey(key) + + n := m.aead.NonceSize() + buf := make([]byte, m.aead.NonceSize(), n+len(sk)+m.aead.Overhead()) + frand.Read(buf) + encrypted := m.aead.Seal(buf, buf, sk, nil) + return m.store.AddSigningKey(sk.PublicKey(), encrypted) } -// Sign returns the signature for a hash. If the key is not found, it returns -// [ErrNotFound]. +// Sign returns the signature for a hash. If the key is not +// found, it returns [ErrNotFound]. func (m *Manager) Sign(key types.PublicKey, hash types.Hash256) (types.Signature, error) { done, err := m.tg.Add() if err != nil { @@ -48,11 +92,23 @@ func (m *Manager) Sign(key types.PublicKey, hash types.Hash256) (types.Signature } defer done() - sk, err := m.store.GetSigningKey(key) + buf, err := m.store.GetSigningKey(key) if err != nil { return types.Signature{}, err } - return sk.SignHash(hash), nil + defer clear(buf) + + sk := make(types.PrivateKey, 0, ed25519.PrivateKeySize) + defer clear(sk) + sk, err = m.aead.Open(sk, buf[:m.aead.NonceSize()], buf[m.aead.NonceSize():], nil) + if err != nil { + return types.Signature{}, fmt.Errorf("failed to decrypt key: %w", err) + } + + if len(sk) != ed25519.PrivateKeySize { + return types.Signature{}, ErrInvalidSize + } + return types.PrivateKey(sk).SignHash(hash), nil } // Delete removes a key from the manager. @@ -71,10 +127,48 @@ func (m *Manager) Close() error { return nil } -// NewManager creates a new key manager. -func NewManager(store Store) *Manager { +// NewManager creates a new key manager. If the store contains +// encrypted keys, the secret must match the secret used to encrypt +// the existing keys. If the secret is incorrect, NewManager returns +// [ErrIncorrectSecret]. +// +// Keys are encrypted using ChaCha20-Poly1305 with a key derived from +// the secret using Argon2ID. +func NewManager(store Store, secret string) (*Manager, error) { + salt, err := store.GetKeySalt() + if errors.Is(err, ErrNotFound) { + salt = frand.Bytes(32) + if err := store.SetKeySalt(salt); err != nil { + return nil, fmt.Errorf("failed to set key salt: %w", err) + } + } else if err != nil { + return nil, fmt.Errorf("failed to get key salt: %w", err) + } + encryptionKey := argon2.IDKey([]byte(secret), salt, 3, 64*1024, 4, 32) + aead, err := chacha20poly1305.NewX(encryptionKey) + if err != nil { + return nil, fmt.Errorf("failed to create AEAD: %w", err) + } + + buf, err := store.GetBytesForVerify() + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, fmt.Errorf("failed to get bytes for verify: %w", err) + } else if err == nil { + defer clear(buf) + + decrypted, err := aead.Open(nil, buf[:aead.NonceSize()], buf[aead.NonceSize():], nil) + if err != nil { + if err.Error() == "message authentication failed" { + return nil, ErrIncorrectSecret + } + return nil, fmt.Errorf("failed to verify encryption key: %w", err) + } + defer clear(decrypted) + } + return &Manager{ + aead: aead, store: store, tg: threadgroup.New(), - } + }, nil } diff --git a/keys/manager_test.go b/keys/manager_test.go index 1b15b8f..f6d01dd 100644 --- a/keys/manager_test.go +++ b/keys/manager_test.go @@ -19,7 +19,10 @@ func TestKeyManager(t *testing.T) { } defer store.Close() - m := keys.NewManager(store) + m, err := keys.NewManager(store, "foo") + if err != nil { + t.Fatal(err) + } defer m.Close() sk := types.GeneratePrivateKey() @@ -48,6 +51,23 @@ func TestKeyManager(t *testing.T) { t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) } + if err := m.Close(); err != nil { + t.Fatal(err) + } + + m, err = keys.NewManager(store, "foobar") + if err != nil { + t.Fatal(err) + } + defer m.Close() + + sig, err = m.Sign(sk.PublicKey(), hash) + if err != nil { + t.Fatal(err) + } else if !sk.PublicKey().VerifyHash(hash, sig) { + t.Fatal("signature failed to verify") + } + // delete the key if err := m.Delete(sk.PublicKey()); err != nil { t.Fatal(err) diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index 0add31c..d3afb4c 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -122,5 +122,6 @@ CREATE TABLE global_settings ( index_mode INTEGER, -- the mode of the data store last_indexed_height INTEGER NOT NULL, -- the height of the last chain index that was processed last_indexed_id BLOB NOT NULL, -- the block ID of the last chain index that was processed - element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree + element_num_leaves INTEGER NOT NULL, -- the number of leaves in the state tree + key_salt BLOB -- the salt used for deriving keys ); diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go index aa1426e..d609558 100644 --- a/persist/sqlite/keys.go +++ b/persist/sqlite/keys.go @@ -1,7 +1,6 @@ package sqlite import ( - "crypto/ed25519" "database/sql" "errors" @@ -11,27 +10,22 @@ import ( // AddSigningKey adds a signing key to the store. If the key already exists, it // is not added again. -func (s *Store) AddSigningKey(sk types.PrivateKey) error { - if len(sk) != ed25519.PrivateKeySize { - return keys.ErrInvalidSize - } +func (s *Store) AddSigningKey(pk types.PublicKey, buf []byte) error { return s.transaction(func(tx *txn) error { - _, err := tx.Exec("INSERT INTO signing_keys (public_key, private_key) VALUES (?, ?) ON CONFLICT (public_key) DO NOTHING", encode(sk.PublicKey()), sk[:]) + _, err := tx.Exec("INSERT INTO signing_keys (public_key, private_key) VALUES (?, ?) ON CONFLICT (public_key) DO NOTHING", encode(pk), buf) return err }) } // GetSigningKey returns the private key corresponding to the given public key. // If the key is not found, it returns [keys.ErrNotFound]. -func (s *Store) GetSigningKey(pk types.PublicKey) (sk types.PrivateKey, err error) { +func (s *Store) GetSigningKey(pk types.PublicKey) (buf []byte, err error) { err = s.transaction(func(tx *txn) error { - err := s.db.QueryRow("SELECT private_key FROM signing_keys WHERE public_key = ?", encode(pk)).Scan(&sk) + err := s.db.QueryRow("SELECT private_key FROM signing_keys WHERE public_key = ?", encode(pk)).Scan(&buf) if errors.Is(err, sql.ErrNoRows) { return keys.ErrNotFound } else if err != nil { return err - } else if len(sk) != ed25519.PrivateKeySize { - return keys.ErrInvalidSize } return nil }) @@ -46,3 +40,44 @@ func (s *Store) DeleteSigningKey(pk types.PublicKey) error { return err }) } + +// KeySalt returns the salt used to derive the key encryption +// key. If no salt has been set, KeySalt returns [keys.ErrNotFound]. +func (s *Store) GetKeySalt() (salt []byte, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT key_salt FROM global_settings").Scan(&salt) + if errors.Is(err, sql.ErrNoRows) { + return keys.ErrNotFound + } + return err + }) + return +} + +// SetKeySalt sets the salt used to derive the key encryption key. +// If a salt has already been set, [keys.ErrKeySaltSet] is returned. +func (s *Store) SetKeySalt(salt []byte) error { + return s.transaction(func(tx *txn) error { + res, err := tx.Exec("UPDATE global_settings SET key_salt = ? WHERE key_salt IS NULL", salt) + if err != nil { + return err + } else if n, _ := res.RowsAffected(); n == 0 { + return errors.New("key salt already set") + } + return nil + }) +} + +// GetBytesForVerify returns random encrypted bytes for verifying +// the encryption key. If there are no keys in the store, it returns +// [keys.ErrNotFound]. +func (s *Store) GetBytesForVerify() (buf []byte, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT private_key FROM signing_keys LIMIT 1").Scan(&buf) + if errors.Is(err, sql.ErrNoRows) { + return keys.ErrNotFound + } + return err + }) + return +} diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go index ba1cf9e..9bdf7c4 100644 --- a/persist/sqlite/keys_test.go +++ b/persist/sqlite/keys_test.go @@ -1,9 +1,9 @@ package sqlite import ( + "bytes" "errors" "path/filepath" - "slices" "testing" "go.sia.tech/core/types" @@ -21,24 +21,20 @@ func TestSigningKeys(t *testing.T) { sk := types.GeneratePrivateKey() - err = store.AddSigningKey(types.PrivateKey(frand.Bytes(8))) - if !errors.Is(err, keys.ErrInvalidSize) { - t.Fatal(err) - } - _, err = store.GetSigningKey(sk.PublicKey()) if !errors.Is(err, keys.ErrNotFound) { t.Fatal(err) } - if err = store.AddSigningKey(sk); err != nil { + expected := frand.Bytes(64) // mock encrypted key + if err = store.AddSigningKey(sk.PublicKey(), expected); err != nil { t.Fatal(err) } - sk2, err := store.GetSigningKey(sk.PublicKey()) + buf, err := store.GetSigningKey(sk.PublicKey()) if err != nil { t.Fatal(err) - } else if !slices.Equal(sk, sk2) { + } else if !bytes.Equal(expected, buf) { t.Fatal("keys don't match") } } From 71f6bd24f344c24b35cbf3fb107efe53b080e444 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 28 Feb 2025 19:52:08 -0800 Subject: [PATCH 371/630] fix lint --- persist/sqlite/keys.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go index d609558..375c994 100644 --- a/persist/sqlite/keys.go +++ b/persist/sqlite/keys.go @@ -41,8 +41,8 @@ func (s *Store) DeleteSigningKey(pk types.PublicKey) error { }) } -// KeySalt returns the salt used to derive the key encryption -// key. If no salt has been set, KeySalt returns [keys.ErrNotFound]. +// GetKeySalt returns the salt used to derive the key encryption +// key. If no salt has been set, it returns [keys.ErrNotFound]. func (s *Store) GetKeySalt() (salt []byte, err error) { err = s.transaction(func(tx *txn) error { err := s.db.QueryRow("SELECT key_salt FROM global_settings").Scan(&salt) From e96d502e69586ed4de3d3ffd0e7c7a1beb2fa64c Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 28 Feb 2025 20:01:58 -0800 Subject: [PATCH 372/630] docs: update readme --- .changeset/add_signing_key_store.md | 3 ++- README.md | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.changeset/add_signing_key_store.md b/.changeset/add_signing_key_store.md index 9436d6f..217495d 100644 --- a/.changeset/add_signing_key_store.md +++ b/.changeset/add_signing_key_store.md @@ -4,8 +4,9 @@ default: minor # Add ed25519 key store -Adds an optional ed25519 signing key store for integrators to store arbitrary private keys for signing transactions. It allows for both generating private keys on the server and importing private keys. +Adds an optional ed25519 key store for integrators to store arbitrary private keys for signing transactions. It allows for both generating private keys on the server and importing private keys. Keys are stored encrypted using a user-provided secret. +The store is disabled by default. It can be enabled through the config file or the CLI flag `--keystore`. If the store is enabled, an encryption key must also be provided through the environment variable `WALLETD_KEYSTORE_SECRET`. *The endpoint will return 404 if the `--public` CLI flag is set. It is only recommended for use on localhost. It is not used by the UI.* diff --git a/README.md b/README.md index f990367..8f3052b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ The priority of configuration settings is as follows: + `WALLETD_API_PASSWORD` - The password required to access the API. + `WALLETD_CONFIG_FILE` - The path to the YAML configuration file. Defaults to `walletd.yml` in the working directory. + `WALLETD_LOG_FILE` - The path to the log file. ++ `WALLETD_KEYSTORE_SECRET` - The secret to use for encrypting stored ed25519 signing keys. ### Command Line Flags ``` @@ -97,6 +98,8 @@ Flags: network to connect to (default "mainnet") -upnp attempt to forward ports and discover IP with UPnP + -keystore + enables the optional ed25519 key store. ``` ### YAML @@ -123,6 +126,8 @@ syncer: enableUPnP: false peers: [] address: :9981 +keystore: + enabled: false index: mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) From 8a14b2333fcfd01117667d205ddd702883c8e076 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 28 Feb 2025 21:15:53 -0800 Subject: [PATCH 373/630] fix wallet test --- keys/manager.go | 3 ++- keys/manager_test.go | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/keys/manager.go b/keys/manager.go index fb36f6d..52b2bbf 100644 --- a/keys/manager.go +++ b/keys/manager.go @@ -5,6 +5,7 @@ import ( "crypto/ed25519" "errors" "fmt" + "strings" "go.sia.tech/core/types" "go.sia.tech/walletd/internal/threadgroup" @@ -158,7 +159,7 @@ func NewManager(store Store, secret string) (*Manager, error) { decrypted, err := aead.Open(nil, buf[:aead.NonceSize()], buf[aead.NonceSize():], nil) if err != nil { - if err.Error() == "message authentication failed" { + if strings.Contains(err.Error(), "message authentication failed") { return nil, ErrIncorrectSecret } return nil, fmt.Errorf("failed to verify encryption key: %w", err) diff --git a/keys/manager_test.go b/keys/manager_test.go index f6d01dd..d049a02 100644 --- a/keys/manager_test.go +++ b/keys/manager_test.go @@ -55,7 +55,12 @@ func TestKeyManager(t *testing.T) { t.Fatal(err) } - m, err = keys.NewManager(store, "foobar") + _, err = keys.NewManager(store, "foobar") + if !errors.Is(err, keys.ErrIncorrectSecret) { + t.Fatalf("expected %v, got %v", keys.ErrIncorrectSecret, err) + } + + m, err = keys.NewManager(store, "foo") if err != nil { t.Fatal(err) } From a34b1c9f4381749489ba4d30fcb0a51b294b5851 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 28 Feb 2025 21:20:57 -0800 Subject: [PATCH 374/630] fix migration --- persist/sqlite/migrations.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index cf4d3f4..77a747e 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -11,7 +11,8 @@ func migrateVersion8(tx *txn, _ *zap.Logger) error { _, err := tx.Exec(`CREATE TABLE signing_keys ( public_key BLOB PRIMARY KEY, private_key BLOB UNIQUE NOT NULL -);`) +); +ALTER TABLE global_settings ADD COLUMN key_salt BLOB;`) return err } From d3e2977c660a35840bd8b952ae0aebd7fc2f5734 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 3 Mar 2025 11:18:23 -0800 Subject: [PATCH 375/630] address review comments --- keys/manager.go | 17 ++++++++--------- persist/sqlite/keys.go | 9 +++------ persist/sqlite/keys_test.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/keys/manager.go b/keys/manager.go index 52b2bbf..62f061a 100644 --- a/keys/manager.go +++ b/keys/manager.go @@ -19,10 +19,8 @@ var ( ErrInvalidSize = errors.New("invalid key size") // ErrNotFound is returned when a signing key is not found. ErrNotFound = errors.New("not found") - - // ErrKeySaltSet is returned when the key salt is already set. - ErrKeySaltSet = errors.New("key salt already set") - + // ErrSaltSet is returned when the key salt is already set. + ErrSaltSet = errors.New("salt already set") // ErrIncorrectSecret is returned when the secret is incorrect. ErrIncorrectSecret = errors.New("incorrect secret") ) @@ -43,11 +41,11 @@ type ( DeleteSigningKey(types.PublicKey) error // KeySalt returns the salt used to derive the key encryption - // key. If no salt has been set, KeySalt returns [keys.ErrNotFound]. + // key. If no salt has been set, KeySalt should return (nil, nil). GetKeySalt() ([]byte, error) // SetKeySalt sets the salt used to derive the key encryption key. - // If a salt has already been set, [keys.ErrKeySaltSet] is returned. + // If a salt has already been set, [keys.ErrSaltSet] is returned. SetKeySalt([]byte) error // GetBytesForVerify returns random encrypted bytes for verifying @@ -137,14 +135,15 @@ func (m *Manager) Close() error { // the secret using Argon2ID. func NewManager(store Store, secret string) (*Manager, error) { salt, err := store.GetKeySalt() - if errors.Is(err, ErrNotFound) { + if err != nil { + return nil, fmt.Errorf("failed to get key salt: %w", err) + } else if len(salt) == 0 { salt = frand.Bytes(32) if err := store.SetKeySalt(salt); err != nil { return nil, fmt.Errorf("failed to set key salt: %w", err) } - } else if err != nil { - return nil, fmt.Errorf("failed to get key salt: %w", err) } + encryptionKey := argon2.IDKey([]byte(secret), salt, 3, 64*1024, 4, 32) aead, err := chacha20poly1305.NewX(encryptionKey) if err != nil { diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go index 375c994..6de47b6 100644 --- a/persist/sqlite/keys.go +++ b/persist/sqlite/keys.go @@ -42,27 +42,24 @@ func (s *Store) DeleteSigningKey(pk types.PublicKey) error { } // GetKeySalt returns the salt used to derive the key encryption -// key. If no salt has been set, it returns [keys.ErrNotFound]. +// key. If no salt has been set, GetKeySalt returns (nil, nil). func (s *Store) GetKeySalt() (salt []byte, err error) { err = s.transaction(func(tx *txn) error { err := s.db.QueryRow("SELECT key_salt FROM global_settings").Scan(&salt) - if errors.Is(err, sql.ErrNoRows) { - return keys.ErrNotFound - } return err }) return } // SetKeySalt sets the salt used to derive the key encryption key. -// If a salt has already been set, [keys.ErrKeySaltSet] is returned. +// If a salt has already been set, [keys.ErrSaltSet] is returned. func (s *Store) SetKeySalt(salt []byte) error { return s.transaction(func(tx *txn) error { res, err := tx.Exec("UPDATE global_settings SET key_salt = ? WHERE key_salt IS NULL", salt) if err != nil { return err } else if n, _ := res.RowsAffected(); n == 0 { - return errors.New("key salt already set") + return keys.ErrSaltSet } return nil }) diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go index 9bdf7c4..bf38570 100644 --- a/persist/sqlite/keys_test.go +++ b/persist/sqlite/keys_test.go @@ -38,3 +38,39 @@ func TestSigningKeys(t *testing.T) { t.Fatal("keys don't match") } } + +func TestSalt(t *testing.T) { + store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + assertSalt := func(t *testing.T, expected []byte) { + t.Helper() + s, err := store.GetKeySalt() + if err != nil { + t.Fatal(err) + } else if expected == nil && s != nil { + t.Fatal("expected nil salt") // bytes.Equal([]byte{}, nil) == true + } else if !bytes.Equal(s, expected) { + t.Fatal("salts don't match") + } + } + + // check salt is initially nil + assertSalt(t, nil) + + expected := frand.Bytes(32) + if err = store.SetKeySalt(expected); err != nil { + t.Fatal(err) + } + assertSalt(t, expected) + + if err = store.SetKeySalt(frand.Bytes(32)); !errors.Is(err, keys.ErrSaltSet) { + t.Fatalf("expected %v, got %v", keys.ErrSaltSet, err) + } + + // check salt was not changed + assertSalt(t, expected) +} From 2b66f24289d16f89dd51a1261c8964496cf379f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:31:05 +0000 Subject: [PATCH 376/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.10.2 to 0.10.3 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.10.2...v0.10.3) Updates `go.sia.tech/coreutils` from 0.11.1 to 0.12.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.11.1...v0.12.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index f63b11c..bb5138f 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.2 - go.sia.tech/coreutils v0.11.1 + go.sia.tech/core v0.10.3 + go.sia.tech/coreutils v0.12.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.0 go.uber.org/zap v1.27.0 @@ -24,14 +24,14 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.49.0 // indirect + github.com/quic-go/quic-go v0.50.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.4.0 // indirect - go.sia.tech/mux v1.3.0 // indirect + go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.33.0 // indirect + golang.org/x/crypto v0.34.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect diff --git a/go.sum b/go.sum index 116007e..ea94fe8 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94= -github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= +github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo= +github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -41,14 +41,14 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.2 h1:flwT7DUFZ/CQKBWJaoy3etvy2EWVtrwXmYITjihkYQ4= -go.sia.tech/core v0.10.2/go.mod h1:7kpJAs7Ju4Ho72Y/rK/Q3lgKXShn8A7KPn+QpM/3SXQ= -go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo= -go.sia.tech/coreutils v0.11.1/go.mod h1:vnY0haOx1InIQR0Pc5YAXDe4WnF6po8dv5bNP73CAnE= +go.sia.tech/core v0.10.3 h1:fNt5Dkqxr+Q9vz3jQcdZqFPNA0LgKD6ZtpxQvkTZxns= +go.sia.tech/core v0.10.3/go.mod h1:JvW51XfqDljOjv7kagSxQJJ00hzYmgRIkA76Krb3p3U= +go.sia.tech/coreutils v0.12.0 h1:DOPFHveZeIxDATv9MW8G3kVEkOMOSfiSiN3jxcQ7Pik= +go.sia.tech/coreutils v0.12.0/go.mod h1:XBMg1imCUr5eYjaEemP9GZXvoOjYNvGTvmR/QHObAJg= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= -go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= -go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58= +go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= +go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.29.0 h1:JHJj6TlQozKGcUqUyL0YXR0I+Poe1kjgcGA5v1/9tjA= @@ -61,8 +61,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.34.0 h1:+/C6tk6rf/+t5DhUketUbD1aNGqiSX3j15Z6xuIDlBA= +golang.org/x/crypto v0.34.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= From 7adf69afa7b820829b541ad36c8a396361773965 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Tue, 11 Mar 2025 10:29:08 +0900 Subject: [PATCH 377/630] ui: v0.29.1 --- .changeset/fix_v2_signing.md | 8 ++++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 .changeset/fix_v2_signing.md diff --git a/.changeset/fix_v2_signing.md b/.changeset/fix_v2_signing.md new file mode 100644 index 0000000..3e3ea46 --- /dev/null +++ b/.changeset/fix_v2_signing.md @@ -0,0 +1,8 @@ +--- +default: minor +--- + +# Fixes sending V2 transactions in the UI + +- Fixes V2 signing for wallets that do not have siafund outputs. Fixes #247 + diff --git a/go.mod b/go.mod index 6edf081..fb25ed9 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.10.3 go.sia.tech/coreutils v0.12.0 go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.29.0 + go.sia.tech/web/walletd v0.29.1 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.34.0 golang.org/x/term v0.29.0 diff --git a/go.sum b/go.sum index ea94fe8..3de409a 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.29.0 h1:JHJj6TlQozKGcUqUyL0YXR0I+Poe1kjgcGA5v1/9tjA= -go.sia.tech/web/walletd v0.29.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.29.1 h1:fWknehIhFuXIeoeTO4n86OL14sx7ynBkJKJJAMR3wFM= +go.sia.tech/web/walletd v0.29.1/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From 580345650dc8248aede777db1718cf87e3aa4ccd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 02:53:01 +0000 Subject: [PATCH 378/630] build(deps): bump the all-dependencies group across 1 directory with 3 updates Bumps the all-dependencies group with 1 update in the / directory: [go.sia.tech/core](https://github.com/SiaFoundation/core). Updates `go.sia.tech/core` from 0.10.3 to 0.10.4 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.10.3...v0.10.4) Updates `golang.org/x/crypto` from 0.34.0 to 0.36.0 - [Commits](https://github.com/golang/crypto/compare/v0.34.0...v0.36.0) Updates `golang.org/x/term` from 0.29.0 to 0.30.0 - [Commits](https://github.com/golang/term/compare/v0.29.0...v0.30.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index fb25ed9..3ec5408 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,13 @@ toolchain go1.23.2 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.3 + go.sia.tech/core v0.10.4 go.sia.tech/coreutils v0.12.0 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.34.0 - golang.org/x/term v0.29.0 + golang.org/x/crypto v0.36.0 + golang.org/x/term v0.30.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -35,8 +35,8 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 3de409a..f6a68cd 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.3 h1:fNt5Dkqxr+Q9vz3jQcdZqFPNA0LgKD6ZtpxQvkTZxns= -go.sia.tech/core v0.10.3/go.mod h1:JvW51XfqDljOjv7kagSxQJJ00hzYmgRIkA76Krb3p3U= +go.sia.tech/core v0.10.4 h1:YdKfEDIqxKFkalCvWk6+HD7APVCZqrLGEEV/i2NKKMI= +go.sia.tech/core v0.10.4/go.mod h1:i/dfvjZRei6kR2tOLl27PexeYFb/jtCzRsplSBn3Fgc= go.sia.tech/coreutils v0.12.0 h1:DOPFHveZeIxDATv9MW8G3kVEkOMOSfiSiN3jxcQ7Pik= go.sia.tech/coreutils v0.12.0/go.mod h1:XBMg1imCUr5eYjaEemP9GZXvoOjYNvGTvmR/QHObAJg= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= @@ -61,22 +61,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.34.0 h1:+/C6tk6rf/+t5DhUketUbD1aNGqiSX3j15Z6xuIDlBA= -golang.org/x/crypto v0.34.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= From 2cd43765ee078421739d8f68119b6382a3ec334b Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Mar 2025 00:05:38 -0700 Subject: [PATCH 379/630] go.sia.tech/walletd -> go.sia.tech/walletd/v2 --- api/api.go | 5 ++++- api/api_test.go | 8 ++++---- api/client.go | 2 +- api/server.go | 6 +++--- cmd/walletd/config.go | 2 +- cmd/walletd/main.go | 8 ++++---- cmd/walletd/miner.go | 2 +- cmd/walletd/node.go | 12 ++++++------ config/config.go | 2 +- go.mod | 2 +- internal/testutil/testutil.go | 2 +- keys/manager.go | 2 +- keys/manager_test.go | 4 ++-- persist/sqlite/addresses.go | 2 +- persist/sqlite/consensus.go | 2 +- persist/sqlite/consensus_test.go | 2 +- persist/sqlite/events.go | 2 +- persist/sqlite/events_test.go | 2 +- persist/sqlite/keys.go | 2 +- persist/sqlite/keys_test.go | 2 +- persist/sqlite/store.go | 2 +- persist/sqlite/utxo.go | 2 +- persist/sqlite/wallet.go | 2 +- wallet/manager.go | 2 +- wallet/wallet.go | 15 +++++++++++++++ wallet/wallet_test.go | 4 ++-- 26 files changed, 58 insertions(+), 40 deletions(-) diff --git a/api/api.go b/api/api.go index d0eef6a..943c6de 100644 --- a/api/api.go +++ b/api/api.go @@ -1,3 +1,6 @@ +// Package api provides a RESTful API client and server for the walletd +// daemon. + package api import ( @@ -6,7 +9,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) // A StateResponse returns information about the current state of the walletd diff --git a/api/api_test.go b/api/api_test.go index 2166493..547a46c 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -15,10 +15,10 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/jape" - "go.sia.tech/walletd/api" - "go.sia.tech/walletd/internal/testutil" - "go.sia.tech/walletd/keys" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/api" + "go.sia.tech/walletd/v2/internal/testutil" + "go.sia.tech/walletd/v2/keys" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" "lukechampine.com/frand" diff --git a/api/client.go b/api/client.go index fb1d48e..064eb64 100644 --- a/api/client.go +++ b/api/client.go @@ -10,7 +10,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/jape" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) // A Client provides methods for interacting with a walletd API server. diff --git a/api/server.go b/api/server.go index 4ef70b3..cb293de 100644 --- a/api/server.go +++ b/api/server.go @@ -18,9 +18,9 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" - "go.sia.tech/walletd/build" - "go.sia.tech/walletd/keys" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/build" + "go.sia.tech/walletd/v2/keys" + "go.sia.tech/walletd/v2/wallet" ) // A ServerOption sets an optional parameter for the server. diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go index 353998f..27bef3e 100644 --- a/cmd/walletd/config.go +++ b/cmd/walletd/config.go @@ -11,7 +11,7 @@ import ( "strconv" "strings" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "golang.org/x/term" "gopkg.in/yaml.v3" ) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 727ee72..e484f4f 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -12,10 +12,10 @@ import ( "go.sia.tech/core/types" cwallet "go.sia.tech/coreutils/wallet" - "go.sia.tech/walletd/api" - "go.sia.tech/walletd/build" - "go.sia.tech/walletd/config" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/api" + "go.sia.tech/walletd/v2/build" + "go.sia.tech/walletd/v2/config" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "go.uber.org/zap/zapcore" "lukechampine.com/flagg" diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index a4e6767..7267602 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -8,7 +8,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils" - "go.sia.tech/walletd/api" + "go.sia.tech/walletd/v2/api" "lukechampine.com/frand" ) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 38ca50b..231c2ad 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -19,12 +19,12 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" - "go.sia.tech/walletd/api" - "go.sia.tech/walletd/build" - "go.sia.tech/walletd/config" - "go.sia.tech/walletd/keys" - "go.sia.tech/walletd/persist/sqlite" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/api" + "go.sia.tech/walletd/v2/build" + "go.sia.tech/walletd/v2/config" + "go.sia.tech/walletd/v2/keys" + "go.sia.tech/walletd/v2/persist/sqlite" + "go.sia.tech/walletd/v2/wallet" "go.sia.tech/web/walletd" "go.uber.org/zap" "lukechampine.com/upnp" diff --git a/config/config.go b/config/config.go index e3d5a75..2e95998 100644 --- a/config/config.go +++ b/config/config.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "gopkg.in/yaml.v3" ) diff --git a/go.mod b/go.mod index 3ec5408..96e344c 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd // v2.0.0 +module go.sia.tech/walletd/v2 // v2.0.0 go 1.23.1 diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 839c154..0b50e10 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -12,7 +12,7 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/coreutils/testutil" - "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/v2/persist/sqlite" "go.uber.org/zap" ) diff --git a/keys/manager.go b/keys/manager.go index 62f061a..d7eb403 100644 --- a/keys/manager.go +++ b/keys/manager.go @@ -8,7 +8,7 @@ import ( "strings" "go.sia.tech/core/types" - "go.sia.tech/walletd/internal/threadgroup" + "go.sia.tech/walletd/v2/internal/threadgroup" "golang.org/x/crypto/argon2" "golang.org/x/crypto/chacha20poly1305" "lukechampine.com/frand" diff --git a/keys/manager_test.go b/keys/manager_test.go index d049a02..7fdc2c3 100644 --- a/keys/manager_test.go +++ b/keys/manager_test.go @@ -6,8 +6,8 @@ import ( "testing" "go.sia.tech/core/types" - "go.sia.tech/walletd/keys" - "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/v2/keys" + "go.sia.tech/walletd/v2/persist/sqlite" "go.uber.org/zap" "lukechampine.com/frand" ) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index b5c9056..4726d09 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -7,7 +7,7 @@ import ( "time" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) // AddressBalance returns the balance of a single address. diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index cf1dcce..9aafb96 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -9,7 +9,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" ) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 8480192..2f3498e 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -9,7 +9,7 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/testutil" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap/zaptest" ) diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index a6b3754..c357c82 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -6,7 +6,7 @@ import ( "fmt" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) // Events returns the events with the given event IDs. If an event is not found, diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go index 1ef067b..fd00c1d 100644 --- a/persist/sqlite/events_test.go +++ b/persist/sqlite/events_test.go @@ -6,7 +6,7 @@ import ( "testing" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "lukechampine.com/frand" ) diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go index 6de47b6..df8a5a2 100644 --- a/persist/sqlite/keys.go +++ b/persist/sqlite/keys.go @@ -5,7 +5,7 @@ import ( "errors" "go.sia.tech/core/types" - "go.sia.tech/walletd/keys" + "go.sia.tech/walletd/v2/keys" ) // AddSigningKey adds a signing key to the store. If the key already exists, it diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go index bf38570..6c1573f 100644 --- a/persist/sqlite/keys_test.go +++ b/persist/sqlite/keys_test.go @@ -7,7 +7,7 @@ import ( "testing" "go.sia.tech/core/types" - "go.sia.tech/walletd/keys" + "go.sia.tech/walletd/v2/keys" "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 1929df5..f73d559 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -10,7 +10,7 @@ import ( "time" "github.com/mattn/go-sqlite3" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "lukechampine.com/frand" ) diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go index 3a8fc56..bf32f1f 100644 --- a/persist/sqlite/utxo.go +++ b/persist/sqlite/utxo.go @@ -6,7 +6,7 @@ import ( "fmt" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) // SiacoinElement returns an unspent Siacoin UTXO by its ID. diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 06e2733..e800d23 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -8,7 +8,7 @@ import ( "time" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/wallet" ) func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.ID, eventIDs []int64) (map[int64][]types.Address, error) { diff --git a/wallet/manager.go b/wallet/manager.go index 182e577..7021b52 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -11,7 +11,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" - "go.sia.tech/walletd/internal/threadgroup" + "go.sia.tech/walletd/v2/internal/threadgroup" "go.uber.org/zap" ) diff --git a/wallet/wallet.go b/wallet/wallet.go index 0174db4..bfff956 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -409,3 +409,18 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f return events } + +// NewSeedPhrase generates a random seed phrase. +func NewSeedPhrase() string { + return wallet.NewSeedPhrase() +} + +// SeedFromPhrase derives a 32-byte seed from the supplied phrase. +func SeedFromPhrase(seed *[32]byte, phrase string) error { + return wallet.SeedFromPhrase(seed, phrase) +} + +// KeyFromSeed returns the Ed25519 key derived from the supplied seed and index. +func KeyFromSeed(seed *[32]byte, index uint64) types.PrivateKey { + return wallet.KeyFromSeed(seed, index) +} diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index d9d3c18..5bd047f 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -18,8 +18,8 @@ import ( "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/testutil" - "go.sia.tech/walletd/persist/sqlite" - "go.sia.tech/walletd/wallet" + "go.sia.tech/walletd/v2/persist/sqlite" + "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" "lukechampine.com/frand" From c4ca561992c034d0b842d2613b542531433f93c4 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 11 Mar 2025 02:25:20 -0700 Subject: [PATCH 380/630] cmd: replace cwallet imports --- cmd/walletd/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index e484f4f..c34fcc3 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -11,7 +11,6 @@ import ( "syscall" "go.sia.tech/core/types" - cwallet "go.sia.tech/coreutils/wallet" "go.sia.tech/walletd/v2/api" "go.sia.tech/walletd/v2/build" "go.sia.tech/walletd/v2/config" @@ -340,10 +339,10 @@ func main() { cmd.Usage() return } - recoveryPhrase := cwallet.NewSeedPhrase() + recoveryPhrase := wallet.NewSeedPhrase() var seed [32]byte - checkFatalError("failed to parse mnemonic phrase", cwallet.SeedFromPhrase(&seed, recoveryPhrase)) - addr := types.StandardUnlockHash(cwallet.KeyFromSeed(&seed, 0).PublicKey()) + checkFatalError("failed to parse mnemonic phrase", wallet.SeedFromPhrase(&seed, recoveryPhrase)) + addr := types.StandardUnlockHash(wallet.KeyFromSeed(&seed, 0).PublicKey()) fmt.Println("Recovery Phrase:", recoveryPhrase) fmt.Println("Address", addr) From b8d36f9f7a9500cb1621fa8cbcb258f33bdfece1 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 12 Mar 2025 20:43:10 -0700 Subject: [PATCH 381/630] add wallet client construct examples --- api/construct_test.go | 83 ++++++++++++++++++++++++++++++++++++++++ api/construct_v2_test.go | 83 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 api/construct_test.go create mode 100644 api/construct_v2_test.go diff --git a/api/construct_test.go b/api/construct_test.go new file mode 100644 index 0000000..2c43efe --- /dev/null +++ b/api/construct_test.go @@ -0,0 +1,83 @@ +package api_test + +import ( + "go.sia.tech/core/types" + "go.sia.tech/walletd/v2/api" + "go.sia.tech/walletd/v2/wallet" +) + +func ExampleWalletClient_Construct() { + const ( + apiAddress = "localhost:9980/api" + apiPassword = "password" + ) + + client := api.NewClient(apiAddress, apiPassword) + + // generate a recovery phrase + phrase := wallet.NewSeedPhrase() + + // derive an address from the recovery phrase + var seed [32]byte + defer clear(seed[:]) + if err := wallet.SeedFromPhrase(&seed, phrase); err != nil { + panic(err) + } + + privateKey := wallet.KeyFromSeed(&seed, 0) + spendPolicy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions{ + PublicKeys: []types.UnlockKey{ + privateKey.PublicKey().UnlockKey(), + }, + SignaturesRequired: 1, + }, + } + address := spendPolicy.Address() + + // add a wallet + w1, err := client.AddWallet(api.WalletUpdateRequest{ + Name: "test", + Description: "test wallet", + }) + if err != nil { + panic(err) + } + + // init the wallet client to interact with the wallet + wc := client.Wallet(w1.ID) + + err = wc.AddAddress(wallet.Address{ + Address: address, + SpendPolicy: &spendPolicy, + }) + if err != nil { + panic(err) + } + + // create a transaction + resp, err := wc.Construct([]types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + }, nil, address) + if err != nil { + panic(err) + } + txn := resp.Transaction + + // sign the transaction + cs, err := client.ConsensusTipState() + if err != nil { + panic(err) + } + + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(txn, sig.ParentID, 0, 0, nil) + sig := privateKey.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + + // broadcast the transaction + if err := client.TxpoolBroadcast(resp.Basis, []types.Transaction{txn}, nil); err != nil { + panic(err) + } +} diff --git a/api/construct_v2_test.go b/api/construct_v2_test.go new file mode 100644 index 0000000..e3c9d78 --- /dev/null +++ b/api/construct_v2_test.go @@ -0,0 +1,83 @@ +package api_test + +import ( + "go.sia.tech/core/types" + "go.sia.tech/walletd/v2/api" + "go.sia.tech/walletd/v2/wallet" +) + +func ExampleWalletClient_ConstructV2() { + const ( + apiAddress = "localhost:9980/api" + apiPassword = "password" + ) + + client := api.NewClient(apiAddress, apiPassword) + + // generate a recovery phrase + phrase := wallet.NewSeedPhrase() + + // derive an address from the recovery phrase + var seed [32]byte + defer clear(seed[:]) + if err := wallet.SeedFromPhrase(&seed, phrase); err != nil { + panic(err) + } + + privateKey := wallet.KeyFromSeed(&seed, 0) + spendPolicy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions{ + PublicKeys: []types.UnlockKey{ + privateKey.PublicKey().UnlockKey(), + }, + SignaturesRequired: 1, + }, + } + address := spendPolicy.Address() + + // add a wallet + w1, err := client.AddWallet(api.WalletUpdateRequest{ + Name: "test", + Description: "test wallet", + }) + if err != nil { + panic(err) + } + + // init the wallet client to interact with the wallet + wc := client.Wallet(w1.ID) + + err = wc.AddAddress(wallet.Address{ + Address: address, + SpendPolicy: &spendPolicy, + }) + if err != nil { + panic(err) + } + + // create a transaction + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + }, nil, address) + if err != nil { + panic(err) + } + txn := resp.Transaction + + // sign the transaction + cs, err := client.ConsensusTipState() + if err != nil { + panic(err) + } + + sigHash := cs.InputSigHash(txn) + sig := privateKey.SignHash(sigHash) + for i := range txn.SiacoinInputs { + txn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + // broadcast the transaction + if err := client.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{txn}); err != nil { + panic(err) + } +} From d1123a37e2495424831c044bbd8a77a9f5525620 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 17 Mar 2025 17:05:47 -0700 Subject: [PATCH 382/630] chore(deps): update core --- cmd/walletd/node.go | 3 +++ go.mod | 9 +++++++-- go.sum | 4 ---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 231c2ad..fec8249 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -112,6 +112,9 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu case "anagami": network, genesisBlock = chain.TestnetAnagami() bootstrapPeers = syncer.AnagamiBootstrapPeers + case "erravimus": + network, genesisBlock = chain.TestnetErravimus() + bootstrapPeers = syncer.ErravimusBootstrapPeers default: return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } diff --git a/go.mod b/go.mod index 96e344c..eefa9dd 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,15 @@ go 1.23.1 toolchain go1.23.2 +replace ( + go.sia.tech/core => ../core + go.sia.tech/coreutils => ../coreutils +) + require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.4 - go.sia.tech/coreutils v0.12.0 + go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 + go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.1 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index f6a68cd..d87b240 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,6 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.4 h1:YdKfEDIqxKFkalCvWk6+HD7APVCZqrLGEEV/i2NKKMI= -go.sia.tech/core v0.10.4/go.mod h1:i/dfvjZRei6kR2tOLl27PexeYFb/jtCzRsplSBn3Fgc= -go.sia.tech/coreutils v0.12.0 h1:DOPFHveZeIxDATv9MW8G3kVEkOMOSfiSiN3jxcQ7Pik= -go.sia.tech/coreutils v0.12.0/go.mod h1:XBMg1imCUr5eYjaEemP9GZXvoOjYNvGTvmR/QHObAJg= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 020a80e2cb744b79594afda89e1ae4a206b2d78d Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 17 Mar 2025 17:21:49 -0700 Subject: [PATCH 383/630] deps --- go.mod | 5 ----- go.sum | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index eefa9dd..02c6ec5 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,6 @@ go 1.23.1 toolchain go1.23.2 -replace ( - go.sia.tech/core => ../core - go.sia.tech/coreutils => ../coreutils -) - require ( github.com/mattn/go-sqlite3 v1.14.24 go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 diff --git a/go.sum b/go.sum index d87b240..6847152 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 h1:7XFNkrIJngVfTIGMuZWz57TXMzJ6w1mbbPdqrjYywr8= +go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0/go.mod h1:i/dfvjZRei6kR2tOLl27PexeYFb/jtCzRsplSBn3Fgc= +go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e h1:/5MZa6nRrq6ghJ+YYKcO5QVOinZWpVDDYgrrxCx3cak= +go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e/go.mod h1:Z14ILJqJkTKyEhaoYvCbW6Y61dJG5NSHFoI+yeDNcI8= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From c29d26df7b8fc3c69520b37742312a493c75bc5b Mon Sep 17 00:00:00 2001 From: linghuying <1599935829@qq.com> Date: Thu, 20 Mar 2025 15:43:35 +0800 Subject: [PATCH 384/630] chore: fix comment Signed-off-by: linghuying <1599935829@qq.com> --- wallet/seed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/seed.go b/wallet/seed.go index ecc4a65..582cd44 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -38,7 +38,7 @@ func NewSeed() Seed { return NewSeedFromEntropy(&entropy) } -// NewSeedFromEntropy returns a the specified seed. +// NewSeedFromEntropy returns the specified seed. func NewSeedFromEntropy(entropy *[32]byte) Seed { return Seed{entropy} } From 99da154874d9dfebdb270f57bc91eadec91ff5fd Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 24 Mar 2025 20:06:25 -0700 Subject: [PATCH 385/630] all: remove key store --- .changeset/add_signing_key_store.md | 26 ----- api/api_test.go | 116 +------------------ api/client.go | 31 ----- api/server.go | 81 ------------- cmd/walletd/node.go | 10 -- go.mod | 2 +- keys/manager.go | 174 ---------------------------- keys/manager_test.go | 82 ------------- persist/sqlite/keys.go | 80 ------------- persist/sqlite/keys_test.go | 76 ------------ 10 files changed, 3 insertions(+), 675 deletions(-) delete mode 100644 .changeset/add_signing_key_store.md delete mode 100644 keys/manager.go delete mode 100644 keys/manager_test.go delete mode 100644 persist/sqlite/keys.go delete mode 100644 persist/sqlite/keys_test.go diff --git a/.changeset/add_signing_key_store.md b/.changeset/add_signing_key_store.md deleted file mode 100644 index 217495d..0000000 --- a/.changeset/add_signing_key_store.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -default: minor ---- - -# Add ed25519 key store - -Adds an optional ed25519 key store for integrators to store arbitrary private keys for signing transactions. It allows for both generating private keys on the server and importing private keys. Keys are stored encrypted using a user-provided secret. - -The store is disabled by default. It can be enabled through the config file or the CLI flag `--keystore`. If the store is enabled, an encryption key must also be provided through the environment variable `WALLETD_KEYSTORE_SECRET`. - -*The endpoint will return 404 if the `--public` CLI flag is set. It is only recommended for use on localhost. It is not used by the UI.* - -```go - -client := api.NewClient(walletAddr, walletdPassword) - -pubKey, err := client.GenerateSigningKey() -if err != nil { - panic(err) -} - -sig, err := client.SignHash(pubKey, hash) -if err != nil { - panic(err) -} -``` diff --git a/api/api_test.go b/api/api_test.go index 547a46c..dd18f04 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -17,7 +17,6 @@ import ( "go.sia.tech/jape" "go.sia.tech/walletd/v2/api" "go.sia.tech/walletd/v2/internal/testutil" - "go.sia.tech/walletd/v2/keys" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "go.uber.org/zap/zaptest" @@ -39,14 +38,8 @@ func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logge } tb.Cleanup(func() { wm.Close() }) - km, err := keys.NewManager(cn.Store, "foo") - if err != nil { - tb.Fatal("failed to create key manager:", err) - } - tb.Cleanup(func() { km.Close() }) - server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithKeyManager(km), api.WithDebug(), api.WithLogger(log)), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(log)), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -1282,12 +1275,6 @@ func TestAPISecurity(t *testing.T) { } defer wm.Close() - km, err := keys.NewManager(cn.Store, "foo") - if err != nil { - t.Fatal(err) - } - defer km.Close() - httpListener, err := net.Listen("tcp", ":0") if err != nil { t.Fatal("failed to listen:", err) @@ -1295,7 +1282,7 @@ func TestAPISecurity(t *testing.T) { defer httpListener.Close() server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithKeyManager(km), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -1312,11 +1299,6 @@ func TestAPISecurity(t *testing.T) { t.Fatal(err) } - // check that the signing key endpoints are working - if _, err := c.GenerateSigningKey(); err != nil { - t.Fatal(err) - } - // create a client with incorrect credentials c = api.NewClient("http://"+httpListener.Addr().String(), "wrong") if _, err := c.ConsensusTip(); err == nil { @@ -1325,13 +1307,6 @@ func TestAPISecurity(t *testing.T) { t.Fatal("expected auth error, got", err) } - // check that the signing key endpoints are working - if _, err := c.GenerateSigningKey(); err == nil { - t.Fatal("expected auth error") - } else if err.Error() == "unauthorized" { - t.Fatal("expected auth error, got", err) - } - // replace the handler with a new one that doesn't require auth replaceHandler() @@ -1358,13 +1333,6 @@ func TestAPISecurity(t *testing.T) { t.Fatal(err) } - // check that the signing endpoint returns 404 when public mode is enabled - if _, err := c.SignHash(frand.Entropy256(), frand.Entropy256()); err == nil { - t.Fatal("expected 404 error") - } else if !strings.Contains(err.Error(), "404") { - t.Fatal("expected 404 error, got", err) - } - // check that a private endpoint is still protected if _, err := c.Wallets(); err == nil { t.Fatal("expected auth error") @@ -1511,83 +1479,3 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } cn.MineBlocks(t, types.VoidAddress, 1) } - -func TestSigning(t *testing.T) { - log := zaptest.NewLogger(t) - n, genesisBlock := testutil.V2Network() - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) - - pk, err := c.GenerateSigningKey() - if err != nil { - t.Fatal(err) - } - - // create a wallet - w, err := c.AddWallet(api.WalletUpdateRequest{ - Name: "primary", - }) - if err != nil { - t.Fatal(err) - } - - wc := c.Wallet(w.ID) - - policy := types.SpendPolicy{Type: types.PolicyTypePublicKey(pk)} - addr := policy.Address() - - err = wc.AddAddress(wallet.Address{ - Address: addr, - SpendPolicy: &policy, - }) - if err != nil { - t.Fatal(err) - } - - // fund the wallet - cn.MineBlocks(t, addr, 1) - cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) - - resp, err := wc.ConstructV2([]types.SiacoinOutput{ - {Value: types.Siacoins(100), Address: addr}, - }, nil, addr) - if err != nil { - t.Fatal(err) - } - - cs, err := c.ConsensusTipState() - if err != nil { - t.Fatal(err) - } - - // sign the transaction - sigHash := cs.InputSigHash(resp.Transaction) - for i, si := range resp.Transaction.SiacoinInputs { - pk := types.PublicKey(si.SatisfiedPolicy.Policy.Type.(types.PolicyTypePublicKey)) - - sig, err := c.SignHash(pk, sigHash) - if err != nil { - t.Fatal(err) - } else if !pk.VerifyHash(sigHash, sig) { - t.Fatal("signature verification failed") - } - resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} - } - - if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { - t.Fatal(err) - } - cn.MineBlocks(t, types.VoidAddress, 1) - - events, err := wc.Events(0, 5) - if err != nil { - t.Fatal(err) - } else if len(events) != 2 { - t.Fatalf("expected 2 events, got %v", len(events)) - } else if events[0].Type != wallet.EventTypeV2Transaction { - t.Fatalf("expected event type %q, got %q", wallet.EventTypeV2Transaction, events[0].Type) - } else if types.TransactionID(events[0].ID) != resp.ID { - t.Fatalf("expected event ID %q, got %q", resp.ID, events[0].ID) - } -} diff --git a/api/client.go b/api/client.go index 064eb64..5b2c9ce 100644 --- a/api/client.go +++ b/api/client.go @@ -2,7 +2,6 @@ package api import ( "fmt" - "net/url" "sync" "time" @@ -278,36 +277,6 @@ func (c *Client) SpentSiafundElement(id types.SiafundOutputID) (resp ElementSpen return } -// GenerateSigningKey generates a new ed25519 private key -// on the server and adds it to the key store. Returns the -// public key. -func (c *Client) GenerateSigningKey() (types.PublicKey, error) { - var resp AddSigningKeyResponse - err := c.c.POST("/keys/generate/ed25519", nil, &resp) - return resp.PublicKey, err -} - -// ImportSigningKey imports an ed25519 signing key into the key store. -// Returns the public key. -func (c *Client) ImportSigningKey(sk types.PrivateKey) (types.PublicKey, error) { - var resp AddSigningKeyResponse - err := c.c.POST("/keys/ed25519", AddSigningKeyRequest{PrivateKey: sk}, &resp) - return resp.PublicKey, err -} - -// DeleteSigningKey deletes an ed25519 signing key from the key store. -func (c *Client) DeleteSigningKey(pk types.PublicKey) error { - return c.c.DELETE(fmt.Sprintf("/keys/ed25519/%s", pk)) -} - -// SignHash signs a hash with the specified key. If the key is not found, it -// returns 404 and [keys.ErrNotFound]. -func (c *Client) SignHash(key types.PublicKey, hash types.Hash256) (types.Signature, error) { - var resp SignHashResponse - err := c.c.POST(fmt.Sprintf("/keys/ed25519/%s/sign", url.PathEscape(key.String())), SignHashRequest{hash}, &resp) - return resp.Signature, err -} - // A WalletClient provides methods for interacting with a particular wallet on a // walletd API server. type WalletClient struct { diff --git a/api/server.go b/api/server.go index cb293de..815a9fd 100644 --- a/api/server.go +++ b/api/server.go @@ -19,7 +19,6 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/syncer" "go.sia.tech/walletd/v2/build" - "go.sia.tech/walletd/v2/keys" "go.sia.tech/walletd/v2/wallet" ) @@ -40,13 +39,6 @@ func WithDebug() ServerOption { } } -// WithKeyManager sets the key manager used by the server. -func WithKeyManager(ks SigningKeyManager) ServerOption { - return func(s *server) { - s.km = ks - } -} - // WithPublicEndpoints sets whether the server should disable authentication // on endpoints that are safe for use when running walletd as a service. func WithPublicEndpoints(public bool) ServerOption { @@ -140,13 +132,6 @@ type ( Reserve([]types.Hash256) error Release([]types.Hash256) } - - // A SigningKeyManager manages ed25519 signing keys. - SigningKeyManager interface { - Add(types.PrivateKey) error - Delete(types.PublicKey) error - Sign(types.PublicKey, types.Hash256) (types.Signature, error) - } ) type server struct { @@ -159,7 +144,6 @@ type server struct { cm ChainManager s Syncer wm WalletManager - km SigningKeyManager scanMu sync.Mutex // for resubscribe scanInProgress bool @@ -1281,63 +1265,6 @@ func (s *server) outputsSiafundHandlerGET(jc jape.Context) { jc.Encode(output) } -func (s *server) keysEd25519GenerateHandlerPOST(jc jape.Context) { - sk := types.GeneratePrivateKey() - defer clear(sk) - if jc.Check("failed to add key", s.km.Add(sk)) != nil { - return - } - jc.Encode(AddSigningKeyResponse{ - PublicKey: sk.PublicKey(), - }) -} - -func (s *server) keysEd25519HandlerPUT(jc jape.Context) { - var req AddSigningKeyRequest - defer clear(req.PrivateKey) - if jc.Decode(&req) != nil { - return - } else if jc.Check("failed to add key", s.km.Add(req.PrivateKey)) != nil { - return - } - - jc.Encode(AddSigningKeyResponse{ - PublicKey: req.PrivateKey.PublicKey(), - }) -} - -func (s *server) keysEd25519HandlerDELETE(jc jape.Context) { - var pk types.PublicKey - if jc.DecodeParam("pub", &pk) != nil { - return - } else if jc.Check("failed to remove key", s.km.Delete(pk)) != nil { - return - } - jc.EmptyResonse() -} - -func (s *server) keysEd25519SignHandlerPOST(jc jape.Context) { - var pub types.PublicKey - if jc.DecodeParam("pub", &pub) != nil { - return - } - var req SignHashRequest - if jc.Decode(&req) != nil { - return - } - - sig, err := s.km.Sign(pub, req.Hash) - if errors.Is(err, keys.ErrNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if jc.Check("failed to sign message", err) != nil { - return - } - jc.Encode(SignHashResponse{ - Signature: sig, - }) -} - func (s *server) debugMineHandler(jc jape.Context) { var req DebugMineRequest if jc.Decode(&req) != nil { @@ -1500,14 +1427,6 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), } - if srv.km != nil && !srv.publicEndpoints { - // key management endpoints are disabled on public nodes - handlers["POST /keys/generate/ed25519"] = wrapAuthHandler(srv.keysEd25519GenerateHandlerPOST) - handlers["PUT /keys/ed25519"] = wrapAuthHandler(srv.keysEd25519HandlerPUT) - handlers["DELETE /keys/ed25519/:pub"] = wrapAuthHandler(srv.keysEd25519HandlerDELETE) - handlers["POST /keys/ed25519/:pub/sign"] = wrapAuthHandler(srv.keysEd25519SignHandlerPOST) - } - if srv.debugEnabled { handlers["POST /debug/mine"] = wrapAuthHandler(srv.debugMineHandler) handlers["GET /debug/pprof/:handler"] = wrapAuthHandler(srv.pprofHandler) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index fec8249..2f4d0e6 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -22,7 +22,6 @@ import ( "go.sia.tech/walletd/v2/api" "go.sia.tech/walletd/v2/build" "go.sia.tech/walletd/v2/config" - "go.sia.tech/walletd/v2/keys" "go.sia.tech/walletd/v2/persist/sqlite" "go.sia.tech/walletd/v2/wallet" "go.sia.tech/web/walletd" @@ -213,15 +212,6 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu if enableDebug { apiOpts = append(apiOpts, api.WithDebug()) } - if cfg.KeyStore.Enabled { - km, err := keys.NewManager(store, cfg.KeyStore.Secret) - if err != nil { - return fmt.Errorf("failed to create key manager: %w", err) - } - defer km.Close() - - apiOpts = append(apiOpts, api.WithKeyManager(km)) - } api := api.NewServer(cm, s, wm, apiOpts...) web := walletd.Handler() server := &http.Server{ diff --git a/go.mod b/go.mod index 02c6ec5..efb5982 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.36.0 golang.org/x/term v0.30.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 @@ -32,6 +31,7 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.34.0 // indirect diff --git a/keys/manager.go b/keys/manager.go deleted file mode 100644 index d7eb403..0000000 --- a/keys/manager.go +++ /dev/null @@ -1,174 +0,0 @@ -package keys - -import ( - "crypto/cipher" - "crypto/ed25519" - "errors" - "fmt" - "strings" - - "go.sia.tech/core/types" - "go.sia.tech/walletd/v2/internal/threadgroup" - "golang.org/x/crypto/argon2" - "golang.org/x/crypto/chacha20poly1305" - "lukechampine.com/frand" -) - -var ( - // ErrInvalidSize is returned when a key has an invalid size. - ErrInvalidSize = errors.New("invalid key size") - // ErrNotFound is returned when a signing key is not found. - ErrNotFound = errors.New("not found") - // ErrSaltSet is returned when the key salt is already set. - ErrSaltSet = errors.New("salt already set") - // ErrIncorrectSecret is returned when the secret is incorrect. - ErrIncorrectSecret = errors.New("incorrect secret") -) - -type ( - // A Store saves and loads ed25519 signing keys. - Store interface { - // GetSigningKey returns the encrypted signing key with the given public key. - // If the key is not found, it returns [ErrNotFound]. The key must be - // decrypted before being used. - GetSigningKey(types.PublicKey) ([]byte, error) - // AddSigningKey adds a signing key to the store. If the key already - // exists, nil is returned. The key must be encrypted before being - // stored. - AddSigningKey(pk types.PublicKey, buf []byte) error - // DeleteSigningKey deletes the signing key with the given public key. - // If the key does not exist, it returns [ErrNotFound]. - DeleteSigningKey(types.PublicKey) error - - // KeySalt returns the salt used to derive the key encryption - // key. If no salt has been set, KeySalt should return (nil, nil). - GetKeySalt() ([]byte, error) - - // SetKeySalt sets the salt used to derive the key encryption key. - // If a salt has already been set, [keys.ErrSaltSet] is returned. - SetKeySalt([]byte) error - - // GetBytesForVerify returns random encrypted bytes for verifying - // the encryption key. - GetBytesForVerify() ([]byte, error) - } - - // A Manager is a key-value store for ed25519 signing keys. - Manager struct { - tg *threadgroup.ThreadGroup - - aead cipher.AEAD - store Store - } -) - -// Add adds a key to the manager. If the key is not the correct -// size, it returns [ErrInvalidSize]. -func (m *Manager) Add(sk types.PrivateKey) error { - if len(sk) != ed25519.PrivateKeySize { - return ErrInvalidSize - } - - done, err := m.tg.Add() - if err != nil { - return err - } - defer done() - - n := m.aead.NonceSize() - buf := make([]byte, m.aead.NonceSize(), n+len(sk)+m.aead.Overhead()) - frand.Read(buf) - encrypted := m.aead.Seal(buf, buf, sk, nil) - return m.store.AddSigningKey(sk.PublicKey(), encrypted) -} - -// Sign returns the signature for a hash. If the key is not -// found, it returns [ErrNotFound]. -func (m *Manager) Sign(key types.PublicKey, hash types.Hash256) (types.Signature, error) { - done, err := m.tg.Add() - if err != nil { - return types.Signature{}, err - } - defer done() - - buf, err := m.store.GetSigningKey(key) - if err != nil { - return types.Signature{}, err - } - defer clear(buf) - - sk := make(types.PrivateKey, 0, ed25519.PrivateKeySize) - defer clear(sk) - sk, err = m.aead.Open(sk, buf[:m.aead.NonceSize()], buf[m.aead.NonceSize():], nil) - if err != nil { - return types.Signature{}, fmt.Errorf("failed to decrypt key: %w", err) - } - - if len(sk) != ed25519.PrivateKeySize { - return types.Signature{}, ErrInvalidSize - } - return types.PrivateKey(sk).SignHash(hash), nil -} - -// Delete removes a key from the manager. -func (m *Manager) Delete(key types.PublicKey) error { - done, err := m.tg.Add() - if err != nil { - return err - } - defer done() - return m.store.DeleteSigningKey(key) -} - -// Close closes the manager. -func (m *Manager) Close() error { - m.tg.Stop() - return nil -} - -// NewManager creates a new key manager. If the store contains -// encrypted keys, the secret must match the secret used to encrypt -// the existing keys. If the secret is incorrect, NewManager returns -// [ErrIncorrectSecret]. -// -// Keys are encrypted using ChaCha20-Poly1305 with a key derived from -// the secret using Argon2ID. -func NewManager(store Store, secret string) (*Manager, error) { - salt, err := store.GetKeySalt() - if err != nil { - return nil, fmt.Errorf("failed to get key salt: %w", err) - } else if len(salt) == 0 { - salt = frand.Bytes(32) - if err := store.SetKeySalt(salt); err != nil { - return nil, fmt.Errorf("failed to set key salt: %w", err) - } - } - - encryptionKey := argon2.IDKey([]byte(secret), salt, 3, 64*1024, 4, 32) - aead, err := chacha20poly1305.NewX(encryptionKey) - if err != nil { - return nil, fmt.Errorf("failed to create AEAD: %w", err) - } - - buf, err := store.GetBytesForVerify() - if err != nil && !errors.Is(err, ErrNotFound) { - return nil, fmt.Errorf("failed to get bytes for verify: %w", err) - } else if err == nil { - defer clear(buf) - - decrypted, err := aead.Open(nil, buf[:aead.NonceSize()], buf[aead.NonceSize():], nil) - if err != nil { - if strings.Contains(err.Error(), "message authentication failed") { - return nil, ErrIncorrectSecret - } - return nil, fmt.Errorf("failed to verify encryption key: %w", err) - } - defer clear(decrypted) - } - - return &Manager{ - aead: aead, - store: store, - tg: threadgroup.New(), - }, nil -} diff --git a/keys/manager_test.go b/keys/manager_test.go deleted file mode 100644 index 7fdc2c3..0000000 --- a/keys/manager_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package keys_test - -import ( - "errors" - "path/filepath" - "testing" - - "go.sia.tech/core/types" - "go.sia.tech/walletd/v2/keys" - "go.sia.tech/walletd/v2/persist/sqlite" - "go.uber.org/zap" - "lukechampine.com/frand" -) - -func TestKeyManager(t *testing.T) { - store, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zap.NewNop()) - if err != nil { - t.Fatal(err) - } - defer store.Close() - - m, err := keys.NewManager(store, "foo") - if err != nil { - t.Fatal(err) - } - defer m.Close() - - sk := types.GeneratePrivateKey() - - if err := m.Add(sk); err != nil { - t.Fatal(err) - } - - // try to add it again - if err := m.Add(sk); err != nil { - t.Fatal(err) - } - - hash := types.Hash256(frand.Entropy256()) - - sig, err := m.Sign(sk.PublicKey(), hash) - if err != nil { - t.Fatal(err) - } else if !sk.PublicKey().VerifyHash(hash, sig) { - t.Fatal("signature failed to verify") - } - - // try to sign with an unknown key - _, err = m.Sign(types.GeneratePrivateKey().PublicKey(), hash) - if !errors.Is(err, keys.ErrNotFound) { - t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) - } - - if err := m.Close(); err != nil { - t.Fatal(err) - } - - _, err = keys.NewManager(store, "foobar") - if !errors.Is(err, keys.ErrIncorrectSecret) { - t.Fatalf("expected %v, got %v", keys.ErrIncorrectSecret, err) - } - - m, err = keys.NewManager(store, "foo") - if err != nil { - t.Fatal(err) - } - defer m.Close() - - sig, err = m.Sign(sk.PublicKey(), hash) - if err != nil { - t.Fatal(err) - } else if !sk.PublicKey().VerifyHash(hash, sig) { - t.Fatal("signature failed to verify") - } - - // delete the key - if err := m.Delete(sk.PublicKey()); err != nil { - t.Fatal(err) - } else if _, err := m.Sign(sk.PublicKey(), hash); !errors.Is(err, keys.ErrNotFound) { - t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) - } -} diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go deleted file mode 100644 index df8a5a2..0000000 --- a/persist/sqlite/keys.go +++ /dev/null @@ -1,80 +0,0 @@ -package sqlite - -import ( - "database/sql" - "errors" - - "go.sia.tech/core/types" - "go.sia.tech/walletd/v2/keys" -) - -// AddSigningKey adds a signing key to the store. If the key already exists, it -// is not added again. -func (s *Store) AddSigningKey(pk types.PublicKey, buf []byte) error { - return s.transaction(func(tx *txn) error { - _, err := tx.Exec("INSERT INTO signing_keys (public_key, private_key) VALUES (?, ?) ON CONFLICT (public_key) DO NOTHING", encode(pk), buf) - return err - }) -} - -// GetSigningKey returns the private key corresponding to the given public key. -// If the key is not found, it returns [keys.ErrNotFound]. -func (s *Store) GetSigningKey(pk types.PublicKey) (buf []byte, err error) { - err = s.transaction(func(tx *txn) error { - err := s.db.QueryRow("SELECT private_key FROM signing_keys WHERE public_key = ?", encode(pk)).Scan(&buf) - if errors.Is(err, sql.ErrNoRows) { - return keys.ErrNotFound - } else if err != nil { - return err - } - return nil - }) - return -} - -// DeleteSigningKey deletes the signing key with the given public key. If the key -// does not exist, it returns nil. -func (s *Store) DeleteSigningKey(pk types.PublicKey) error { - return s.transaction(func(tx *txn) error { - _, err := tx.Exec("DELETE FROM signing_keys WHERE public_key = ?", encode(pk)) - return err - }) -} - -// GetKeySalt returns the salt used to derive the key encryption -// key. If no salt has been set, GetKeySalt returns (nil, nil). -func (s *Store) GetKeySalt() (salt []byte, err error) { - err = s.transaction(func(tx *txn) error { - err := s.db.QueryRow("SELECT key_salt FROM global_settings").Scan(&salt) - return err - }) - return -} - -// SetKeySalt sets the salt used to derive the key encryption key. -// If a salt has already been set, [keys.ErrSaltSet] is returned. -func (s *Store) SetKeySalt(salt []byte) error { - return s.transaction(func(tx *txn) error { - res, err := tx.Exec("UPDATE global_settings SET key_salt = ? WHERE key_salt IS NULL", salt) - if err != nil { - return err - } else if n, _ := res.RowsAffected(); n == 0 { - return keys.ErrSaltSet - } - return nil - }) -} - -// GetBytesForVerify returns random encrypted bytes for verifying -// the encryption key. If there are no keys in the store, it returns -// [keys.ErrNotFound]. -func (s *Store) GetBytesForVerify() (buf []byte, err error) { - err = s.transaction(func(tx *txn) error { - err := s.db.QueryRow("SELECT private_key FROM signing_keys LIMIT 1").Scan(&buf) - if errors.Is(err, sql.ErrNoRows) { - return keys.ErrNotFound - } - return err - }) - return -} diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go deleted file mode 100644 index 6c1573f..0000000 --- a/persist/sqlite/keys_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package sqlite - -import ( - "bytes" - "errors" - "path/filepath" - "testing" - - "go.sia.tech/core/types" - "go.sia.tech/walletd/v2/keys" - "go.uber.org/zap/zaptest" - "lukechampine.com/frand" -) - -func TestSigningKeys(t *testing.T) { - store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) - if err != nil { - t.Fatal(err) - } - defer store.Close() - - sk := types.GeneratePrivateKey() - - _, err = store.GetSigningKey(sk.PublicKey()) - if !errors.Is(err, keys.ErrNotFound) { - t.Fatal(err) - } - - expected := frand.Bytes(64) // mock encrypted key - if err = store.AddSigningKey(sk.PublicKey(), expected); err != nil { - t.Fatal(err) - } - - buf, err := store.GetSigningKey(sk.PublicKey()) - if err != nil { - t.Fatal(err) - } else if !bytes.Equal(expected, buf) { - t.Fatal("keys don't match") - } -} - -func TestSalt(t *testing.T) { - store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) - if err != nil { - t.Fatal(err) - } - defer store.Close() - - assertSalt := func(t *testing.T, expected []byte) { - t.Helper() - s, err := store.GetKeySalt() - if err != nil { - t.Fatal(err) - } else if expected == nil && s != nil { - t.Fatal("expected nil salt") // bytes.Equal([]byte{}, nil) == true - } else if !bytes.Equal(s, expected) { - t.Fatal("salts don't match") - } - } - - // check salt is initially nil - assertSalt(t, nil) - - expected := frand.Bytes(32) - if err = store.SetKeySalt(expected); err != nil { - t.Fatal(err) - } - assertSalt(t, expected) - - if err = store.SetKeySalt(frand.Bytes(32)); !errors.Is(err, keys.ErrSaltSet) { - t.Fatalf("expected %v, got %v", keys.ErrSaltSet, err) - } - - // check salt was not changed - assertSalt(t, expected) -} From 629b4eae249669d3b27e7a06b9c9c10658552929 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 08:10:45 +0000 Subject: [PATCH 386/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.29.1 to 0.29.2 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.29.1...walletd@0.29.2) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index efb5982..619a81f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e go.sia.tech/jape v0.12.1 - go.sia.tech/web/walletd v0.29.1 + go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.30.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 6847152..68e30cf 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.29.1 h1:fWknehIhFuXIeoeTO4n86OL14sx7ynBkJKJJAMR3wFM= -go.sia.tech/web/walletd v0.29.1/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.29.2 h1:bi0A4ZDROEAmh3CtJ+fJ/3dXfobc37kjqmnmJTNPmM4= +go.sia.tech/web/walletd v0.29.2/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From 2ac0d27ccbf0f3504c7c6d6bb794a7a4032fbed8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 25 Mar 2025 08:20:10 +0000 Subject: [PATCH 387/630] chore: prepare release 2.1.0 --- .changeset/add_spent_element_endpoints.md | 17 ----------------- .changeset/fix_v2_signing.md | 8 -------- CHANGELOG.md | 22 ++++++++++++++++++++++ go.mod | 2 +- 4 files changed, 23 insertions(+), 26 deletions(-) delete mode 100644 .changeset/add_spent_element_endpoints.md delete mode 100644 .changeset/fix_v2_signing.md diff --git a/.changeset/add_spent_element_endpoints.md b/.changeset/add_spent_element_endpoints.md deleted file mode 100644 index a117027..0000000 --- a/.changeset/add_spent_element_endpoints.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -default: minor ---- - -# Added Spent Element Endpoints - -Added two new endpoints `[GET] /outputs/siacoin/:id/spent` and `[GET] /outputs/siafund/:id/spent`. These endpoints will return a boolean, indicating whether the UTXO was spent, and the transaction it was spent in. These endpoints are designed to make verifying Atomic swaps easier. - -#### Example Usage - -```` -$ curl http://localhost:9980/api/outputs/siacoin/9b89152bb967130326702c9bfb51109e9f80274ec314ba58d9ef49b881340f2f/spent -{ - spent: true, - event: {} -} -``` diff --git a/.changeset/fix_v2_signing.md b/.changeset/fix_v2_signing.md deleted file mode 100644 index 3e3ea46..0000000 --- a/.changeset/fix_v2_signing.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -default: minor ---- - -# Fixes sending V2 transactions in the UI - -- Fixes V2 signing for wallets that do not have siafund outputs. Fixes #247 - diff --git a/CHANGELOG.md b/CHANGELOG.md index 1281049..958be22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## 2.1.0 (2025-03-25) + +### Features + +#### Added Spent Element Endpoints + +Added two new endpoints `[GET] /outputs/siacoin/:id/spent` and `[GET] /outputs/siafund/:id/spent`. These endpoints will return a boolean, indicating whether the UTXO was spent, and the transaction it was spent in. These endpoints are designed to make verifying Atomic swaps easier. + +##### Example Usage + +```` +$ curl http://localhost:9980/api/outputs/siacoin/9b89152bb967130326702c9bfb51109e9f80274ec314ba58d9ef49b881340f2f/spent +{ + spent: true, + event: {} +} +``` + +#### Fixes sending V2 transactions in the UI + +- Fixes V2 signing for wallets that do not have siafund outputs. Fixes #247 + ## 2.0.0 (2025-02-21) ### Breaking Changes diff --git a/go.mod b/go.mod index 619a81f..fd1a2a7 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.0.0 +module go.sia.tech/walletd/v2 // v2.1.0 go 1.23.1 From 1dc3f2551a8e36c6e5bbe46fdb6a2b12e6b769ac Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 25 Mar 2025 10:17:26 -0700 Subject: [PATCH 388/630] easier support for local testnets --- .changeset/support_custom_networks.md | 66 +++++++++++++++++++++++++++ cmd/walletd/node.go | 39 +++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 .changeset/support_custom_networks.md diff --git a/.changeset/support_custom_networks.md b/.changeset/support_custom_networks.md new file mode 100644 index 0000000..ba75985 --- /dev/null +++ b/.changeset/support_custom_networks.md @@ -0,0 +1,66 @@ +--- +default: minor +--- + +# Add support for custom networks + +Adds support for loading custom network parameters from a local file. This makes it easier to setup local testnets for development. A network file can be specified by using a file path for the `--network` CLI flag. The file should be JSON or YAML formatted with the following structure: + +```json +{ + "network": { + "name": "", + "initialCoinbase": "0", + "minimumCoinbase": "0", + "initialTarget": "0000000000000000000000000000000000000000000000000000000000000000", + "blockInterval": 0, + "maturityDelay": 0, + "hardforkDevAddr": { + "height": 0, + "oldAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "newAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkTax": { + "height": 0 + }, + "hardforkStorageProof": { + "height": 0 + }, + "hardforkOak": { + "height": 0, + "fixHeight": 0, + "genesisTimestamp": "0001-01-01T00:00:00Z" + }, + "hardforkASIC": { + "height": 0, + "oakTime": 0, + "oakTarget": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "hardforkFoundation": { + "height": 0, + "primaryAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "failsafeAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkV2": { + "allowHeight": 0, + "requireHeight": 0 + } + }, + "genesis": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "0001-01-01T00:00:00Z", + "siacoinOutputs": [ + + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + } + ], + "minerPayouts": null, + "transactions": null + } +} +``` \ No newline at end of file diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 2f4d0e6..c2ff152 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "net" @@ -26,6 +27,7 @@ import ( "go.sia.tech/walletd/v2/wallet" "go.sia.tech/web/walletd" "go.uber.org/zap" + "gopkg.in/yaml.v3" "lukechampine.com/upnp" ) @@ -97,6 +99,35 @@ func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error return d.ExternalIP() } +func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { + f, err := os.Open(fp) + if err != nil { + return nil, types.Block{}, fmt.Errorf("failed to open network file: %w", err) + } + defer f.Close() + + var network struct { + Network consensus.Network `json:"network" yaml:"network"` + Genesis types.Block `json:"genesis" yaml:"genesis"` + } + + switch filepath.Ext(fp) { + case ".yml", ".yaml": + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + if err := dec.Decode(&network); err != nil { + return nil, types.Block{}, fmt.Errorf("failed to decode YAML network file: %w", err) + } + case ".json": + if err := json.NewDecoder(f).Decode(&network); err != nil { + return nil, types.Block{}, fmt.Errorf("failed to decode JSON network file: %w", err) + } + default: + return nil, types.Block{}, errors.New("unknown network file format") + } + return &network.Network, network.Genesis, nil +} + func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebug bool) error { var network *consensus.Network var genesisBlock types.Block @@ -115,7 +146,13 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu network, genesisBlock = chain.TestnetErravimus() bootstrapPeers = syncer.ErravimusBootstrapPeers default: - return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") + var err error + network, genesisBlock, err = loadCustomNetwork(cfg.Consensus.Network) + if errors.Is(err, os.ErrNotExist) { + return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") + } else if err != nil { + return fmt.Errorf("failed to load custom network: %w", err) + } } bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) From 2f0a39a5e1cafa06a25e9a2293865d5ea4966624 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 25 Mar 2025 10:20:50 -0700 Subject: [PATCH 389/630] use zen for example --- .changeset/support_custom_networks.md | 67 +++++++++++++++------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/.changeset/support_custom_networks.md b/.changeset/support_custom_networks.md index ba75985..897c9a1 100644 --- a/.changeset/support_custom_networks.md +++ b/.changeset/support_custom_networks.md @@ -9,58 +9,65 @@ Adds support for loading custom network parameters from a local file. This makes ```json { "network": { - "name": "", - "initialCoinbase": "0", - "minimumCoinbase": "0", - "initialTarget": "0000000000000000000000000000000000000000000000000000000000000000", - "blockInterval": 0, - "maturityDelay": 0, + "name": "zen", + "initialCoinbase": "300000000000000000000000000000", + "minimumCoinbase": "30000000000000000000000000000", + "initialTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "blockInterval": 600000000000, + "maturityDelay": 144, "hardforkDevAddr": { - "height": 0, + "height": 1, "oldAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", "newAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" }, "hardforkTax": { - "height": 0 + "height": 2 }, "hardforkStorageProof": { - "height": 0 + "height": 5 }, "hardforkOak": { - "height": 0, - "fixHeight": 0, - "genesisTimestamp": "0001-01-01T00:00:00Z" + "height": 10, + "fixHeight": 12, + "genesisTimestamp": "2023-01-13T00:53:20-08:00" }, "hardforkASIC": { - "height": 0, - "oakTime": 0, - "oakTarget": "0000000000000000000000000000000000000000000000000000000000000000" + "height": 20, + "oakTime": 10000000000000, + "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000" }, "hardforkFoundation": { - "height": 0, - "primaryAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "height": 30, + "primaryAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", "failsafeAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" }, "hardforkV2": { - "allowHeight": 0, - "requireHeight": 0 + "allowHeight": 112000, + "requireHeight": 114000 } }, "genesis": { "parentID": "0000000000000000000000000000000000000000000000000000000000000000", "nonce": 0, - "timestamp": "0001-01-01T00:00:00Z", - "siacoinOutputs": [ - - ], - "siafundOutputs": [ - { - "value": 10000, - "address": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" - } - ], + "timestamp": "2023-01-13T00:53:20-08:00", "minerPayouts": null, - "transactions": null + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] } } ``` \ No newline at end of file From 2dd56ef449738a59bdd4d9353292e0ce2f6f8ad5 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 26 Mar 2025 19:01:28 -0700 Subject: [PATCH 390/630] remove yaml support, update readme --- README.md | 72 ++++++++++++++++++++++++++++++++++++++++++++- cmd/walletd/main.go | 2 +- cmd/walletd/node.go | 16 ++-------- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8f3052b..bed0c4b 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Flags: -index.mode string address index mode (personal, full, none) (default "full") -network string - network to connect to (default "mainnet") + network to connect to; must be one of 'mainnet', 'zen', 'anagami', or the path to a custom network file for a local testnet -upnp attempt to forward ports and discover IP with UPnP -keystore @@ -183,4 +183,74 @@ services: ```sh docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/siafoundation/walletd:master . +``` + +### Creating a local testnet + +You can create a custom local testnet by creating a network.json file locally and passing the path to the `--network` CLI flag (i.e. `walletd --network="/var/lib/testnet.json"`). An example file is shown below. You can adjust the parameters of your testnet to increase mining speed and test hardfork activations. + +```json +{ + "network": { + "name": "zen", + "initialCoinbase": "300000000000000000000000000000", + "minimumCoinbase": "30000000000000000000000000000", + "initialTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "blockInterval": 600000000000, + "maturityDelay": 144, + "hardforkDevAddr": { + "height": 1, + "oldAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "newAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkTax": { + "height": 2 + }, + "hardforkStorageProof": { + "height": 5 + }, + "hardforkOak": { + "height": 10, + "fixHeight": 12, + "genesisTimestamp": "2023-01-13T00:53:20-08:00" + }, + "hardforkASIC": { + "height": 20, + "oakTime": 10000000000000, + "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000" + }, + "hardforkFoundation": { + "height": 30, + "primaryAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "failsafeAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkV2": { + "allowHeight": 112000, + "requireHeight": 114000 + } + }, + "genesis": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "2023-01-13T00:53:20-08:00", + "minerPayouts": null, + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] + } +} ``` \ No newline at end of file diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index c34fcc3..addb5d3 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -218,7 +218,7 @@ func main() { rootCmd.BoolVar(&cfg.KeyStore.Enabled, "keystore", cfg.KeyStore.Enabled, "enables the keystore") rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") - rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") + rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to; must be one of 'mainnet', 'zen', 'anagami', or the path to a custom network file for a local testnet") rootCmd.BoolVar(&cfg.Syncer.EnableUPnP, "upnp", cfg.Syncer.EnableUPnP, "attempt to forward ports and discover IP with UPnP") rootCmd.BoolVar(&cfg.Syncer.Bootstrap, "bootstrap", cfg.Syncer.Bootstrap, "attempt to bootstrap the network") diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index c2ff152..087561d 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -27,7 +27,6 @@ import ( "go.sia.tech/walletd/v2/wallet" "go.sia.tech/web/walletd" "go.uber.org/zap" - "gopkg.in/yaml.v3" "lukechampine.com/upnp" ) @@ -111,19 +110,8 @@ func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { Genesis types.Block `json:"genesis" yaml:"genesis"` } - switch filepath.Ext(fp) { - case ".yml", ".yaml": - dec := yaml.NewDecoder(f) - dec.KnownFields(true) - if err := dec.Decode(&network); err != nil { - return nil, types.Block{}, fmt.Errorf("failed to decode YAML network file: %w", err) - } - case ".json": - if err := json.NewDecoder(f).Decode(&network); err != nil { - return nil, types.Block{}, fmt.Errorf("failed to decode JSON network file: %w", err) - } - default: - return nil, types.Block{}, errors.New("unknown network file format") + if err := json.NewDecoder(f).Decode(&network); err != nil { + return nil, types.Block{}, fmt.Errorf("failed to decode JSON network file: %w", err) } return &network.Network, network.Genesis, nil } From 84e748d3d6d483d49d228c70df9488baaa7590f9 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 27 Mar 2025 18:45:51 -0700 Subject: [PATCH 391/630] update changeset --- .changeset/support_custom_networks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/support_custom_networks.md b/.changeset/support_custom_networks.md index 897c9a1..92bac20 100644 --- a/.changeset/support_custom_networks.md +++ b/.changeset/support_custom_networks.md @@ -4,7 +4,7 @@ default: minor # Add support for custom networks -Adds support for loading custom network parameters from a local file. This makes it easier to setup local testnets for development. A network file can be specified by using a file path for the `--network` CLI flag. The file should be JSON or YAML formatted with the following structure: +Adds support for loading custom network parameters from a local file. This makes it easier to setup local testnets for development. A network file can be specified by using a file path for the `--network` CLI flag. The file should be JSON formatted with the following structure: ```json { From 0cf7b62f6836134eda8025eb4c915681ad0b6f73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Mar 2025 01:54:15 +0000 Subject: [PATCH 392/630] build(deps): bump golang.org/x/net in the go_modules group Bumps the go_modules group with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.34.0 to 0.36.0 - [Commits](https://github.com/golang/net/compare/v0.34.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index fd1a2a7..6bfd035 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module go.sia.tech/walletd/v2 // v2.1.0 go 1.23.1 -toolchain go1.23.2 +toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.24 @@ -34,7 +34,7 @@ require ( golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.34.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect diff --git a/go.sum b/go.sum index 68e30cf..fd74010 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= From bc47fde5d46c10c2f9888c5739ae8862aaa02c4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 16:51:03 +0000 Subject: [PATCH 393/630] build(deps): bump go.sia.tech/core in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/core](https://github.com/SiaFoundation/core). Updates `go.sia.tech/core` from 0.10.5-0.20250317164759-1a1e10e046f0 to 0.10.5 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/commits/v0.10.5) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 6bfd035..5e958ff 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.24 - go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 + go.sia.tech/core v0.10.5 go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.2 @@ -33,10 +33,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.36.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.37.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.22.0 // indirect + golang.org/x/tools v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index fd74010..b5bbc98 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0 h1:7XFNkrIJngVfTIGMuZWz57TXMzJ6w1mbbPdqrjYywr8= -go.sia.tech/core v0.10.5-0.20250317164759-1a1e10e046f0/go.mod h1:i/dfvjZRei6kR2tOLl27PexeYFb/jtCzRsplSBn3Fgc= +go.sia.tech/core v0.10.5 h1:r+lIeViMkKslu7dPhxzX2Nsvo2lyvTsW+5s0fKIbgXc= +go.sia.tech/core v0.10.5/go.mod h1:42VPNZYiAR29qFK2RppyB4DtUYhHl6qS+q00Un4IBqs= go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e h1:/5MZa6nRrq6ghJ+YYKcO5QVOinZWpVDDYgrrxCx3cak= go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e/go.mod h1:Z14ILJqJkTKyEhaoYvCbW6Y61dJG5NSHFoI+yeDNcI8= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= @@ -65,10 +65,10 @@ golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= @@ -79,8 +79,8 @@ golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From de631869b720dc057977d74f1c82ff5fc520b84f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 17:25:42 +0000 Subject: [PATCH 394/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.24 to 1.14.27 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.24...v1.14.27) Updates `golang.org/x/term` from 0.30.0 to 0.31.0 - [Commits](https://github.com/golang/term/compare/v0.30.0...v0.31.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.27 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.31.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 5e958ff..d64a54f 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,13 @@ go 1.23.1 toolchain go1.24.1 require ( - github.com/mattn/go-sqlite3 v1.14.24 + github.com/mattn/go-sqlite3 v1.14.27 go.sia.tech/core v0.10.5 go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 - golang.org/x/term v0.30.0 + golang.org/x/term v0.31.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -36,7 +36,7 @@ require ( golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.37.0 // indirect golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect + golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index b5bbc98..434e6a5 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU= +github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -71,10 +71,10 @@ golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= From 8942a3c7bb71a736f88323a9440a426fafda065a Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 20 Mar 2025 11:43:33 -0700 Subject: [PATCH 395/630] add tpool aware utxo endpoints --- ...s_endpoints_can_now_exclude_tpool_utxos.md | 5 + api/api.go | 14 +++ api/server.go | 22 +++- persist/sqlite/addresses.go | 69 ++++++---- persist/sqlite/sql.go | 38 ++++++ persist/sqlite/wallet.go | 18 +++ wallet/addresses.go | 89 ++++++++++++- wallet/addresses_test.go | 118 ++++++++++++++++++ wallet/manager.go | 4 +- wallet/seed.go | 2 +- wallet/wallet_test.go | 42 +++---- 11 files changed, 365 insertions(+), 56 deletions(-) create mode 100644 .changeset/address_endpoints_can_now_exclude_tpool_utxos.md create mode 100644 wallet/addresses_test.go diff --git a/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md b/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md new file mode 100644 index 0000000..e4d9c65 --- /dev/null +++ b/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Address endpoints can now exclude tpool utxos diff --git a/api/api.go b/api/api.go index 943c6de..dcb1456 100644 --- a/api/api.go +++ b/api/api.go @@ -191,6 +191,20 @@ type SiafundElementsResponse struct { Outputs []types.SiafundElement `json:"outputs"` } +// AddressSiacoinElementsResponse is the response type for any endpoint that returns +// siacoin UTXOs +type AddressSiacoinElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []wallet.UnspentSiacoinElement `json:"outputs"` +} + +// AddressSiafundElementsResponse is the response type for any endpoint that returns +// siafund UTXOs +type AddressSiafundElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []wallet.UnspentSiafundElement `json:"outputs"` +} + // ElementSpentResponse is the response type for /outputs/siacoin/:id/spent and // /outputs/siafund/:id/spent. type ElementSpentResponse struct { diff --git a/api/server.go b/api/server.go index 815a9fd..b81a832 100644 --- a/api/server.go +++ b/api/server.go @@ -111,8 +111,8 @@ type ( AddressBalance(address types.Address) (wallet.Balance, error) AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) AddressUnconfirmedEvents(address types.Address) ([]wallet.Event, error) - AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) - AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + AddressSiacoinOutputs(address types.Address, tpool bool, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, tpool bool, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) Events(eventIDs []types.Hash256) ([]wallet.Event, error) @@ -1188,16 +1188,21 @@ func (s *server) addressesAddrOutputsSCHandler(jc jape.Context) { return } + var useTPool bool + if jc.DecodeForm("tpool", &useTPool) != nil { + return + } + offset, limit := 0, 1000 if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - utxos, basis, err := s.wm.AddressSiacoinOutputs(addr, offset, limit) + utxos, basis, err := s.wm.AddressSiacoinOutputs(addr, useTPool, offset, limit) if jc.Check("couldn't load utxos", err) != nil { return } - jc.Encode(SiacoinElementsResponse{ + jc.Encode(AddressSiacoinElementsResponse{ Basis: basis, Outputs: utxos, }) @@ -1209,16 +1214,21 @@ func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { return } + var useTPool bool + if jc.DecodeForm("tpool", &useTPool) != nil { + return + } + offset, limit := 0, 1000 if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - utxos, basis, err := s.wm.AddressSiafundOutputs(addr, offset, limit) + utxos, basis, err := s.wm.AddressSiafundOutputs(addr, useTPool, offset, limit) if jc.Check("couldn't load utxos", err) != nil { return } - jc.Encode(SiafundElementsResponse{ + jc.Encode(AddressSiafundElementsResponse{ Basis: basis, Outputs: utxos, }) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 4726d09..2cc23eb 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -70,22 +70,40 @@ func (s *Store) AddressEvents(address types.Address, offset, limit int) (events } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { +func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height FROM siacoin_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address=$1 AND se.maturity_height <= $2 AND se.spent_index_id IS NULL - LIMIT $3 OFFSET $4` + WHERE sa.sia_address = ? AND se.maturity_height <= ? AND se.spent_index_id IS NULL` + + params := []any{encode(address), basis.Height} + if len(tpoolSpent) > 0 { + query += ` AND se.ID NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `)` + params = append(params, queryArgsFunc(tpoolSpent, func(v types.SiacoinOutputID) any { + return encode(v) + })...) + } + + query += ` ORDER BY se.maturity_height DESC, se.id DESC + LIMIT ? OFFSET ?` - rows, err := tx.Query(query, encode(address), index.Height, limit, offset) + params = append(params, limit, offset) + + rows, err := tx.Query(query, params...) if err != nil { return err } defer rows.Close() for rows.Next() { - siacoin, err := scanSiacoinElement(rows) + siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } @@ -110,33 +128,45 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIn siacoins[i].StateElement.MerkleProof = proof } } - - basis, err = getScanBasis(tx) - if err != nil { - return fmt.Errorf("failed to get basis: %w", err) - } return nil }) return } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { +func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + query := `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height FROM siafund_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address = $1 AND se.spent_index_id IS NULL - LIMIT $2 OFFSET $3` + WHERE sa.sia_address=? AND se.spent_index_id IS NULL` + + params := []any{encode(address)} + + if len(tpoolSpent) > 0 { + query += ` AND se.id NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `) + LIMIT ? OFFSET ?` + params = append(params, queryArgsFunc(tpoolSpent, func(v types.SiafundOutputID) any { + return encode(v) + })...) + } - rows, err := tx.Query(query, encode(address), limit, offset) + params = append(params, limit, offset) + + rows, err := tx.Query(query, params...) if err != nil { return err } defer rows.Close() for rows.Next() { - siafund, err := scanSiafundElement(rows) + siafund, err := scanUnspentSiafundElement(rows, basis.Height) if err != nil { return fmt.Errorf("failed to scan siafund element: %w", err) } @@ -160,11 +190,6 @@ func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) siafunds[i].StateElement.MerkleProof = proof } } - - basis, err = getScanBasis(tx) - if err != nil { - return fmt.Errorf("failed to get basis: %w", err) - } return nil }) return diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index 2ea2424..e5b3aac 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "math/rand" + "strings" "time" _ "github.com/mattn/go-sqlite3" // import sqlite3 driver @@ -188,3 +189,40 @@ func setDBVersion(tx *txn, version int64) error { func jitterSleep(t time.Duration) { time.Sleep(t + time.Duration(rand.Int63n(int64(t/2)))) } + +func queryPlaceHolders(n int) string { + if n == 0 { + return "" + } else if n == 1 { + return "?" + } + var b strings.Builder + b.Grow(((n - 1) * 2) + 1) // ?,? + for i := 0; i < n-1; i++ { + b.WriteString("?,") + } + b.WriteString("?") + return b.String() +} + +func queryArgs[T any](args []T) []any { + if len(args) == 0 { + return nil + } + out := make([]any, len(args)) + for i, arg := range args { + out[i] = arg + } + return out +} + +func queryArgsFunc[T any](args []T, fn func(t T) any) []any { + if len(args) == 0 { + return nil + } + out := make([]any, len(args)) + for i, arg := range args { + out[i] = fn(arg) + } + return out +} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index e800d23..35fe994 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -608,6 +608,24 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti return } +func scanUnspentSiacoinElement(s scanner, basisHeight uint64) (se wallet.UnspentSiacoinElement, err error) { + var confirmationHeight uint64 + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.StateElement.MerkleProof), &se.StateElement.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address), &confirmationHeight) + if confirmationHeight <= basisHeight { + se.Confirmations = 1 + basisHeight - confirmationHeight + } + return +} + +func scanUnspentSiafundElement(s scanner, basisHeight uint64) (se wallet.UnspentSiafundElement, err error) { + var confirmationHeight uint64 + err = s.Scan(decode(&se.ID), &se.StateElement.LeafIndex, decode(&se.StateElement.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address), &confirmationHeight) + if confirmationHeight <= basisHeight { + se.Confirmations = 1 + basisHeight - confirmationHeight + } + return +} + func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.StateElement.MerkleProof), &se.StateElement.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) return diff --git a/wallet/addresses.go b/wallet/addresses.go index e14dae2..f1b2702 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -12,13 +12,94 @@ func (m *Manager) AddressBalance(address types.Address) (balance Balance, err er } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { - return m.store.AddressSiacoinOutputs(address, m.chain.Tip(), offset, limit) +func (m *Manager) AddressSiacoinOutputs(address types.Address, excludePool bool, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) { + if !excludePool { + return m.store.AddressSiacoinOutputs(address, nil, offset, limit) + } + created := make(map[types.SiacoinOutputID]types.SiacoinElement) + var spent []types.SiacoinOutputID + for _, txn := range m.chain.PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + if sci.UnlockConditions.UnlockHash() != address { + continue + } + + delete(created, sci.ParentID) + spent = append(spent, sci.ParentID) + } + + for i, sco := range txn.SiacoinOutputs { + if sco.Address != address { + continue + } + + outputID := txn.SiacoinOutputID(i) + sce := types.SiacoinElement{ + ID: outputID, + StateElement: types.StateElement{ + LeafIndex: types.UnassignedLeafIndex, + }, + SiacoinOutput: sco, + } + created[sce.ID] = sce + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + if sci.Parent.SiacoinOutput.Address == address { + spent = append(spent, sci.Parent.ID) + } + + delete(created, sci.Parent.ID) + spent = append(spent, sci.Parent.ID) + } + + for i, sco := range txn.SiacoinOutputs { + if sco.Address != address { + continue + } + + sce := txn.EphemeralSiacoinOutput(i) + created[sce.ID] = sce + } + } + + outputs, basis, err := m.store.AddressSiacoinOutputs(address, spent, offset, limit) + if err != nil { + return nil, types.ChainIndex{}, err + } else if len(outputs) == limit { + return outputs, basis, nil + } + for _, sce := range created { + outputs = append(outputs, UnspentSiacoinElement{ + SiacoinElement: sce, + }) + } + return outputs, basis, nil } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { - return m.store.AddressSiafundOutputs(address, offset, limit) +func (m *Manager) AddressSiafundOutputs(address types.Address, excludePool bool, offset, limit int) (outputs []UnspentSiafundElement, basis types.ChainIndex, err error) { + if !excludePool { + return m.store.AddressSiafundOutputs(address, nil, offset, limit) + } + + var spent []types.SiafundOutputID + for _, txn := range m.chain.PoolTransactions() { + for _, input := range txn.SiafundInputs { + if input.UnlockConditions.UnlockHash() == address { + spent = append(spent, input.ParentID) + } + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, input := range txn.SiafundInputs { + if input.Parent.SiafundOutput.Address == address { + spent = append(spent, input.Parent.ID) + } + } + } + return m.store.AddressSiafundOutputs(address, spent, offset, limit) } // AddressEvents returns the events of a single address. diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go new file mode 100644 index 0000000..2637517 --- /dev/null +++ b/wallet/addresses_test.go @@ -0,0 +1,118 @@ +package wallet_test + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/v2/persist/sqlite" + "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap/zaptest" +) + +func TestAddressUseTpool(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr1 := uc.UnlockHash() + + network, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: addr1, Value: types.Siacoins(100)}, + } + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + mineAndSync(t, cm, db, types.VoidAddress, 1) + + assertSiacoinElement := func(t *testing.T, id types.SiacoinOutputID, value types.Currency, confirmations uint64) { + t.Helper() + + utxos, _, err := wm.AddressSiacoinOutputs(addr1, true, 0, 1) + if err != nil { + t.Fatal(err) + } + for _, sce := range utxos { + if sce.ID == id { + if !sce.SiacoinOutput.Value.Equals(value) { + t.Fatalf("expected value %v, got %v", value, sce.SiacoinOutput.Value) + } else if sce.Confirmations != confirmations { + t.Fatalf("expected confirmations %d, got %d", confirmations, sce.Confirmations) + } + return + } + } + t.Fatalf("expected siacoin element with ID %q not found", id) + } + + airdropID := genesisBlock.Transactions[0].SiacoinOutputID(0) + assertSiacoinElement(t, airdropID, types.Siacoins(100), 2) + + utxos, basis, err := wm.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } + + cs := cm.TipState() + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: types.Siacoins(25), + }, + { + Address: addr1, + Value: types.Siacoins(75), + }, + }, + } + sigHash := cs.InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + pk.SignHash(sigHash), + } + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + + assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) + mineAndSync(t, cm, db, types.VoidAddress, 1) + assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) +} diff --git a/wallet/manager.go b/wallet/manager.go index 7021b52..fb1a236 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -84,8 +84,8 @@ type ( AddressBalance(address types.Address) (balance Balance, err error) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) - AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) - AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) Events(eventIDs []types.Hash256) ([]Event, error) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) diff --git a/wallet/seed.go b/wallet/seed.go index 582cd44..ecc4a65 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -38,7 +38,7 @@ func NewSeed() Seed { return NewSeedFromEntropy(&entropy) } -// NewSeedFromEntropy returns the specified seed. +// NewSeedFromEntropy returns a the specified seed. func NewSeedFromEntropy(entropy *[32]byte) Seed { return Seed{entropy} } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 5bd047f..cef34f6 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1654,14 +1654,14 @@ func TestFullIndex(t *testing.T) { assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) // send half siacoins to the second address - utxos, _, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, _, err := wm.AddressSiacoinOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } for _, se := range utxos { if sce, err := wm.SiacoinElement(types.SiacoinOutputID(se.ID)); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(sce, se) { + } else if !reflect.DeepEqual(sce, se.SiacoinElement) { t.Fatalf("expected %v, got %v", se, sce) } } @@ -1670,7 +1670,7 @@ func TestFullIndex(t *testing.T) { txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { - Parent: utxos[0], + Parent: utxos[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{ Type: policy, @@ -1711,7 +1711,7 @@ func TestFullIndex(t *testing.T) { t.Fatalf("expected transaction event, got %v", events[0].Type) } - sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) + sf, _, err := wm.AddressSiafundOutputs(addr2, false, 0, 100) if err != nil { t.Fatal(err) } @@ -1719,7 +1719,7 @@ func TestFullIndex(t *testing.T) { for _, se := range sf { if sfe, err := wm.SiafundElement(types.SiafundOutputID(se.ID)); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(sfe, se) { + } else if !reflect.DeepEqual(sfe, se.SiafundElement) { t.Fatalf("expected %v, got %v", se, sfe) } } @@ -1729,7 +1729,7 @@ func TestFullIndex(t *testing.T) { txn = types.V2Transaction{ SiafundInputs: []types.V2SiafundInput{ { - Parent: sf[0], + Parent: sf[0].SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{ Type: policy, @@ -1888,7 +1888,7 @@ func TestEvents(t *testing.T) { assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) // send half siacoins to the second address - utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, basis, err := wm.AddressSiacoinOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } else if basis != cm.Tip() { @@ -1899,7 +1899,7 @@ func TestEvents(t *testing.T) { txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { - Parent: utxos[0], + Parent: utxos[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{ Type: policy, @@ -1959,7 +1959,7 @@ func TestEvents(t *testing.T) { t.Fatalf("expected event %v to match %v", expected, events2[0]) } - sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) + sf, _, err := wm.AddressSiafundOutputs(addr2, false, 0, 100) if err != nil { t.Fatal(err) } @@ -1969,7 +1969,7 @@ func TestEvents(t *testing.T) { txn = types.V2Transaction{ SiafundInputs: []types.V2SiafundInput{ { - Parent: sf[0], + Parent: sf[0].SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{ Type: policy, @@ -2689,7 +2689,7 @@ func TestScanV2(t *testing.T) { t.Fatal(err) } - utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + utxos, basis, err := wm.AddressSiacoinOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } else if basis != cm.Tip() { @@ -2701,7 +2701,7 @@ func TestScanV2(t *testing.T) { policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{Type: policy}, }, @@ -3311,10 +3311,10 @@ func TestEventTypes(t *testing.T) { } defer wm.Close() - spendableSiacoinUTXOs := func(t *testing.T) ([]types.SiacoinElement, types.ChainIndex) { + spendableSiacoinUTXOs := func(t *testing.T) ([]wallet.UnspentSiacoinElement, types.ChainIndex) { t.Helper() - sces, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + sces, basis, err := wm.AddressSiacoinOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } else if basis != cm.Tip() { @@ -3477,7 +3477,7 @@ func TestEventTypes(t *testing.T) { txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sce[0], + Parent: sce[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, @@ -3536,7 +3536,7 @@ func TestEventTypes(t *testing.T) { FileContracts: []types.V2FileContract{fc}, SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sce[0], + Parent: sce[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, @@ -3624,7 +3624,7 @@ func TestEventTypes(t *testing.T) { FileContracts: []types.V2FileContract{fc}, SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sce[0], + Parent: sce[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, @@ -3717,7 +3717,7 @@ func TestEventTypes(t *testing.T) { FileContracts: []types.V2FileContract{fc}, SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sces[0], + Parent: sces[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, @@ -3781,7 +3781,7 @@ func TestEventTypes(t *testing.T) { resolutionTxn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sces[0], + Parent: sces[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, @@ -3809,7 +3809,7 @@ func TestEventTypes(t *testing.T) { }) t.Run("siafund claim", func(t *testing.T) { - sfe, basis, err := wm.AddressSiafundOutputs(addr, 0, 100) + sfe, basis, err := wm.AddressSiafundOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } else if basis != cm.Tip() { @@ -3824,7 +3824,7 @@ func TestEventTypes(t *testing.T) { txn := types.V2Transaction{ SiafundInputs: []types.V2SiafundInput{ { - Parent: sfe[0], + Parent: sfe[0].SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: policy, }, From b908c11252527e25f6a8a96d8bfb2ae812462829 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 1 Apr 2025 11:15:24 -0700 Subject: [PATCH 396/630] api: add test --- api/api.go | 14 ++++++++ api/api_test.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++--- api/client.go | 10 +++--- 3 files changed, 109 insertions(+), 9 deletions(-) diff --git a/api/api.go b/api/api.go index dcb1456..d125a5d 100644 --- a/api/api.go +++ b/api/api.go @@ -191,6 +191,20 @@ type SiafundElementsResponse struct { Outputs []types.SiafundElement `json:"outputs"` } +// UnspentSiacoinElementsResponse is the response type for any endpoint that returns +// siacoin UTXOs +type UnspentSiacoinElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []wallet.UnspentSiacoinElement `json:"outputs"` +} + +// UnspentSiafundElementsResponse is the response type for any endpoint that returns +// siafund UTXOs +type UnspentSiafundElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []wallet.UnspentSiafundElement `json:"outputs"` +} + // AddressSiacoinElementsResponse is the response type for any endpoint that returns // siacoin UTXOs type AddressSiacoinElementsResponse struct { diff --git a/api/api_test.go b/api/api_test.go index dd18f04..1666dcc 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -452,7 +452,7 @@ func TestAddresses(t *testing.T) { t.Fatal("transaction should appear in history") } - outputs, basis, err := c.AddressSiacoinOutputs(addr, 0, 100) + outputs, basis, err := c.AddressSiacoinOutputs(addr, false, 0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { @@ -1099,7 +1099,7 @@ func TestSpentElement(t *testing.T) { // trigger initial scan cn.MineBlocks(t, types.VoidAddress, 1) - sce, basis, err := c.AddressSiacoinOutputs(senderAddr, 0, 100) + sce, basis, err := c.AddressSiacoinOutputs(senderAddr, false, 0, 100) if err != nil { t.Fatal(err) } else if len(sce) != 1 { @@ -1120,7 +1120,7 @@ func TestSpentElement(t *testing.T) { txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{ { - Parent: sce[0], + Parent: sce[0].SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: senderPolicy, }, @@ -1187,7 +1187,7 @@ func TestSpentElement(t *testing.T) { txn = types.V2Transaction{ SiafundInputs: []types.V2SiafundInput{ { - Parent: sfe[0], + Parent: sfe[0].SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: senderPolicy, }, @@ -1479,3 +1479,89 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } cn.MineBlocks(t, types.VoidAddress, 1) } + +func TestAddressTPool(t *testing.T) { + log := zaptest.NewLogger(t) + + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + n, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: addr1, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + assertSiacoinElement := func(t *testing.T, id types.SiacoinOutputID, value types.Currency, confirmations uint64) { + t.Helper() + + utxos, _, err := c.AddressSiacoinOutputs(addr1, true, 0, 1) + if err != nil { + t.Fatal(err) + } + for _, sce := range utxos { + if sce.ID == id { + if !sce.SiacoinOutput.Value.Equals(value) { + t.Fatalf("expected value %v, got %v", value, sce.SiacoinOutput.Value) + } else if sce.Confirmations != confirmations { + t.Fatalf("expected confirmations %d, got %d", confirmations, sce.Confirmations) + } + return + } + } + t.Fatalf("expected siacoin element with ID %q not found", id) + } + + cn.MineBlocks(t, types.VoidAddress, 1) + + airdropID := genesisBlock.Transactions[0].SiacoinOutputID(0) + assertSiacoinElement(t, airdropID, types.Siacoins(100), 2) + + utxos, basis, err := c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: types.Siacoins(25), + }, + { + Address: addr1, + Value: types.Siacoins(75), + }, + }, + } + sigHash := cs.InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + pk.SignHash(sigHash), + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + + assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) + cn.MineBlocks(t, types.VoidAddress, 1) + assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) +} diff --git a/api/client.go b/api/client.go index 5b2c9ce..c1af5d6 100644 --- a/api/client.go +++ b/api/client.go @@ -244,15 +244,15 @@ func (c *Client) AddressUnconfirmedEvents(addr types.Address) (resp []wallet.Eve } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { - var resp SiacoinElementsResponse - err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) +func (c *Client) AddressSiacoinOutputs(addr types.Address, useTpool bool, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + var resp UnspentSiacoinElementsResponse + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) return resp.Outputs, resp.Basis, err } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { - var resp SiafundElementsResponse +func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + var resp UnspentSiafundElementsResponse err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) return resp.Outputs, resp.Basis, err } From e197f95b7fcf8a0dc5d8f781b8362aa00df2dfe1 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 1 Apr 2025 11:28:01 -0700 Subject: [PATCH 397/630] add debug to config --- cmd/walletd/main.go | 5 ++--- cmd/walletd/node.go | 4 ++-- config/config.go | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index addb5d3..923a18d 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -207,11 +207,10 @@ func main() { var minerAddrStr string var minerBlocks int - var enableDebug bool rootCmd := flagg.Root rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) - rootCmd.BoolVar(&enableDebug, "debug", false, "enable debug mode with additional profiling and mining endpoints") + rootCmd.BoolVar(&cfg.Debug, "debug", cfg.Debug, "enable debug mode with additional profiling and mining endpoints") rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") rootCmd.BoolVar(&cfg.HTTP.PublicEndpoints, "http.public", cfg.HTTP.PublicEndpoints, "disables auth on endpoints that should be publicly accessible when running walletd as a service") @@ -325,7 +324,7 @@ func main() { // redirect stdlib log to zap zap.RedirectStdLog(log.Named("stdlib")) - checkFatalError("failed to run node", runNode(ctx, cfg, log, enableDebug)) + checkFatalError("failed to run node", runNode(ctx, cfg, log)) case versionCmd: if len(cmd.Args()) != 0 { cmd.Usage() diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 087561d..28af026 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -116,7 +116,7 @@ func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { return &network.Network, network.Genesis, nil } -func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebug bool) error { +func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -234,7 +234,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebu api.WithPublicEndpoints(cfg.HTTP.PublicEndpoints), api.WithBasicAuth(cfg.HTTP.Password), } - if enableDebug { + if cfg.Debug { apiOpts = append(apiOpts, api.WithDebug()) } api := api.NewServer(cm, s, wm, apiOpts...) diff --git a/config/config.go b/config/config.go index 2e95998..733b3b3 100644 --- a/config/config.go +++ b/config/config.go @@ -71,6 +71,7 @@ type ( Name string `yaml:"name,omitempty"` Directory string `yaml:"directory,omitempty"` AutoOpenWebUI bool `yaml:"autoOpenWebUI,omitempty"` + Debug bool `yaml:"debug,omitempty"` HTTP HTTP `yaml:"http,omitempty"` Consensus Consensus `yaml:"consensus,omitempty"` From fc9c70b83368ae1fd51482023b5c6a0674d742c6 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 5 Apr 2025 12:39:32 -0700 Subject: [PATCH 398/630] api: make broadcast tpool aware --- ...s_endpoints_can_now_exclude_tpool_utxos.md | 4 +- api/api_test.go | 113 ++++++++++++++++++ api/server.go | 16 ++- 3 files changed, 130 insertions(+), 3 deletions(-) diff --git a/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md b/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md index e4d9c65..7611dd1 100644 --- a/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md +++ b/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md @@ -2,4 +2,6 @@ default: minor --- -# Address endpoints can now exclude tpool utxos +# Address endpoints can now exclude transaction pool utxos + +# Transaction broadcasts can now discover parents already in the transaction pool. diff --git a/api/api_test.go b/api/api_test.go index 1666dcc..b210f2a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1565,3 +1565,116 @@ func TestAddressTPool(t *testing.T) { cn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) } + +func TestEphemeralTransactions(t *testing.T) { + log := zaptest.NewLogger(t) + pk := types.GeneratePrivateKey() + sp := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + addr1 := sp.Address() + + n, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: addr1, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + cn.MineBlocks(t, types.VoidAddress, 1) + + sces, basis, err := c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + + { + Parent: sces[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: types.Siacoins(50), + }, + { + Address: addr1, + Value: types.Siacoins(50), + }, + }, + } + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + sigHash := cs.InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + expectedOutputID := txn.SiacoinOutputID(txn.ID(), 1) + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + + sces, basis, err = c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sces) != 1 { + t.Fatalf("expected 1 siacoin element, got %v", len(sces)) + } else if sces[0].ID != expectedOutputID { + t.Fatalf("expected siacoin element ID %q, got %q", expectedOutputID, sces[0].ID) + } else if sces[0].StateElement.LeafIndex != types.UnassignedLeafIndex { + t.Fatalf("expected siacoin element to have unassigned leaf index, got %v", sces[0].StateElement.LeafIndex) + } + + txn2 := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: sces[0].SiacoinOutput.Value, + }, + }, + } + sigHash = cs.InputSigHash(txn2) + txn2.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // mine a block so the basis is behind + cn.MineBlocks(t, types.VoidAddress, 1) + + sces, _, err = c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sces) != 1 { + t.Fatalf("expected no siacoin elements, got %v", len(sces)) + } else if sces[0].ID != expectedOutputID { + t.Fatalf("expected siacoin element ID %q, got %q", expectedOutputID, sces[0].ID) + } else if sces[0].StateElement.LeafIndex == types.UnassignedLeafIndex { + t.Fatalf("expected siacoin element to have leaf index, got %v", sces[0].StateElement.LeafIndex) + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn2}); err != nil { + t.Fatal(err) + } + + sces, _, err = c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sces) != 0 { + t.Fatalf("expected no siacoin elements, got %v", len(sces)) + } +} diff --git a/api/server.go b/api/server.go index b81a832..5105181 100644 --- a/api/server.go +++ b/api/server.go @@ -70,6 +70,7 @@ type ( AddPoolTransactions(txns []types.Transaction) (bool, error) AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) (bool, error) UnconfirmedParents(txn types.Transaction) []types.Transaction + V2TransactionSet(basis types.ChainIndex, txn types.V2Transaction) (types.ChainIndex, []types.V2Transaction, error) UpdateV2TransactionSet(txns []types.V2Transaction, from types.ChainIndex, to types.ChainIndex) ([]types.V2Transaction, error) } @@ -315,6 +316,10 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { return } if len(tbr.Transactions) != 0 { + if len(tbr.Transactions) == 1 { + tbr.Transactions = append(s.cm.UnconfirmedParents(tbr.Transactions[0]), tbr.Transactions...) + } + _, err := s.cm.AddPoolTransactions(tbr.Transactions) if err != nil { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) @@ -323,8 +328,15 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { s.s.BroadcastTransactionSet(tbr.Transactions) } if len(tbr.V2Transactions) != 0 { - _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions) - if err != nil { + if len(tbr.V2Transactions) == 1 { + var err error + tbr.Basis, tbr.V2Transactions, err = s.cm.V2TransactionSet(tbr.Basis, tbr.V2Transactions[0]) + if jc.Check("couldn't create v2 transaction set", err) != nil { + return + } + } + + if _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions); err != nil { jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return } From c9864d4fcc07b46891220702ec475b12e127a6a4 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 5 Apr 2025 12:43:29 -0700 Subject: [PATCH 399/630] fix typo --- wallet/seed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/seed.go b/wallet/seed.go index ecc4a65..582cd44 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -38,7 +38,7 @@ func NewSeed() Seed { return NewSeedFromEntropy(&entropy) } -// NewSeedFromEntropy returns a the specified seed. +// NewSeedFromEntropy returns the specified seed. func NewSeedFromEntropy(entropy *[32]byte) Seed { return Seed{entropy} } From 6d4ff211db68885368832a464f1f6acd8146818d Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 5 Apr 2025 12:44:46 -0700 Subject: [PATCH 400/630] fix siafund element pagination --- persist/sqlite/addresses.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 2cc23eb..36e665c 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -150,13 +150,15 @@ func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types. params := []any{encode(address)} if len(tpoolSpent) > 0 { - query += ` AND se.id NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `) - LIMIT ? OFFSET ?` + query += ` AND se.id NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `)` params = append(params, queryArgsFunc(tpoolSpent, func(v types.SiafundOutputID) any { return encode(v) })...) } + query += ` ORDER BY se.id DESC + LIMIT ? OFFSET ?` + params = append(params, limit, offset) rows, err := tx.Query(query, params...) From 12e8ceac987c0cc1552ade5e31836e6cec39ab33 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 5 Apr 2025 13:36:45 -0700 Subject: [PATCH 401/630] api: comment --- api/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/server.go b/api/server.go index 5105181..181c1b2 100644 --- a/api/server.go +++ b/api/server.go @@ -317,6 +317,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } if len(tbr.Transactions) != 0 { if len(tbr.Transactions) == 1 { + // if there's only one transaction, best-effort check for parents tbr.Transactions = append(s.cm.UnconfirmedParents(tbr.Transactions[0]), tbr.Transactions...) } @@ -329,6 +330,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } if len(tbr.V2Transactions) != 0 { if len(tbr.V2Transactions) == 1 { + // if there's only one transaction, best-effort check for parents var err error tbr.Basis, tbr.V2Transactions, err = s.cm.V2TransactionSet(tbr.Basis, tbr.V2Transactions[0]) if jc.Check("couldn't create v2 transaction set", err) != nil { From 72c576c6600c5cad9ad46430f73907124775dc09 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 5 Apr 2025 14:08:17 -0700 Subject: [PATCH 402/630] add use tpool to API client --- api/api_test.go | 2 +- api/client.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index b210f2a..7713b9b 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1166,7 +1166,7 @@ func TestSpentElement(t *testing.T) { t.Fatalf("expected error to contain %q, got %q", "not found", err) } - sfe, basis, err := c.AddressSiafundOutputs(senderAddr, 0, 100) + sfe, basis, err := c.AddressSiafundOutputs(senderAddr, false, 0, 100) if err != nil { t.Fatal(err) } else if len(sfe) != 1 { diff --git a/api/client.go b/api/client.go index c1af5d6..f58b9da 100644 --- a/api/client.go +++ b/api/client.go @@ -251,9 +251,9 @@ func (c *Client) AddressSiacoinOutputs(addr types.Address, useTpool bool, offset } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { +func (c *Client) AddressSiafundOutputs(addr types.Address, useTpool bool, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { var resp UnspentSiafundElementsResponse - err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) return resp.Outputs, resp.Basis, err } From 6666819bf7acdfcbbc304a64e2a9469bf38db25b Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 01:45:56 -0700 Subject: [PATCH 403/630] sqlite: address comments --- persist/sqlite/addresses.go | 8 ++------ persist/sqlite/sql.go | 25 +++---------------------- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 36e665c..3d2f30b 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -86,9 +86,7 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types. params := []any{encode(address), basis.Height} if len(tpoolSpent) > 0 { query += ` AND se.ID NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `)` - params = append(params, queryArgsFunc(tpoolSpent, func(v types.SiacoinOutputID) any { - return encode(v) - })...) + params = append(params, encodeSlice(tpoolSpent)...) } query += ` ORDER BY se.maturity_height DESC, se.id DESC @@ -151,9 +149,7 @@ func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types. if len(tpoolSpent) > 0 { query += ` AND se.id NOT IN (` + queryPlaceHolders(len(tpoolSpent)) + `)` - params = append(params, queryArgsFunc(tpoolSpent, func(v types.SiafundOutputID) any { - return encode(v) - })...) + params = append(params, encodeSlice(tpoolSpent)...) } query += ` ORDER BY se.id DESC diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index e5b3aac..5b54163 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -193,36 +193,17 @@ func jitterSleep(t time.Duration) { func queryPlaceHolders(n int) string { if n == 0 { return "" - } else if n == 1 { - return "?" } - var b strings.Builder - b.Grow(((n - 1) * 2) + 1) // ?,? - for i := 0; i < n-1; i++ { - b.WriteString("?,") - } - b.WriteString("?") - return b.String() -} - -func queryArgs[T any](args []T) []any { - if len(args) == 0 { - return nil - } - out := make([]any, len(args)) - for i, arg := range args { - out[i] = arg - } - return out + return strings.Repeat("?,", n-1) + "?" } -func queryArgsFunc[T any](args []T, fn func(t T) any) []any { +func encodeSlice[T any](args []T) []any { if len(args) == 0 { return nil } out := make([]any, len(args)) for i, arg := range args { - out[i] = fn(arg) + out[i] = encode(arg) } return out } From bea5679e3617520cfa086a83b9d06c61ba8d7fb6 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 14:02:07 -0700 Subject: [PATCH 404/630] exclude -> use --- wallet/addresses.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wallet/addresses.go b/wallet/addresses.go index f1b2702..686db47 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -12,8 +12,8 @@ func (m *Manager) AddressBalance(address types.Address) (balance Balance, err er } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (m *Manager) AddressSiacoinOutputs(address types.Address, excludePool bool, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) { - if !excludePool { +func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) { + if !usePool { return m.store.AddressSiacoinOutputs(address, nil, offset, limit) } created := make(map[types.SiacoinOutputID]types.SiacoinElement) @@ -79,8 +79,8 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, excludePool bool, } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (m *Manager) AddressSiafundOutputs(address types.Address, excludePool bool, offset, limit int) (outputs []UnspentSiafundElement, basis types.ChainIndex, err error) { - if !excludePool { +func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, offset, limit int) (outputs []UnspentSiafundElement, basis types.ChainIndex, err error) { + if !usePool { return m.store.AddressSiafundOutputs(address, nil, offset, limit) } From f524cf9034ca39745b52450677de576018a6219a Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 14:52:09 -0700 Subject: [PATCH 405/630] api, wallet: fix tpool race --- api/api_test.go | 81 +++++++++++++++++++++++++++++++ wallet/addresses.go | 81 ++++++++----------------------- wallet/addresses_test.go | 2 +- wallet/manager.go | 101 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 201 insertions(+), 64 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 7713b9b..5e29792 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -2,6 +2,7 @@ package api_test import ( "bytes" + "context" "encoding/hex" "encoding/json" "fmt" @@ -1678,3 +1679,83 @@ func TestEphemeralTransactions(t *testing.T) { t.Fatalf("expected no siacoin elements, got %v", len(sces)) } } + +func TestBroadcastRace(t *testing.T) { + log := zap.NewNop() + pk := types.GeneratePrivateKey() + sp := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + addr1 := sp.Address() + + n, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100000), + Address: addr1, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + for { + select { + case <-ctx.Done(): + return + default: + cn.MineBlocks(t, types.VoidAddress, 1) + } + } + }() + + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for i := 0; i < 100; i++ { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sces, basis, err := c.AddressSiacoinOutputs(addr1, true, 0, 100) + if err != nil { + panic(err) + } + + burn := types.Siacoins(1) + rem := sces[0].SiacoinOutput.Value.Sub(burn) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: burn, + }, + { + Address: addr1, + Value: rem, + }, + }, + } + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + sigHash := cs.InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + t.Log("broadcasting", txn.ID(), cs.Index) + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + } + } +} diff --git a/wallet/addresses.go b/wallet/addresses.go index 686db47..bce8c93 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -16,53 +16,12 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, off if !usePool { return m.store.AddressSiacoinOutputs(address, nil, offset, limit) } - created := make(map[types.SiacoinOutputID]types.SiacoinElement) - var spent []types.SiacoinOutputID - for _, txn := range m.chain.PoolTransactions() { - for _, sci := range txn.SiacoinInputs { - if sci.UnlockConditions.UnlockHash() != address { - continue - } - - delete(created, sci.ParentID) - spent = append(spent, sci.ParentID) - } - - for i, sco := range txn.SiacoinOutputs { - if sco.Address != address { - continue - } - - outputID := txn.SiacoinOutputID(i) - sce := types.SiacoinElement{ - ID: outputID, - StateElement: types.StateElement{ - LeafIndex: types.UnassignedLeafIndex, - }, - SiacoinOutput: sco, - } - created[sce.ID] = sce - } - } - for _, txn := range m.chain.V2PoolTransactions() { - for _, sci := range txn.SiacoinInputs { - if sci.Parent.SiacoinOutput.Address == address { - spent = append(spent, sci.Parent.ID) - } - - delete(created, sci.Parent.ID) - spent = append(spent, sci.Parent.ID) - } - for i, sco := range txn.SiacoinOutputs { - if sco.Address != address { - continue - } + m.mu.Lock() + defer m.mu.Unlock() - sce := txn.EphemeralSiacoinOutput(i) - created[sce.ID] = sce - } - } + spent := m.poolSCSpent[address] + created := m.poolSCCreated[address] outputs, basis, err := m.store.AddressSiacoinOutputs(address, spent, offset, limit) if err != nil { @@ -79,27 +38,29 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, off } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, offset, limit int) (outputs []UnspentSiafundElement, basis types.ChainIndex, err error) { +func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) { if !usePool { return m.store.AddressSiafundOutputs(address, nil, offset, limit) } - var spent []types.SiafundOutputID - for _, txn := range m.chain.PoolTransactions() { - for _, input := range txn.SiafundInputs { - if input.UnlockConditions.UnlockHash() == address { - spent = append(spent, input.ParentID) - } - } + m.mu.Lock() + defer m.mu.Unlock() + + spent := m.poolSFSpent[address] + created := m.poolSFCreated[address] + + outputs, basis, err := m.store.AddressSiafundOutputs(address, spent, offset, limit) + if err != nil { + return nil, types.ChainIndex{}, err + } else if len(outputs) == limit { + return outputs, basis, nil } - for _, txn := range m.chain.V2PoolTransactions() { - for _, input := range txn.SiafundInputs { - if input.Parent.SiafundOutput.Address == address { - spent = append(spent, input.Parent.ID) - } - } + for _, sfe := range created { + outputs = append(outputs, UnspentSiafundElement{ + SiafundElement: sfe, + }) } - return m.store.AddressSiafundOutputs(address, spent, offset, limit) + return outputs, basis, nil } // AddressEvents returns the events of a single address. diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 2637517..8c6031c 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -7,7 +7,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/v2/internal/testutil" "go.sia.tech/walletd/v2/persist/sqlite" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap/zaptest" diff --git a/wallet/manager.go b/wallet/manager.go index fb1a236..25ee2a2 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -58,7 +58,8 @@ type ( Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) - OnReorg(fn func(types.ChainIndex)) (cancel func()) + OnReorg(func(types.ChainIndex)) (cancel func()) + OnPoolChange(func()) (cancel func()) UpdatesSince(index types.ChainIndex, max int) (rus []chain.RevertUpdate, aus []chain.ApplyUpdate, err error) } @@ -118,8 +119,12 @@ type ( log *zap.Logger tg *threadgroup.ThreadGroup - mu sync.Mutex // protects the fields below - used map[types.Hash256]time.Time + mu sync.Mutex // protects the fields below + used map[types.Hash256]time.Time + poolSCCreated map[types.Address][]types.SiacoinElement + poolSFCreated map[types.Address][]types.SiafundElement + poolSCSpent map[types.Address][]types.SiacoinOutputID + poolSFSpent map[types.Address][]types.SiafundOutputID } ) @@ -616,6 +621,81 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch return nil } +func (m *Manager) resetPool() { + siacoinsCreated := make(map[types.SiacoinOutputID]types.SiacoinElement) + siacoinsSpent := make(map[types.SiacoinOutputID]types.Address) + + siafundsCreated := make(map[types.SiafundOutputID]types.SiafundElement) + siafundsSpent := make(map[types.SiafundOutputID]types.Address) + + for _, txn := range m.chain.PoolTransactions() { + for _, input := range txn.SiacoinInputs { + siacoinsSpent[input.ParentID] = input.UnlockConditions.UnlockHash() + delete(siacoinsCreated, input.ParentID) + } + for i, sco := range txn.SiacoinOutputs { + scoid := txn.SiacoinOutputID(i) + siacoinsCreated[scoid] = types.SiacoinElement{ + ID: scoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiacoinOutput: sco, + } + } + + for _, input := range txn.SiafundInputs { + siafundsSpent[input.ParentID] = input.UnlockConditions.UnlockHash() + delete(siafundsCreated, input.ParentID) + } + for i, sfo := range txn.SiafundOutputs { + sfoid := txn.SiafundOutputID(i) + siafundsCreated[sfoid] = types.SiafundElement{ + ID: sfoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiafundOutput: sfo, + } + } + } + + for _, txn := range m.chain.V2PoolTransactions() { + for _, input := range txn.SiacoinInputs { + siacoinsSpent[input.Parent.ID] = input.Parent.SiacoinOutput.Address + delete(siacoinsCreated, input.Parent.ID) + } + for i := range txn.SiacoinOutputs { + sce := txn.EphemeralSiacoinOutput(i) + siacoinsCreated[sce.ID] = sce + } + + for _, input := range txn.SiafundInputs { + siafundsSpent[input.Parent.ID] = input.Parent.SiafundOutput.Address + delete(siafundsCreated, input.Parent.ID) + } + for i := range txn.SiafundOutputs { + sfe := txn.EphemeralSiafundOutput(i) + siafundsCreated[sfe.ID] = sfe + } + } + + m.poolSCCreated = make(map[types.Address][]types.SiacoinElement) + m.poolSCSpent = make(map[types.Address][]types.SiacoinOutputID) + m.poolSFCreated = make(map[types.Address][]types.SiafundElement) + m.poolSFSpent = make(map[types.Address][]types.SiafundOutputID) + + for _, sce := range siacoinsCreated { + m.poolSCCreated[sce.SiacoinOutput.Address] = append(m.poolSCCreated[sce.SiacoinOutput.Address], sce) + } + for id, addr := range siacoinsSpent { + m.poolSCSpent[addr] = append(m.poolSCSpent[addr], id) + } + + for _, sfe := range siafundsCreated { + m.poolSFCreated[sfe.SiafundOutput.Address] = append(m.poolSFCreated[sfe.SiafundOutput.Address], sfe) + } + for id, addr := range siafundsSpent { + m.poolSFSpent[addr] = append(m.poolSFSpent[addr], id) + } +} + // NewManager creates a new wallet manager. func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { m := &Manager{ @@ -629,6 +709,12 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) tg: threadgroup.New(), used: make(map[types.Hash256]time.Time), + + poolSCSpent: make(map[types.Address][]types.SiacoinOutputID), + poolSCCreated: make(map[types.Address][]types.SiacoinElement), + + poolSFSpent: make(map[types.Address][]types.SiafundOutputID), + poolSFCreated: make(map[types.Address][]types.SiafundElement), } for _, opt := range opts { @@ -653,6 +739,13 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) } }) + unsubscribePool := cm.OnPoolChange(func() { + select { + case reorgChan <- struct{}{}: + default: + } + }) + go func() { ctx, cancel, err := m.tg.AddWithContext(context.Background()) if errors.Is(err, threadgroup.ErrClosed) { @@ -683,6 +776,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) go func() { defer unsubscribe() + defer unsubscribePool() log := m.log.Named("sync") ctx, cancel, err := m.tg.AddWithContext(context.Background()) @@ -724,6 +818,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) panic("failed to sync store: " + err.Error()) } } + m.resetPool() m.mu.Unlock() } }() From f0595b4d05eeeb791ae23b0e8444c2736346f113 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 14:58:21 -0700 Subject: [PATCH 406/630] testutil: fix race --- internal/testutil/testutil.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 0b50e10..b7f0651 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -30,13 +30,18 @@ func (cn *ConsensusNode) WaitForSync(tb testing.TB) { tb.Helper() for i := 0; i < 1000; i++ { - index, err := cn.Store.LastCommittedIndex() - if err != nil { - tb.Fatal(err) - } else if index == cn.Chain.Tip() { + select { + case <-tb.Context().Done(): return + default: + index, err := cn.Store.LastCommittedIndex() + if err != nil { + tb.Fatal(err) + } else if index == cn.Chain.Tip() { + return + } + time.Sleep(10 * time.Millisecond) } - time.Sleep(10 * time.Millisecond) } tb.Fatal("timeout waiting for sync") } @@ -46,8 +51,13 @@ func (cn *ConsensusNode) MineBlocks(tb testing.TB, addr types.Address, n int) { tb.Helper() for i := 0; i < n; i++ { - testutil.MineBlocks(tb, cn.Chain, addr, 1) - cn.WaitForSync(tb) + select { + case <-tb.Context().Done(): + return + default: + testutil.MineBlocks(tb, cn.Chain, addr, 1) + cn.WaitForSync(tb) + } } } From b60e4ddcdaf0ab9273a30c31492781d63a4c5704 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 15:00:56 -0700 Subject: [PATCH 407/630] wallet: comments --- wallet/manager.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wallet/manager.go b/wallet/manager.go index 25ee2a2..d96797c 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -119,8 +119,11 @@ type ( log *zap.Logger tg *threadgroup.ThreadGroup - mu sync.Mutex // protects the fields below - used map[types.Hash256]time.Time + mu sync.Mutex // protects the fields below + used map[types.Hash256]time.Time + // tracks the state of utxos in the transaction pool + // this local state is used to remove a race between + // the wallet indexing and the chain manager poolSCCreated map[types.Address][]types.SiacoinElement poolSFCreated map[types.Address][]types.SiafundElement poolSCSpent map[types.Address][]types.SiacoinOutputID @@ -621,6 +624,11 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch return nil } +// resetPool resets the tracked transaction pool state. This is used when the +// transaction pool is changed such as when a transaction is broadcast or +// when a reorg occurs. +// +// It is expected that the caller holds the manager's lock. func (m *Manager) resetPool() { siacoinsCreated := make(map[types.SiacoinOutputID]types.SiacoinElement) siacoinsSpent := make(map[types.SiacoinOutputID]types.Address) @@ -793,6 +801,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) } m.mu.Lock() + m.resetPool() // update the store lastTip, err := store.LastCommittedIndex() if err != nil { @@ -818,7 +827,6 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) panic("failed to sync store: " + err.Error()) } } - m.resetPool() m.mu.Unlock() } }() From 1ee67cde4ef3587d88d91bf8c66752959233a44b Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 15:20:35 -0700 Subject: [PATCH 408/630] fix flaky tests --- wallet/addresses_test.go | 36 +++++++++++------------------------- wallet/wallet_test.go | 4 ++++ 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 8c6031c..a8f5c66 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -1,32 +1,17 @@ package wallet_test import ( - "path/filepath" + "context" "testing" "go.sia.tech/core/types" - "go.sia.tech/coreutils" - "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/v2/internal/testutil" - "go.sia.tech/walletd/v2/persist/sqlite" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap/zaptest" ) func TestAddressUseTpool(t *testing.T) { log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() // mine a single payout to the wallet pk := types.GeneratePrivateKey() @@ -37,12 +22,9 @@ func TestAddressUseTpool(t *testing.T) { genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ {Address: addr1, Value: types.Siacoins(100)}, } - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) + cn := testutil.NewConsensusNode(t, network, genesisBlock, log) + cm := cn.Chain + db := cn.Store wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) if err != nil { @@ -50,7 +32,7 @@ func TestAddressUseTpool(t *testing.T) { } defer wm.Close() - mineAndSync(t, cm, db, types.VoidAddress, 1) + cn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement := func(t *testing.T, id types.SiacoinOutputID, value types.Currency, confirmations uint64) { t.Helper() @@ -111,8 +93,12 @@ func TestAddressUseTpool(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - + tip, err := wm.Tip() + if err != nil { + t.Fatal(err) + } + wm.Scan(context.Background(), tip) // force reindexing of the tpool assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) - mineAndSync(t, cm, db, types.VoidAddress, 1) + cn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index cef34f6..05a566b 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -234,6 +234,8 @@ func TestSelectSiacoins(t *testing.T) { }) if err != nil { t.Fatal(err) + } else if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) } mineAndSync := func(t *testing.T, addr types.Address, n int) { @@ -402,6 +404,8 @@ func TestSelectSiafunds(t *testing.T) { }) if err != nil { t.Fatal(err) + } else if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) } mineAndSync := func(t *testing.T, addr types.Address, n int) { From 30154e0254a2c5a12ea2e9c817edc8b426dc67c2 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 15:37:07 -0700 Subject: [PATCH 409/630] wallet: use local pool state for selecting elements --- wallet/addresses.go | 39 ++++++---- wallet/manager.go | 176 +++++++++++--------------------------------- 2 files changed, 68 insertions(+), 147 deletions(-) diff --git a/wallet/addresses.go b/wallet/addresses.go index bce8c93..8a595c3 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -20,8 +20,17 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, off m.mu.Lock() defer m.mu.Unlock() - spent := m.poolSCSpent[address] - created := m.poolSCCreated[address] + spent := m.poolAddressSCSpent[address] + var created []UnspentSiacoinElement + for _, sce := range m.poolSCCreated { + if sce.SiacoinOutput.Address != address { + continue + } + + created = append(created, UnspentSiacoinElement{ + SiacoinElement: sce, + }) + } outputs, basis, err := m.store.AddressSiacoinOutputs(address, spent, offset, limit) if err != nil { @@ -29,12 +38,7 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, off } else if len(outputs) == limit { return outputs, basis, nil } - for _, sce := range created { - outputs = append(outputs, UnspentSiacoinElement{ - SiacoinElement: sce, - }) - } - return outputs, basis, nil + return append(outputs, created...), basis, nil } // AddressSiafundOutputs returns the unspent siafund outputs for an address. @@ -46,8 +50,16 @@ func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, off m.mu.Lock() defer m.mu.Unlock() - spent := m.poolSFSpent[address] - created := m.poolSFCreated[address] + spent := m.poolAddressSFSpent[address] + var created []UnspentSiafundElement + for _, sfe := range m.poolSFCreated { + if sfe.SiafundOutput.Address != address { + continue + } + created = append(created, UnspentSiafundElement{ + SiafundElement: sfe, + }) + } outputs, basis, err := m.store.AddressSiafundOutputs(address, spent, offset, limit) if err != nil { @@ -55,12 +67,7 @@ func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, off } else if len(outputs) == limit { return outputs, basis, nil } - for _, sfe := range created { - outputs = append(outputs, UnspentSiafundElement{ - SiafundElement: sfe, - }) - } - return outputs, basis, nil + return append(outputs, created...), basis, nil } // AddressEvents returns the events of a single address. diff --git a/wallet/manager.go b/wallet/manager.go index d96797c..84d4d5e 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -124,10 +124,12 @@ type ( // tracks the state of utxos in the transaction pool // this local state is used to remove a race between // the wallet indexing and the chain manager - poolSCCreated map[types.Address][]types.SiacoinElement - poolSFCreated map[types.Address][]types.SiafundElement - poolSCSpent map[types.Address][]types.SiacoinOutputID - poolSFSpent map[types.Address][]types.SiafundOutputID + poolSCCreated map[types.SiacoinOutputID]types.SiacoinElement + poolSFCreated map[types.SiafundOutputID]types.SiafundElement + poolSCSpent map[types.SiacoinOutputID]bool + poolSFSpent map[types.SiafundOutputID]bool + poolAddressSCSpent map[types.Address][]types.SiacoinOutputID + poolAddressSFSpent map[types.Address][]types.SiafundOutputID } ) @@ -343,42 +345,17 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU return true, nil } - ephemeral := make(map[types.SiacoinOutputID]types.SiacoinElement) - inPool := make(map[types.SiacoinOutputID]bool) - for _, txn := range m.chain.PoolTransactions() { - for _, sci := range txn.SiacoinInputs { - inPool[sci.ParentID] = true - delete(ephemeral, sci.ParentID) - } - for i, sco := range txn.SiacoinOutputs { - exists, err := relevantAddr(sco.Address) - if err != nil { - return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) - } else if exists { - scoid := txn.SiacoinOutputID(i) - ephemeral[scoid] = types.SiacoinElement{ - ID: scoid, - StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, - SiacoinOutput: sco, - } - } - } - } - for _, txn := range m.chain.V2PoolTransactions() { - for _, sci := range txn.SiacoinInputs { - inPool[sci.Parent.ID] = true - delete(ephemeral, sci.Parent.ID) - } - for i, sco := range txn.SiacoinOutputs { - exists, err := relevantAddr(sco.Address) - if err != nil { - return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) - } else if exists { - sce := txn.EphemeralSiacoinOutput(i) - ephemeral[sce.ID] = sce - } + var ephemeral []types.SiacoinElement + for _, sce := range m.poolSCCreated { + exists, err := relevantAddr(sce.SiacoinOutput.Address) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sce.SiacoinOutput.Address, err) + } else if !exists { + continue } + ephemeral = append(ephemeral, sce) } + inPool := m.poolSCSpent var inputSum types.Currency var selected []types.SiacoinElement @@ -453,59 +430,6 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.Sia return nil, m.chain.Tip(), 0, nil } - knownAddresses := make(map[types.Address]bool) - relevantAddr := func(addr types.Address) (bool, error) { - if exists, ok := knownAddresses[addr]; ok { - return exists, nil - } - _, err := m.store.WalletAddress(walletID, addr) - if errors.Is(err, ErrNotFound) { - knownAddresses[addr] = false - return false, nil - } else if err != nil { - return false, err - } - knownAddresses[addr] = true - return true, nil - } - - ephemeral := make(map[types.SiafundOutputID]types.SiafundElement) - inPool := make(map[types.SiafundOutputID]bool) - for _, txn := range m.chain.PoolTransactions() { - for _, sfi := range txn.SiafundInputs { - inPool[sfi.ParentID] = true - delete(ephemeral, sfi.ParentID) - } - for i, sfo := range txn.SiafundOutputs { - exists, err := relevantAddr(sfo.Address) - if err != nil { - return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) - } else if exists { - sfoid := txn.SiafundOutputID(i) - ephemeral[sfoid] = types.SiafundElement{ - ID: sfoid, - StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, - SiafundOutput: sfo, - } - } - } - } - for _, txn := range m.chain.V2PoolTransactions() { - for _, sfi := range txn.SiafundInputs { - inPool[sfi.Parent.ID] = true - delete(ephemeral, sfi.Parent.ID) - } - for i, sfo := range txn.SiafundOutputs { - exists, err := relevantAddr(sfo.Address) - if err != nil { - return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) - } else if exists { - sfe := txn.EphemeralSiafundOutput(i) - ephemeral[sfe.ID] = sfe - } - } - } - var inputSum uint64 var selected []types.SiafundElement var utxoIDs []types.Hash256 @@ -524,7 +448,7 @@ top: } for _, sfe := range utxos { - if inPool[sfe.ID] || m.utxosLocked(types.Hash256(sfe.ID)) != nil { + if m.poolSFSpent[sfe.ID] || m.utxosLocked(types.Hash256(sfe.ID)) != nil { continue } @@ -630,20 +554,24 @@ func syncStore(ctx context.Context, store Store, cm ChainManager, index types.Ch // // It is expected that the caller holds the manager's lock. func (m *Manager) resetPool() { - siacoinsCreated := make(map[types.SiacoinOutputID]types.SiacoinElement) - siacoinsSpent := make(map[types.SiacoinOutputID]types.Address) + m.poolSCCreated = make(map[types.SiacoinOutputID]types.SiacoinElement) + m.poolSCSpent = make(map[types.SiacoinOutputID]bool) - siafundsCreated := make(map[types.SiafundOutputID]types.SiafundElement) - siafundsSpent := make(map[types.SiafundOutputID]types.Address) + m.poolSFCreated = make(map[types.SiafundOutputID]types.SiafundElement) + m.poolSFSpent = make(map[types.SiafundOutputID]bool) + + m.poolAddressSCSpent = make(map[types.Address][]types.SiacoinOutputID) + m.poolAddressSFSpent = make(map[types.Address][]types.SiafundOutputID) for _, txn := range m.chain.PoolTransactions() { for _, input := range txn.SiacoinInputs { - siacoinsSpent[input.ParentID] = input.UnlockConditions.UnlockHash() - delete(siacoinsCreated, input.ParentID) + m.poolSCSpent[input.ParentID] = true + m.poolAddressSCSpent[input.UnlockConditions.UnlockHash()] = append(m.poolAddressSCSpent[input.UnlockConditions.UnlockHash()], input.ParentID) + delete(m.poolSCCreated, input.ParentID) } for i, sco := range txn.SiacoinOutputs { scoid := txn.SiacoinOutputID(i) - siacoinsCreated[scoid] = types.SiacoinElement{ + m.poolSCCreated[scoid] = types.SiacoinElement{ ID: scoid, StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, SiacoinOutput: sco, @@ -651,12 +579,12 @@ func (m *Manager) resetPool() { } for _, input := range txn.SiafundInputs { - siafundsSpent[input.ParentID] = input.UnlockConditions.UnlockHash() - delete(siafundsCreated, input.ParentID) + m.poolSFSpent[input.ParentID] = true + delete(m.poolSFCreated, input.ParentID) } for i, sfo := range txn.SiafundOutputs { sfoid := txn.SiafundOutputID(i) - siafundsCreated[sfoid] = types.SiafundElement{ + m.poolSFCreated[sfoid] = types.SiafundElement{ ID: sfoid, StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, SiafundOutput: sfo, @@ -666,42 +594,25 @@ func (m *Manager) resetPool() { for _, txn := range m.chain.V2PoolTransactions() { for _, input := range txn.SiacoinInputs { - siacoinsSpent[input.Parent.ID] = input.Parent.SiacoinOutput.Address - delete(siacoinsCreated, input.Parent.ID) + m.poolSCSpent[input.Parent.ID] = true + m.poolAddressSCSpent[input.Parent.SiacoinOutput.Address] = append(m.poolAddressSCSpent[input.Parent.SiacoinOutput.Address], input.Parent.ID) + delete(m.poolSCCreated, input.Parent.ID) } for i := range txn.SiacoinOutputs { sce := txn.EphemeralSiacoinOutput(i) - siacoinsCreated[sce.ID] = sce + m.poolSCCreated[sce.ID] = sce } for _, input := range txn.SiafundInputs { - siafundsSpent[input.Parent.ID] = input.Parent.SiafundOutput.Address - delete(siafundsCreated, input.Parent.ID) + m.poolSFSpent[input.Parent.ID] = true + m.poolAddressSFSpent[input.Parent.SiafundOutput.Address] = append(m.poolAddressSFSpent[input.Parent.SiafundOutput.Address], input.Parent.ID) + delete(m.poolSFCreated, input.Parent.ID) } for i := range txn.SiafundOutputs { sfe := txn.EphemeralSiafundOutput(i) - siafundsCreated[sfe.ID] = sfe + m.poolSFCreated[sfe.ID] = sfe } } - - m.poolSCCreated = make(map[types.Address][]types.SiacoinElement) - m.poolSCSpent = make(map[types.Address][]types.SiacoinOutputID) - m.poolSFCreated = make(map[types.Address][]types.SiafundElement) - m.poolSFSpent = make(map[types.Address][]types.SiafundOutputID) - - for _, sce := range siacoinsCreated { - m.poolSCCreated[sce.SiacoinOutput.Address] = append(m.poolSCCreated[sce.SiacoinOutput.Address], sce) - } - for id, addr := range siacoinsSpent { - m.poolSCSpent[addr] = append(m.poolSCSpent[addr], id) - } - - for _, sfe := range siafundsCreated { - m.poolSFCreated[sfe.SiafundOutput.Address] = append(m.poolSFCreated[sfe.SiafundOutput.Address], sfe) - } - for id, addr := range siafundsSpent { - m.poolSFSpent[addr] = append(m.poolSFSpent[addr], id) - } } // NewManager creates a new wallet manager. @@ -718,11 +629,14 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) used: make(map[types.Hash256]time.Time), - poolSCSpent: make(map[types.Address][]types.SiacoinOutputID), - poolSCCreated: make(map[types.Address][]types.SiacoinElement), + poolSCSpent: make(map[types.SiacoinOutputID]bool), + poolSCCreated: make(map[types.SiacoinOutputID]types.SiacoinElement), + + poolSFSpent: make(map[types.SiafundOutputID]bool), + poolSFCreated: make(map[types.SiafundOutputID]types.SiafundElement), - poolSFSpent: make(map[types.Address][]types.SiafundOutputID), - poolSFCreated: make(map[types.Address][]types.SiafundElement), + poolAddressSCSpent: make(map[types.Address][]types.SiacoinOutputID), + poolAddressSFSpent: make(map[types.Address][]types.SiafundOutputID), } for _, opt := range opts { From f10c57d282d10f5c8b6cb01acc1a37c3d925d0d8 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 7 Apr 2025 15:50:16 -0700 Subject: [PATCH 410/630] wallet: fix flaky test again --- wallet/addresses_test.go | 7 +------ wallet/manager.go | 2 ++ wallet/manager_test.go | 9 +++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 wallet/manager_test.go diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index a8f5c66..6eec13a 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -1,7 +1,6 @@ package wallet_test import ( - "context" "testing" "go.sia.tech/core/types" @@ -93,11 +92,7 @@ func TestAddressUseTpool(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - tip, err := wm.Tip() - if err != nil { - t.Fatal(err) - } - wm.Scan(context.Background(), tip) // force reindexing of the tpool + wm.SyncPool() // force reindexing of the tpool assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) cn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) diff --git a/wallet/manager.go b/wallet/manager.go index 84d4d5e..0c1782b 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -524,6 +524,8 @@ func (m *Manager) Close() error { return nil } +// syncStore syncs the state of the store with the chain manager. The sync will +// complete when the store reaches the current tip or the context is canceled. func syncStore(ctx context.Context, store Store, cm ChainManager, index types.ChainIndex, batchSize int) error { for index != cm.Tip() { select { diff --git a/wallet/manager_test.go b/wallet/manager_test.go new file mode 100644 index 0000000..bf7bd1a --- /dev/null +++ b/wallet/manager_test.go @@ -0,0 +1,9 @@ +package wallet + +// SyncPool forces a sync of the transaction pool for testing +// purposes. +func (m *Manager) SyncPool() { + m.mu.Lock() + defer m.mu.Unlock() + m.resetPool() +} From 79659a9133c7e9a92ef1fa92f6e5538735ef0395 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 9 Apr 2025 12:14:06 -0700 Subject: [PATCH 411/630] update coreutils --- cmd/walletd/node.go | 2 +- go.mod | 14 +++++----- go.sum | 22 +++++++-------- internal/testutil/testutil.go | 2 +- persist/sqlite/consensus_test.go | 4 +-- wallet/wallet_test.go | 46 ++++++++++++++++---------------- 6 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 28af026..c583f5a 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -149,7 +149,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } defer bdb.Close() - dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock) + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } diff --git a/go.mod b/go.mod index d64a54f..10b3972 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.23.1 toolchain go1.24.1 +replace go.sia.tech/coreutils => ../coreutils + require ( github.com/mattn/go-sqlite3 v1.14.27 - go.sia.tech/core v0.10.5 - go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e + go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d + go.sia.tech/coreutils v0.12.2-0.20250409155456-115c8c9fa6b4 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 @@ -24,19 +26,19 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.50.0 // indirect + github.com/quic-go/quic-go v0.50.1 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.4.0 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.36.0 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.31.0 // indirect ) diff --git a/go.sum b/go.sum index 434e6a5..63d42ba 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo= -github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= +github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -41,10 +41,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.5 h1:r+lIeViMkKslu7dPhxzX2Nsvo2lyvTsW+5s0fKIbgXc= -go.sia.tech/core v0.10.5/go.mod h1:42VPNZYiAR29qFK2RppyB4DtUYhHl6qS+q00Un4IBqs= -go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e h1:/5MZa6nRrq6ghJ+YYKcO5QVOinZWpVDDYgrrxCx3cak= -go.sia.tech/coreutils v0.12.2-0.20250317235740-9e6e9fe76b2e/go.mod h1:Z14ILJqJkTKyEhaoYvCbW6Y61dJG5NSHFoI+yeDNcI8= +go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d h1:iXSffVVq7SwisMIdI7FVioSCP+sPANIuPrbpFBGnWBU= +go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d/go.mod h1:AFTwGXQ8VQUyD9qasJTzTKBO9y/jianSqalHrwrzyxA= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -61,22 +59,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index b7f0651..4d8be74 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -69,7 +69,7 @@ func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, } tb.Cleanup(func() { l.Close() }) - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis, nil) if err != nil { tb.Fatal(err) } diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 2f3498e..36adbf8 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -68,7 +68,7 @@ func TestPruneSiacoins(t *testing.T) { addr := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -211,7 +211,7 @@ func TestPruneSiafunds(t *testing.T) { network, genesisBlock := testutil.Network() // send the siafund airdrop to the wallet genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 05a566b..2180e82 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -125,7 +125,7 @@ func TestReserve(t *testing.T) { defer bdb.Close() network, genesisBlock := testutil.V2Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -202,7 +202,7 @@ func TestSelectSiacoins(t *testing.T) { network.InitialCoinbase = types.Siacoins(100) network.MinimumCoinbase = types.Siacoins(100) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -379,7 +379,7 @@ func TestSelectSiafunds(t *testing.T) { network.InitialCoinbase = types.Siacoins(100) network.MinimumCoinbase = types.Siacoins(100) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -516,7 +516,7 @@ func TestReorg(t *testing.T) { network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -724,7 +724,7 @@ func TestEphemeralBalance(t *testing.T) { network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -920,7 +920,7 @@ func TestWalletAddresses(t *testing.T) { network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -1053,7 +1053,7 @@ func TestScan(t *testing.T) { // send the siafunds to the owned address genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -1220,7 +1220,7 @@ func TestSiafunds(t *testing.T) { // send the siafunds to the owned address genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -1377,7 +1377,7 @@ func TestOrphans(t *testing.T) { network.HardforkV2.AllowHeight = 200 network.HardforkV2.RequireHeight = 201 - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -1571,7 +1571,7 @@ func TestFullIndex(t *testing.T) { defer bdb.Close() network, genesisBlock := testV2Network(addr2) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -1795,7 +1795,7 @@ func TestEvents(t *testing.T) { defer bdb.Close() network, genesisBlock := testV2Network(addr2) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2051,7 +2051,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { addr1 := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2239,7 +2239,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { addr1 := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2444,7 +2444,7 @@ func TestV2(t *testing.T) { network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2555,7 +2555,7 @@ func TestScanV2(t *testing.T) { addr := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testV2Network(addr) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2747,7 +2747,7 @@ func TestReorgV2(t *testing.T) { network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -2974,7 +2974,7 @@ func TestOrphansV2(t *testing.T) { defer bdb.Close() network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -3189,7 +3189,7 @@ func TestDeleteWallet(t *testing.T) { network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -3289,7 +3289,7 @@ func TestEventTypes(t *testing.T) { network, genesisBlock := testV2Network(addr) // raise the require height to test v1 events network.HardforkV2.RequireHeight = 250 - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -3876,7 +3876,7 @@ func TestSiafundClaims(t *testing.T) { genesis.Transactions[0].SiafundOutputs[0].Address = addr siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value - store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + store, genesisState, err := chain.NewDBStore(bdb, network, genesis, nil) if err != nil { t.Fatal(err) } @@ -4122,7 +4122,7 @@ func TestV2SiafundClaims(t *testing.T) { genesis.Transactions[0].SiafundOutputs[0].Address = addr siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value - store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + store, genesisState, err := chain.NewDBStore(bdb, network, genesis, nil) if err != nil { t.Fatal(err) } @@ -4356,7 +4356,7 @@ func TestReset(t *testing.T) { } defer bdb.Close() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } @@ -4367,7 +4367,7 @@ func TestReset(t *testing.T) { t.Fatal(err) } defer bdb2.Close() - store2, genesisState2, err := chain.NewDBStore(bdb2, network, genesisBlock) + store2, genesisState2, err := chain.NewDBStore(bdb2, network, genesisBlock, nil) if err != nil { t.Fatal(err) } From 389c31d65503fd34a10993b17d8a7e424f3e8ecc Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 9 Apr 2025 12:39:33 -0700 Subject: [PATCH 412/630] chore: update deps --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 10b3972..4c261bd 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,10 @@ go 1.23.1 toolchain go1.24.1 -replace go.sia.tech/coreutils => ../coreutils - require ( github.com/mattn/go-sqlite3 v1.14.27 go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d - go.sia.tech/coreutils v0.12.2-0.20250409155456-115c8c9fa6b4 + go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5 go.sia.tech/jape v0.12.1 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 63d42ba..b2f3b9f 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d h1:iXSffVVq7SwisMIdI7FVioSCP+sPANIuPrbpFBGnWBU= go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d/go.mod h1:AFTwGXQ8VQUyD9qasJTzTKBO9y/jianSqalHrwrzyxA= +go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5 h1:GzcwIi+Vx3KpxQyTw4kdBOS4s/jarMuUi7C8vApPEIU= +go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5/go.mod h1:UASHkZuV8pezqFtNsEoFP6YrspNHGRiRM+EbgL4cQv0= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 9630bcf4e2dd231b38260832fe5da7fcdd11faea Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 9 Apr 2025 12:45:26 -0700 Subject: [PATCH 413/630] fix test off by one --- api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 5e29792..34a0b01 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -550,8 +550,8 @@ func TestConsensusUpdates(t *testing.T) { t.Fatal(err) } else if len(reverted) != 0 { t.Fatal("expected no reverted blocks") - } else if len(applied) != 11 { // genesis + 10 mined blocks (chain manager off-by-one) - t.Fatalf("expected 11 applied blocks, got %v", len(applied)) + } else if len(applied) != 10 { // genesis + 10 mined blocks + t.Fatalf("expected 10 applied blocks, got %v", len(applied)) } for i, cau := range applied { From 4c38390cbe3ef84d5a0031375cf44373e61580de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:30:26 +0000 Subject: [PATCH 414/630] build(deps): bump golang.org/x/net in the go_modules group Bumps the go_modules group with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/net/compare/v0.37.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4c261bd..ed7a28b 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( golang.org/x/crypto v0.37.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.37.0 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect diff --git a/go.sum b/go.sum index b2f3b9f..05e1096 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= From fdb84def7b5ddcb2dc4eb780ee1535afc640fa03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 17:16:03 +0000 Subject: [PATCH 415/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.27 to 1.14.28 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.27...v1.14.28) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.28 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ed7a28b..00683ee 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.1 toolchain go1.24.1 require ( - github.com/mattn/go-sqlite3 v1.14.27 + github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5 go.sia.tech/jape v0.12.1 diff --git a/go.sum b/go.sum index 05e1096..f15082b 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU= -github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= From 12543f67992f62cf8237b5915d60e5c8f1d8e189 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 07:21:14 +0000 Subject: [PATCH 416/630] chore: prepare release 2.2.0 --- ...s_endpoints_can_now_exclude_tpool_utxos.md | 7 -- .changeset/support_custom_networks.md | 73 ----------------- CHANGELOG.md | 78 +++++++++++++++++++ go.mod | 2 +- 4 files changed, 79 insertions(+), 81 deletions(-) delete mode 100644 .changeset/address_endpoints_can_now_exclude_tpool_utxos.md delete mode 100644 .changeset/support_custom_networks.md diff --git a/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md b/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md deleted file mode 100644 index 7611dd1..0000000 --- a/.changeset/address_endpoints_can_now_exclude_tpool_utxos.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Address endpoints can now exclude transaction pool utxos - -# Transaction broadcasts can now discover parents already in the transaction pool. diff --git a/.changeset/support_custom_networks.md b/.changeset/support_custom_networks.md deleted file mode 100644 index 92bac20..0000000 --- a/.changeset/support_custom_networks.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -default: minor ---- - -# Add support for custom networks - -Adds support for loading custom network parameters from a local file. This makes it easier to setup local testnets for development. A network file can be specified by using a file path for the `--network` CLI flag. The file should be JSON formatted with the following structure: - -```json -{ - "network": { - "name": "zen", - "initialCoinbase": "300000000000000000000000000000", - "minimumCoinbase": "30000000000000000000000000000", - "initialTarget": "0000000100000000000000000000000000000000000000000000000000000000", - "blockInterval": 600000000000, - "maturityDelay": 144, - "hardforkDevAddr": { - "height": 1, - "oldAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", - "newAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" - }, - "hardforkTax": { - "height": 2 - }, - "hardforkStorageProof": { - "height": 5 - }, - "hardforkOak": { - "height": 10, - "fixHeight": 12, - "genesisTimestamp": "2023-01-13T00:53:20-08:00" - }, - "hardforkASIC": { - "height": 20, - "oakTime": 10000000000000, - "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000" - }, - "hardforkFoundation": { - "height": 30, - "primaryAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", - "failsafeAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" - }, - "hardforkV2": { - "allowHeight": 112000, - "requireHeight": 114000 - } - }, - "genesis": { - "parentID": "0000000000000000000000000000000000000000000000000000000000000000", - "nonce": 0, - "timestamp": "2023-01-13T00:53:20-08:00", - "minerPayouts": null, - "transactions": [ - { - "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", - "siacoinOutputs": [ - { - "value": "1000000000000000000000000000000000000", - "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" - } - ], - "siafundOutputs": [ - { - "value": 10000, - "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" - } - ] - } - ] - } -} -``` \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 958be22..6248f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,81 @@ +## 2.2.0 (2025-04-22) + +### Features + +#### Address endpoints can now exclude transaction pool utxos + +## Transaction broadcasts can now discover parents already in the transaction pool. + +#### Add support for custom networks + +Adds support for loading custom network parameters from a local file. This makes it easier to setup local testnets for development. A network file can be specified by using a file path for the `--network` CLI flag. The file should be JSON formatted with the following structure: + +```json +{ + "network": { + "name": "zen", + "initialCoinbase": "300000000000000000000000000000", + "minimumCoinbase": "30000000000000000000000000000", + "initialTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "blockInterval": 600000000000, + "maturityDelay": 144, + "hardforkDevAddr": { + "height": 1, + "oldAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "newAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkTax": { + "height": 2 + }, + "hardforkStorageProof": { + "height": 5 + }, + "hardforkOak": { + "height": 10, + "fixHeight": 12, + "genesisTimestamp": "2023-01-13T00:53:20-08:00" + }, + "hardforkASIC": { + "height": 20, + "oakTime": 10000000000000, + "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000" + }, + "hardforkFoundation": { + "height": 30, + "primaryAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "failsafeAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "hardforkV2": { + "allowHeight": 112000, + "requireHeight": 114000 + } + }, + "genesis": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "2023-01-13T00:53:20-08:00", + "minerPayouts": null, + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] + } +} +``` + ## 2.1.0 (2025-03-25) ### Features diff --git a/go.mod b/go.mod index 00683ee..e54b439 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.1.0 +module go.sia.tech/walletd/v2 // v2.2.0 go 1.23.1 From 2082804a5368700a6b572b05de2f835418ba4762 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 23 Apr 2025 08:05:30 -0700 Subject: [PATCH 417/630] update core and coreutils --- api/api_test.go | 2 +- api/client.go | 87 +++++++++++++++++++++++++------------------------ api/server.go | 20 ++++++------ go.mod | 15 ++++----- go.sum | 34 +++++++++++-------- 5 files changed, 82 insertions(+), 76 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 34a0b01..c07c8cf 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1248,7 +1248,7 @@ func TestDebugMine(t *testing.T) { Password: "password", } - err := jc.POST("/debug/mine", api.DebugMineRequest{ + err := jc.POST(context.Background(), "/debug/mine", api.DebugMineRequest{ Blocks: 5, Address: types.VoidAddress, }, nil) diff --git a/api/client.go b/api/client.go index f58b9da..1e9cd9b 100644 --- a/api/client.go +++ b/api/client.go @@ -1,6 +1,7 @@ package api import ( + "context" "fmt" "sync" "time" @@ -41,13 +42,13 @@ func (c *Client) BaseURL() string { // State returns information about the current state of the walletd daemon. func (c *Client) State() (resp StateResponse, err error) { - err = c.c.GET("/state", &resp) + err = c.c.GET(context.Background(), "/state", &resp) return } // TxpoolBroadcast broadcasts a set of transaction to the network. func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (err error) { - err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{ + err = c.c.POST(context.Background(), "/txpool/broadcast", TxpoolBroadcastRequest{ Basis: basis, Transactions: txns, V2Transactions: v2txns, @@ -58,7 +59,7 @@ func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transactio // TxpoolTransactions returns all transactions in the transaction pool. func (c *Client) TxpoolTransactions() (basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction, err error) { var resp TxpoolTransactionsResponse - err = c.c.GET("/txpool/transactions", &resp) + err = c.c.GET(context.Background(), "/txpool/transactions", &resp) return resp.Basis, resp.Transactions, resp.V2Transactions, err } @@ -70,40 +71,40 @@ func (c *Client) V2UpdateTransactionSetBasis(txnset []types.V2Transaction, from, Transactions: txnset, } var resp TxpoolUpdateV2TransactionsResponse - err := c.c.POST("/txpool/transactions/v2/basis", req, &resp) + err := c.c.POST(context.Background(), "/txpool/transactions/v2/basis", req, &resp) return resp.Basis, resp.Transactions, err } // TxpoolParents returns the parents of a transaction that are currently in the // transaction pool. func (c *Client) TxpoolParents(txn types.Transaction) (resp []types.Transaction, err error) { - err = c.c.POST("/txpool/parents", txn, &resp) + err = c.c.POST(context.Background(), "/txpool/parents", txn, &resp) return } // TxpoolFee returns the recommended fee (per weight unit) to ensure a high // probability of inclusion in the next block. func (c *Client) TxpoolFee() (resp types.Currency, err error) { - err = c.c.GET("/txpool/fee", &resp) + err = c.c.GET(context.Background(), "/txpool/fee", &resp) return } // ConsensusNetwork returns the node's network metadata. func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { resp = new(consensus.Network) - err = c.c.GET("/consensus/network", resp) + err = c.c.GET(context.Background(), "/consensus/network", resp) return } // ConsensusBlocksID returns the block with the given id. func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err error) { - err = c.c.GET(fmt.Sprintf("/consensus/blocks/%v", bid), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/blocks/%v", bid), &resp) return } // ConsensusIndex returns the consensus index at the specified height. func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { - err = c.c.GET(fmt.Sprintf("/consensus/index/%d", height), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/index/%d", height), &resp) return } @@ -117,7 +118,7 @@ func (c *Client) ConsensusUpdates(index types.ChainIndex, limit int) ([]chain.Re } var resp ConsensusUpdatesResponse - if err := c.c.GET(fmt.Sprintf("/consensus/updates/%s?limit=%d", indexBuf, limit), &resp); err != nil { + if err := c.c.GET(context.Background(), fmt.Sprintf("/consensus/updates/%s?limit=%d", indexBuf, limit), &resp); err != nil { return nil, nil, err } @@ -152,7 +153,7 @@ func (c *Client) ConsensusUpdates(index types.ChainIndex, limit int) ([]chain.Re // ConsensusTipState returns the current tip state. func (c *Client) ConsensusTipState() (resp consensus.State, err error) { - if err = c.c.GET("/consensus/tipstate", &resp); err != nil { + if err = c.c.GET(context.Background(), "/consensus/tipstate", &resp); err != nil { return } resp.Network, err = c.getNetwork() @@ -161,50 +162,50 @@ func (c *Client) ConsensusTipState() (resp consensus.State, err error) { // ConsensusTip returns the current tip index. func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { - err = c.c.GET("/consensus/tip", &resp) + err = c.c.GET(context.Background(), "/consensus/tip", &resp) return } // SyncerPeers returns the current peers of the syncer. func (c *Client) SyncerPeers() (resp []GatewayPeer, err error) { - err = c.c.GET("/syncer/peers", &resp) + err = c.c.GET(context.Background(), "/syncer/peers", &resp) return } // SyncerConnect adds the address as a peer of the syncer. func (c *Client) SyncerConnect(addr string) (err error) { - err = c.c.POST("/syncer/connect", addr, nil) + err = c.c.POST(context.Background(), "/syncer/connect", addr, nil) return } // SyncerBroadcastBlock broadcasts a block to all peers. func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { - err = c.c.POST("/syncer/broadcast/block", b, nil) + err = c.c.POST(context.Background(), "/syncer/broadcast/block", b, nil) return } // Wallets returns the set of tracked wallets. func (c *Client) Wallets() (ws []wallet.Wallet, err error) { - err = c.c.GET("/wallets", &ws) + err = c.c.GET(context.Background(), "/wallets", &ws) return } // AddWallet adds a wallet to the set of tracked wallets. func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) { - err = c.c.POST("/wallets", uw, &w) + err = c.c.POST(context.Background(), "/wallets", uw, &w) return } // UpdateWallet updates a wallet. func (c *Client) UpdateWallet(id wallet.ID, uw WalletUpdateRequest) (w wallet.Wallet, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v", id), uw, &w) return } // RemoveWallet deletes a wallet. If the wallet is currently subscribed, it will // be unsubscribed. func (c *Client) RemoveWallet(id wallet.ID) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v", id)) + err = c.c.DELETE(context.Background(), fmt.Sprintf("/wallets/%v", id)) return } @@ -215,65 +216,65 @@ func (c *Client) Wallet(id wallet.ID) *WalletClient { // ScanStatus returns the current state of wallet scanning. func (c *Client) ScanStatus() (resp RescanResponse, err error) { - err = c.c.GET("/rescan", &resp) + err = c.c.GET(context.Background(), "/rescan", &resp) return } // Rescan rescans the blockchain starting from the specified height. func (c *Client) Rescan(height uint64) (err error) { - err = c.c.POST("/rescan", height, nil) + err = c.c.POST(context.Background(), "/rescan", height, nil) return } // AddressBalance returns the balance of a single address. func (c *Client) AddressBalance(addr types.Address) (resp BalanceResponse, err error) { - err = c.c.GET(fmt.Sprintf("/addresses/%v/balance", addr), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%v/balance", addr), &resp) return } // AddressEvents returns the events of a single address. func (c *Client) AddressEvents(addr types.Address, offset, limit int) (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/addresses/%v/events?offset=%d&limit=%d", addr, offset, limit), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%v/events?offset=%d&limit=%d", addr, offset, limit), &resp) return } // AddressUnconfirmedEvents returns the unconfirmed events for a single address. func (c *Client) AddressUnconfirmedEvents(addr types.Address) (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/addresses/%v/events/unconfirmed", addr), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/addresses/%v/events/unconfirmed", addr), &resp) return } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. func (c *Client) AddressSiacoinOutputs(addr types.Address, useTpool bool, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { var resp UnspentSiacoinElementsResponse - err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) + err := c.c.GET(context.Background(), fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) return resp.Outputs, resp.Basis, err } // AddressSiafundOutputs returns the unspent siafund outputs for an address. func (c *Client) AddressSiafundOutputs(addr types.Address, useTpool bool, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { var resp UnspentSiafundElementsResponse - err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) + err := c.c.GET(context.Background(), fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d&tpool=%t", addr, offset, limit, useTpool), &resp) return resp.Outputs, resp.Basis, err } // Event returns the event with the specified ID. func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/events/%v", id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/events/%v", id), &resp) return } // SpentSiacoinElement returns whether a siacoin output has been spent and the // event that spent it. func (c *Client) SpentSiacoinElement(id types.SiacoinOutputID) (resp ElementSpentResponse, err error) { - err = c.c.GET(fmt.Sprintf("/outputs/siacoin/%v/spent", id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/outputs/siacoin/%v/spent", id), &resp) return } // SpentSiafundElement returns whether a siafund output has been spent and the // event that spent it. func (c *Client) SpentSiafundElement(id types.SiafundOutputID) (resp ElementSpentResponse, err error) { - err = c.c.GET(fmt.Sprintf("/outputs/siafund/%v/spent", id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/outputs/siafund/%v/spent", id), &resp) return } @@ -287,57 +288,57 @@ type WalletClient struct { // AddAddress adds the specified address and associated metadata to the // wallet. func (c *WalletClient) AddAddress(a wallet.Address) (err error) { - err = c.c.PUT(fmt.Sprintf("/wallets/%v/addresses", c.id), a) + err = c.c.PUT(context.Background(), fmt.Sprintf("/wallets/%v/addresses", c.id), a) return } // RemoveAddress removes the specified address from the wallet. func (c *WalletClient) RemoveAddress(addr types.Address) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v/addresses/%v", c.id, addr)) + err = c.c.DELETE(context.Background(), fmt.Sprintf("/wallets/%v/addresses/%v", c.id, addr)) return } // Addresses the addresses controlled by the wallet. func (c *WalletClient) Addresses() (resp []wallet.Address, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/addresses", c.id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/addresses", c.id), &resp) return } // Balance returns the current wallet balance. func (c *WalletClient) Balance() (resp BalanceResponse, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/balance", c.id), &resp) return } // Events returns all events relevant to the wallet. func (c *WalletClient) Events(offset, limit int) (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.id, offset, limit), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.id, offset, limit), &resp) return } // UnconfirmedEvents returns all unconfirmed events relevant to the wallet. func (c *WalletClient) UnconfirmedEvents() (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/events/unconfirmed", c.id), &resp) + err = c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/events/unconfirmed", c.id), &resp) return } // SiacoinOutputs returns the set of unspent outputs controlled by the wallet. func (c *WalletClient) SiacoinOutputs(offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { var resp SiacoinElementsResponse - err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &resp) + err := c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &resp) return resp.Outputs, resp.Basis, err } // SiafundOutputs returns the set of unspent outputs controlled by the wallet. func (c *WalletClient) SiafundOutputs(offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { var resp SiafundElementsResponse - err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &resp) + err := c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &resp) return resp.Outputs, resp.Basis, err } // Reserve reserves a set outputs for use in a transaction. func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOutputID, duration time.Duration) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.id), WalletReserveRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/reserve", c.id), WalletReserveRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, }, nil) @@ -346,7 +347,7 @@ func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOut // Release releases a set of previously-reserved outputs. func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOutputID) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/release", c.id), WalletReleaseRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/release", c.id), WalletReleaseRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, }, nil) @@ -355,7 +356,7 @@ func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOut // Fund funds a siacoin transaction. func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, changeAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fund", c.id), WalletFundRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/fund", c.id), WalletFundRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, @@ -365,7 +366,7 @@ func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, change // FundSF funds a siafund transaction. func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, claimAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fundsf", c.id), WalletFundSFRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/fundsf", c.id), WalletFundSFRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, @@ -377,7 +378,7 @@ func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, // Construct constructs a transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned // along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/transaction", c.id), WalletConstructRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/construct/transaction", c.id), WalletConstructRequest{ Siacoins: siacoins, Siafunds: siafunds, ChangeAddress: change, @@ -388,7 +389,7 @@ func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []type // ConstructV2 constructs a V2 transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned // along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. func (c *WalletClient) ConstructV2(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructV2Response, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ + err = c.c.POST(context.Background(), fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ Siacoins: siacoins, Siafunds: siafunds, ChangeAddress: change, diff --git a/api/server.go b/api/server.go index 181c1b2..99dfb0a 100644 --- a/api/server.go +++ b/api/server.go @@ -271,7 +271,7 @@ func (s *server) syncerConnectHandler(jc jape.Context) { if jc.Check("couldn't connect to peer", err) != nil { return } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { @@ -286,7 +286,7 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { } else { s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) txpoolParentsHandler(jc jape.Context) { @@ -345,7 +345,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions) } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { @@ -424,7 +424,7 @@ func (s *server) walletsIDHandlerDELETE(jc jape.Context) { } else if jc.Check("couldn't remove wallet", err) != nil { return } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) rescanHandlerGET(jc jape.Context) { @@ -484,7 +484,7 @@ func (s *server) rescanHandlerPOST(jc jape.Context) { } }() - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { @@ -495,7 +495,7 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { } else if jc.Check("couldn't add address", s.wm.AddAddress(id, addr)) != nil { return } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { @@ -511,7 +511,7 @@ func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { } else if jc.Check("couldn't remove address", err) != nil { return } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { @@ -684,7 +684,7 @@ func (s *server) walletsReserveHandler(jc jape.Context) { if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids)) != nil { return } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) walletsReleaseHandler(jc jape.Context) { @@ -701,7 +701,7 @@ func (s *server) walletsReleaseHandler(jc jape.Context) { ids = append(ids, types.Hash256(id)) } s.wm.Release(ids) - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) walletsFundHandler(jc jape.Context) { @@ -1317,7 +1317,7 @@ func (s *server) debugMineHandler(jc jape.Context) { log.Debug("mined block", zap.Stringer("blockID", b.ID())) n-- } - jc.EmptyResonse() + jc.Encode(nil) } func (s *server) pprofHandler(jc jape.Context) { diff --git a/go.mod b/go.mod index e54b439..33ede7e 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module go.sia.tech/walletd/v2 // v2.2.0 -go 1.23.1 +go 1.23.2 toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d - go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5 - go.sia.tech/jape v0.12.1 + go.sia.tech/core v0.11.0 + go.sia.tech/coreutils v0.13.0 + go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.31.0 @@ -24,7 +24,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.50.1 // indirect + github.com/quic-go/quic-go v0.51.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.4.0 // indirect go.sia.tech/mux v1.4.0 // indirect @@ -32,11 +32,10 @@ require ( go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.37.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/net v0.39.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect - golang.org/x/tools v0.31.0 // indirect + golang.org/x/tools v0.32.0 // indirect ) diff --git a/go.sum b/go.sum index f15082b..7bebd02 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= -github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/quic-go/quic-go v0.51.0 h1:K8exxe9zXxeRKxaXxi/GpUqYiTrtdiWP8bo1KFya6Wc= +github.com/quic-go/quic-go v0.51.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -41,12 +41,20 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d h1:iXSffVVq7SwisMIdI7FVioSCP+sPANIuPrbpFBGnWBU= -go.sia.tech/core v0.10.6-0.20250407154704-81a030aad05d/go.mod h1:AFTwGXQ8VQUyD9qasJTzTKBO9y/jianSqalHrwrzyxA= -go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5 h1:GzcwIi+Vx3KpxQyTw4kdBOS4s/jarMuUi7C8vApPEIU= -go.sia.tech/coreutils v0.12.2-0.20250409194146-7bb9065821f5/go.mod h1:UASHkZuV8pezqFtNsEoFP6YrspNHGRiRM+EbgL4cQv0= -go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= -go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= +go.sia.tech/core v0.10.5 h1:r+lIeViMkKslu7dPhxzX2Nsvo2lyvTsW+5s0fKIbgXc= +go.sia.tech/core v0.10.5/go.mod h1:42VPNZYiAR29qFK2RppyB4DtUYhHl6qS+q00Un4IBqs= +go.sia.tech/core v0.10.6-0.20250417200824-1f23320dea57 h1:68FWxiSQk87hZcePTsGERBJbz6e08tvLzr4rdb653Gc= +go.sia.tech/core v0.10.6-0.20250417200824-1f23320dea57/go.mod h1:TGJeylBR5Y5Qbq4clzncuUEGgLKx6nKaQmfOXr+jzk4= +go.sia.tech/core v0.11.0 h1:Rfb1D6DMs96bwhArluvYWqa+xlcgRAqU9uOSy75h58M= +go.sia.tech/core v0.11.0/go.mod h1:1qqyJUN04kEMEBUfofp0Pkf9s2RnHzGKfy+Hec2dbCk= +go.sia.tech/coreutils v0.12.1 h1:7IaxAOtDkQXr8iI5SnZkaYOV7s+SSpUk9MTYv+OZqbg= +go.sia.tech/coreutils v0.12.1/go.mod h1:QJhEM0LatYcqCncptF0ekihRk3DjGUkVtjWStoU0u70= +go.sia.tech/coreutils v0.12.2-0.20250421154359-b16204938056 h1:6yzoNuQigVjPNIEDsh78rjCZzQbh2NKDW2cVtgaZXZM= +go.sia.tech/coreutils v0.12.2-0.20250421154359-b16204938056/go.mod h1:Xo6po8jPbwYWApzsQgYF0qcBUZtXLPZ6kjxKrCiYM3k= +go.sia.tech/coreutils v0.13.0 h1:+0MrBzkZCfhd1gATTy472qnQpbKsYpguEoGLORF4mlw= +go.sia.tech/coreutils v0.13.0/go.mod h1:YQi5rdlHYGx2+tCYTgB/Vd6I4E82sTKJrvf2hc5M/iU= +go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 h1:hpLnok8mp/n1RwvrAbIwf8d/hQeue1E1jn2JxjIumeo= +go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6/go.mod h1:0AKGMZZKD/fUQXOT747F6A8ZIxooEb2Fr9TdN2eSye4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= @@ -67,8 +75,8 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= @@ -77,10 +85,8 @@ golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= -golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 8b4304739aedfdaace505ffbed6f5354259d55d0 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 23 Apr 2025 08:55:39 -0700 Subject: [PATCH 418/630] update dependencies (again) --- go.mod | 2 +- go.sum | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 33ede7e..75700c0 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.11.0 - go.sia.tech/coreutils v0.13.0 + go.sia.tech/coreutils v0.13.1 go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 7bebd02..df5c598 100644 --- a/go.sum +++ b/go.sum @@ -41,18 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.10.5 h1:r+lIeViMkKslu7dPhxzX2Nsvo2lyvTsW+5s0fKIbgXc= -go.sia.tech/core v0.10.5/go.mod h1:42VPNZYiAR29qFK2RppyB4DtUYhHl6qS+q00Un4IBqs= -go.sia.tech/core v0.10.6-0.20250417200824-1f23320dea57 h1:68FWxiSQk87hZcePTsGERBJbz6e08tvLzr4rdb653Gc= -go.sia.tech/core v0.10.6-0.20250417200824-1f23320dea57/go.mod h1:TGJeylBR5Y5Qbq4clzncuUEGgLKx6nKaQmfOXr+jzk4= go.sia.tech/core v0.11.0 h1:Rfb1D6DMs96bwhArluvYWqa+xlcgRAqU9uOSy75h58M= go.sia.tech/core v0.11.0/go.mod h1:1qqyJUN04kEMEBUfofp0Pkf9s2RnHzGKfy+Hec2dbCk= -go.sia.tech/coreutils v0.12.1 h1:7IaxAOtDkQXr8iI5SnZkaYOV7s+SSpUk9MTYv+OZqbg= -go.sia.tech/coreutils v0.12.1/go.mod h1:QJhEM0LatYcqCncptF0ekihRk3DjGUkVtjWStoU0u70= -go.sia.tech/coreutils v0.12.2-0.20250421154359-b16204938056 h1:6yzoNuQigVjPNIEDsh78rjCZzQbh2NKDW2cVtgaZXZM= -go.sia.tech/coreutils v0.12.2-0.20250421154359-b16204938056/go.mod h1:Xo6po8jPbwYWApzsQgYF0qcBUZtXLPZ6kjxKrCiYM3k= -go.sia.tech/coreutils v0.13.0 h1:+0MrBzkZCfhd1gATTy472qnQpbKsYpguEoGLORF4mlw= -go.sia.tech/coreutils v0.13.0/go.mod h1:YQi5rdlHYGx2+tCYTgB/Vd6I4E82sTKJrvf2hc5M/iU= +go.sia.tech/coreutils v0.13.1 h1:wmRL+eAc+KM/2qjmjSJ6jM2se2qOVq5b4SkpktOT/QE= +go.sia.tech/coreutils v0.13.1/go.mod h1:ZLPl0GBcQEYvnBqauc7ZWydAMplfZ6/qd7jGjSo+4X0= go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 h1:hpLnok8mp/n1RwvrAbIwf8d/hQeue1E1jn2JxjIumeo= go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6/go.mod h1:0AKGMZZKD/fUQXOT747F6A8ZIxooEb2Fr9TdN2eSye4= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 381cd01ee026f9605e3c343670f5ce9651f6969d Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 23 Apr 2025 22:44:39 -0700 Subject: [PATCH 419/630] add changeset --- .changeset/update_core_to_v0110_and_coreutils_to_v0131.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_core_to_v0110_and_coreutils_to_v0131.md diff --git a/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md b/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md new file mode 100644 index 0000000..08372de --- /dev/null +++ b/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core to v0.11.0 and coreutils to v0.13.1 From 527ab895cbe990c25880b28e31f1a972c00f3b3f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Apr 2025 05:44:55 +0000 Subject: [PATCH 420/630] chore: prepare release 2.2.1 --- .changeset/update_core_to_v0110_and_coreutils_to_v0131.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_core_to_v0110_and_coreutils_to_v0131.md diff --git a/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md b/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md deleted file mode 100644 index 08372de..0000000 --- a/.changeset/update_core_to_v0110_and_coreutils_to_v0131.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core to v0.11.0 and coreutils to v0.13.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6248f8c..74baed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.2.1 (2025-04-24) + +### Fixes + +- Update core to v0.11.0 and coreutils to v0.13.1 + ## 2.2.0 (2025-04-22) ### Features diff --git a/go.mod b/go.mod index 75700c0..dddd772 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.2.0 +module go.sia.tech/walletd/v2 // v2.2.1 go 1.23.2 From 55686ccdb3bc2aa15cb63b1acfbbe4046494d1e8 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 24 Apr 2025 10:49:56 +0200 Subject: [PATCH 421/630] remove leftover keystore code --- README.md | 21 ++++++++------------- cmd/walletd/main.go | 18 ++++-------------- config/config.go | 7 ------- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index bed0c4b..1b30288 100644 --- a/README.md +++ b/README.md @@ -23,24 +23,24 @@ Setup guides are available at https://docs.sia.tech In "personal" index mode, `walletd` will only index addresses that are registered in the wallet. This mode is recommended for most users, as it provides a good balance between -comprehensiveness and resource usage for personal wallets. This is the default +comprehensiveness and resource usage for personal wallets. This is the default mode for `walletd`. -When adding addresses with existing history on chain, users will need to manually +When adding addresses with existing history on chain, users will need to manually initiate a rescan to index the new transactions. This can take some to complete, -depending on the number of blocks that need to be scanned. When adding addresses +depending on the number of blocks that need to be scanned. When adding addresses with no existing history, a rescan is not necessary. **Full** In "full" index mode, `walletd` will index the entire blockchain including all addresses -and UTXOs. This is the most comprehensive mode, but it also requires the most -resources. This mode is recommended for exchanges or wallet builders that need +and UTXOs. This is the most comprehensive mode, but it also requires the most +resources. This mode is recommended for exchanges or wallet builders that need to support a large or unknown number of addresses. **None** -In "none" index mode, `walletd` will treat the database as read-only and not +In "none" index mode, `walletd` will treat the database as read-only and not index any new data. This mode is only useful in situations where another process is managing the database and `walletd` is only being used to read data. @@ -64,7 +64,6 @@ The priority of configuration settings is as follows: + `WALLETD_API_PASSWORD` - The password required to access the API. + `WALLETD_CONFIG_FILE` - The path to the YAML configuration file. Defaults to `walletd.yml` in the working directory. + `WALLETD_LOG_FILE` - The path to the log file. -+ `WALLETD_KEYSTORE_SECRET` - The secret to use for encrypting stored ed25519 signing keys. ### Command Line Flags ``` @@ -98,8 +97,6 @@ Flags: network to connect to; must be one of 'mainnet', 'zen', 'anagami', or the path to a custom network file for a local testnet -upnp attempt to forward ports and discover IP with UPnP - -keystore - enables the optional ed25519 key store. ``` ### YAML @@ -126,8 +123,6 @@ syncer: enableUPnP: false peers: [] address: :9981 -keystore: - enabled: false index: mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) @@ -154,7 +149,7 @@ CGO_ENABLED=1 go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '- ``` ## Docker Image -`walletd` includes a Dockerfile for building a Docker image. For building and +`walletd` includes a Dockerfile for building a Docker image. For building and running `walletd` within a Docker container. The image can also be pulled from `ghcr.io/siafoundation/walletd`. ```sh @@ -253,4 +248,4 @@ You can create a custom local testnet by creating a network.json file locally an ] } } -``` \ No newline at end of file +``` diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 923a18d..617659d 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -21,11 +21,10 @@ import ( ) const ( - apiPasswordEnvVar = "WALLETD_API_PASSWORD" - configFileEnvVar = "WALLETD_CONFIG_FILE" - dataDirEnvVar = "WALLETD_DATA_DIR" - logFileEnvVar = "WALLETD_LOG_FILE_PATH" - keystoreSecretEnvVar = "WALLETD_KEYSTORE_SECRET" + apiPasswordEnvVar = "WALLETD_API_PASSWORD" + configFileEnvVar = "WALLETD_CONFIG_FILE" + dataDirEnvVar = "WALLETD_DATA_DIR" + logFileEnvVar = "WALLETD_LOG_FILE_PATH" ) const ( @@ -76,10 +75,6 @@ var cfg = config.Config{ Mode: wallet.IndexModePersonal, BatchSize: 1000, }, - KeyStore: config.KeyStore{ - Enabled: false, - Secret: os.Getenv(keystoreSecretEnvVar), - }, Log: config.Log{ Level: "info", File: config.LogFile{ @@ -214,7 +209,6 @@ func main() { rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") rootCmd.BoolVar(&cfg.HTTP.PublicEndpoints, "http.public", cfg.HTTP.PublicEndpoints, "disables auth on endpoints that should be publicly accessible when running walletd as a service") - rootCmd.BoolVar(&cfg.KeyStore.Enabled, "keystore", cfg.KeyStore.Enabled, "enables the keystore") rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to; must be one of 'mainnet', 'zen', 'anagami', or the path to a custom network file for a local testnet") @@ -260,10 +254,6 @@ func main() { checkFatalError("failed to parse index mode", cfg.Index.Mode.UnmarshalText([]byte(indexModeStr))) - if cfg.KeyStore.Enabled && cfg.KeyStore.Secret == "" { - checkFatalError("keystore is enabled but no secret was provided", errors.New("missing keystore secret")) - } - var logCores []zapcore.Core if cfg.Log.StdOut.Enabled { // if no log level is set for stdout, use the global log level diff --git a/config/config.go b/config/config.go index 733b3b3..0a0312e 100644 --- a/config/config.go +++ b/config/config.go @@ -25,12 +25,6 @@ type ( Peers []string `yaml:"peers,omitempty"` } - // KeyStore contains the configuration for the key store. - KeyStore struct { - Enabled bool `yaml:"enabled,omitempty"` - Secret string `yaml:"secret,omitempty"` - } - // Consensus contains the configuration for the consensus set. Consensus struct { Network string `yaml:"network,omitempty"` @@ -78,7 +72,6 @@ type ( Syncer Syncer `yaml:"syncer,omitempty"` Log Log `yaml:"log,omitempty"` Index Index `yaml:"index,omitempty"` - KeyStore KeyStore `yaml:"keystore,omitempty"` } ) From a2899860958e65a152d47778d68ec788f7581986 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:05:20 +0000 Subject: [PATCH 422/630] build(deps): bump the all-dependencies group with 3 updates Bumps the all-dependencies group with 3 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/jape](https://github.com/SiaFoundation/jape). Updates `go.sia.tech/core` from 0.11.0 to 0.12.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.11.0...v0.12.0) Updates `go.sia.tech/coreutils` from 0.13.1 to 0.13.2 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.13.1...v0.13.2) Updates `go.sia.tech/jape` from 0.12.2-0.20241010144215-1468bf476af6 to 0.13.1 - [Release notes](https://github.com/SiaFoundation/jape/releases) - [Commits](https://github.com/SiaFoundation/jape/commits/v0.13.1) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.13.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/jape dependency-version: 0.13.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index dddd772..758d83d 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.11.0 - go.sia.tech/coreutils v0.13.1 - go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 + go.sia.tech/core v0.12.0 + go.sia.tech/coreutils v0.13.2 + go.sia.tech/jape v0.13.1 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.31.0 diff --git a/go.sum b/go.sum index df5c598..de0bd92 100644 --- a/go.sum +++ b/go.sum @@ -41,12 +41,12 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.11.0 h1:Rfb1D6DMs96bwhArluvYWqa+xlcgRAqU9uOSy75h58M= -go.sia.tech/core v0.11.0/go.mod h1:1qqyJUN04kEMEBUfofp0Pkf9s2RnHzGKfy+Hec2dbCk= -go.sia.tech/coreutils v0.13.1 h1:wmRL+eAc+KM/2qjmjSJ6jM2se2qOVq5b4SkpktOT/QE= -go.sia.tech/coreutils v0.13.1/go.mod h1:ZLPl0GBcQEYvnBqauc7ZWydAMplfZ6/qd7jGjSo+4X0= -go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6 h1:hpLnok8mp/n1RwvrAbIwf8d/hQeue1E1jn2JxjIumeo= -go.sia.tech/jape v0.12.2-0.20241010144215-1468bf476af6/go.mod h1:0AKGMZZKD/fUQXOT747F6A8ZIxooEb2Fr9TdN2eSye4= +go.sia.tech/core v0.12.0 h1:nuHjUE3MnYPQ+BKo44DD64332jBGVZlH657c5RIomXw= +go.sia.tech/core v0.12.0/go.mod h1:ycpNTb9Y7Vtnq6HQ3iqOxeywLnjDs0udD3ZVq6KE13Y= +go.sia.tech/coreutils v0.13.2 h1:SlxQ6onhdjGrTB8I/TxFwIbFTM0mzH5amDoAQMkjmOI= +go.sia.tech/coreutils v0.13.2/go.mod h1:RELgLyNq1oCRBAXnMvGUefASRJWjju6jgS6ioYbZGFE= +go.sia.tech/jape v0.13.1 h1:gBTQCIXVFzUTs6mFA3NcKo/55SzlFYCzb4WbvGeaAF4= +go.sia.tech/jape v0.13.1/go.mod h1:ZZe2SKMWfpV0mpZWld/Rv4GGXpr51liyfUkes2ikw54= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= From 8a4b0c67e61a88fd98935ecd236e18e5f5673818 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 29 Apr 2025 15:39:59 +0200 Subject: [PATCH 423/630] fix build --- api/server.go | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/api/server.go b/api/server.go index 99dfb0a..c6b38c6 100644 --- a/api/server.go +++ b/api/server.go @@ -80,10 +80,10 @@ type ( Peers() []*syncer.Peer PeerInfo(addr string) (syncer.PeerInfo, error) Connect(ctx context.Context, addr string) (*syncer.Peer, error) - BroadcastHeader(types.BlockHeader) - BroadcastTransactionSet(txns []types.Transaction) - BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) - BroadcastV2BlockOutline(bo gateway.V2BlockOutline) + BroadcastHeader(types.BlockHeader) error + BroadcastTransactionSet(txns []types.Transaction) error + BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) error + BroadcastV2BlockOutline(bo gateway.V2BlockOutline) error } // A WalletManager manages wallets, keyed by name. @@ -282,9 +282,13 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { return } if b.V2 == nil { - s.s.BroadcastHeader(b.Header()) + if jc.Check("failed to broadcast header", s.s.BroadcastHeader(b.Header())) != nil { + return + } } else { - s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { + return + } } jc.Encode(nil) } @@ -326,7 +330,9 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return } - s.s.BroadcastTransactionSet(tbr.Transactions) + if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { + return + } } if len(tbr.V2Transactions) != 0 { if len(tbr.V2Transactions) == 1 { @@ -342,7 +348,9 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return } - s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions) + if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { + return + } } jc.Encode(nil) @@ -1309,9 +1317,13 @@ func (s *server) debugMineHandler(jc jape.Context) { } if b.V2 == nil { - s.s.BroadcastHeader(b.Header()) + if jc.Check("failed to broadcast header", s.s.BroadcastHeader(b.Header())) != nil { + return + } } else { - s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { + return + } } log.Debug("mined block", zap.Stringer("blockID", b.ID())) From 1efb0637c3db25a270a58b9a0182ae8fa025cb6c Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 29 Apr 2025 15:57:06 +0200 Subject: [PATCH 424/630] mock syncer --- internal/testutil/testutil.go | 61 ++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 4d8be74..9c90b02 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -1,6 +1,7 @@ package testutil import ( + "context" "net" "path/filepath" "testing" @@ -21,8 +22,11 @@ type ( ConsensusNode struct { Store *sqlite.Store Chain *chain.Manager - Syncer *syncer.Syncer + Syncer *MockSyncer } + + // MockSyncer is a no-op syncer implementation + MockSyncer struct{} ) // WaitForSync waits for the store to sync to the current tip of the chain manager. @@ -81,23 +85,10 @@ func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, } tb.Cleanup(func() { store.Close() }) - peerStore, err := sqlite.NewPeerStore(store) - if err != nil { - tb.Fatal(err) - } - - s := syncer.New(l, cm, peerStore, gateway.Header{ - GenesisID: genesis.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l.Addr().String(), - }) - tb.Cleanup(func() { s.Close() }) - go s.Run() - return &ConsensusNode{ Store: store, Chain: cm, - Syncer: s, + Syncer: &MockSyncer{}, } } @@ -110,3 +101,43 @@ func V1Network() (*consensus.Network, types.Block) { func V2Network() (*consensus.Network, types.Block) { return testutil.V2Network() } + +// Addr is a no-op +func (s *MockSyncer) Addr() string { + return "" +} + +// BroadcastHeader is a no-op +func (s *MockSyncer) BroadcastHeader(bh types.BlockHeader) error { + return nil +} + +// BroadcastTransactionSet is a no-op +func (s *MockSyncer) BroadcastTransactionSet(txns []types.Transaction) error { + return nil +} + +// BroadcastV2TransactionSet is a no-op +func (s *MockSyncer) BroadcastV2TransactionSet(basis types.ChainIndex, txns []types.V2Transaction) error { + return nil +} + +// BroadcastV2BlockOutline is a no-op +func (s *MockSyncer) BroadcastV2BlockOutline(outline gateway.V2BlockOutline) error { + return nil +} + +// Connect is a no-op +func (s *MockSyncer) Connect(ctx context.Context, addr string) (*syncer.Peer, error) { + return &syncer.Peer{}, nil +} + +// PeerInfo is a no-op +func (s *MockSyncer) PeerInfo(addr string) (syncer.PeerInfo, error) { + return syncer.PeerInfo{}, nil +} + +// Peers is a no-op +func (s *MockSyncer) Peers() []*syncer.Peer { + return nil +} From ac88932d163fe487ba54c12d92cb9f23be91289f Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 29 Apr 2025 12:02:07 -0700 Subject: [PATCH 425/630] add check endpoint --- ..._addresses_that_have_been_seen_on_chain.md | 7 +++ api/api.go | 30 ++++------- api/server.go | 31 +++++++++++ persist/sqlite/address_test.go | 52 +++++++++++++++++++ persist/sqlite/addresses.go | 29 +++++++++++ wallet/addresses.go | 6 +++ wallet/manager.go | 9 ++++ 7 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 .changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md create mode 100644 persist/sqlite/address_test.go diff --git a/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md b/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md new file mode 100644 index 0000000..62df047 --- /dev/null +++ b/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md @@ -0,0 +1,7 @@ +--- +default: minor +--- + +# Added `[POST] /check/addresses` to check for addresses that have been seen on chain + +This endpoint is useful for scanning the chain for look-aheads when in full index mode diff --git a/api/api.go b/api/api.go index d125a5d..185517d 100644 --- a/api/api.go +++ b/api/api.go @@ -219,30 +219,18 @@ type AddressSiafundElementsResponse struct { Outputs []wallet.UnspentSiafundElement `json:"outputs"` } +type CheckAddressesRequest struct { + Addresses []types.Address `json:"addresses"` +} + +// AddressKnownResponse is the response type for /addresses/known. +type CheckAddressesResponse struct { + Known bool `json:"known"` +} + // ElementSpentResponse is the response type for /outputs/siacoin/:id/spent and // /outputs/siafund/:id/spent. type ElementSpentResponse struct { Spent bool `json:"spent"` Event *wallet.Event `json:"event,omitempty"` } - -// An AddSigningKeyRequest is a request to add an ed25519 signing key to the -// key store. -type AddSigningKeyRequest struct { - PrivateKey types.PrivateKey `json:"privateKey"` -} - -// An AddSigningKeyResponse is the response to an AddSigningKeyRequest. -type AddSigningKeyResponse struct { - PublicKey types.PublicKey `json:"publicKey"` -} - -// A SignHashRequest is a request to sign a hash with a key. -type SignHashRequest struct { - Hash types.Hash256 `json:"hash"` -} - -// A SignHashResponse is the response to a SignHashRequest. -type SignHashResponse struct { - Signature types.Signature `json:"signature"` -} diff --git a/api/server.go b/api/server.go index c6b38c6..9fe7f7e 100644 --- a/api/server.go +++ b/api/server.go @@ -132,6 +132,16 @@ type ( Reserve([]types.Hash256) error Release([]types.Hash256) + + // CheckAddresses returns true if any of the addresses are known to + // the server. + // + // In full index mode, this returns true if any addresses + // have been seen on chain. + // + // In personal index mode, this returns true only if the address + // is registered to a wallet. + CheckAddresses([]types.Address) (bool, error) } ) @@ -1297,6 +1307,25 @@ func (s *server) outputsSiafundHandlerGET(jc jape.Context) { jc.Encode(output) } +func (s *server) checkAddressesHandlerPOST(jc jape.Context) { + var req CheckAddressesRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Addresses) > 10000 { + jc.Error(errors.New("too many addresses"), http.StatusBadRequest) + return + } + + ok, err := s.wm.CheckAddresses(req.Addresses) + if jc.Check("couldn't check addresses", err) != nil { + return + } + + jc.Encode(CheckAddressesResponse{ + Known: ok, + }) +} + func (s *server) debugMineHandler(jc jape.Context) { var req DebugMineRequest if jc.Decode(&req) != nil { @@ -1438,6 +1467,8 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), "GET /outputs/siafund/:id/spent": wrapPublicAuthHandler(srv.outputsSiafundSpentHandlerGET), + "POST /check/addresses": wrapPublicAuthHandler(srv.checkAddressesHandlerPOST), + "GET /events/:id": wrapPublicAuthHandler(srv.eventsHandlerGET), "GET /rescan": wrapAuthHandler(srv.rescanHandlerGET), diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go new file mode 100644 index 0000000..2467d72 --- /dev/null +++ b/persist/sqlite/address_test.go @@ -0,0 +1,52 @@ +package sqlite + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +func TestAddressesKnown(t *testing.T) { + log := zaptest.NewLogger(t) + + // generate a large number of random addresses + addresses := make([]types.Address, 1000) + for i := range len(addresses) { + addresses[i] = frand.Entropy256() + } + + // create a new database + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if known, err := db.CheckAddresses(addresses); err != nil { + t.Fatal(err) + } else if known { + t.Fatal("expected no addresses to be known") + } + + // add a random address to the database + address := addresses[frand.Intn(len(addresses))] + + w, err := db.AddWallet(wallet.Wallet{}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{ + Address: address, + }); err != nil { + t.Fatal(err) + } + + if known, err := db.CheckAddresses(addresses); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected addresses to be known") + } +} diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 3d2f30b..30de08e 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -10,6 +10,35 @@ import ( "go.sia.tech/walletd/v2/wallet" ) +// CheckAddresses returns true if any of the addresses have been seen on the +// blockchain. This is a quick way to scan wallets for lookaheads. +// +// If the index mode is not full, this function will only return true if +// an address is registered with a wallet. +func (s *Store) CheckAddresses(addresses []types.Address) (known bool, err error) { + err = s.transaction(func(tx *txn) error { + stmt, err := tx.Prepare(`SELECT true FROM sia_addresses WHERE sia_address=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, addr := range addresses { + if err := stmt.QueryRow(encode(addr)).Scan(&known); err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return fmt.Errorf("failed to query address: %w", err) + } + if known { + return nil + } + } + return nil + }) + return +} + // AddressBalance returns the balance of a single address. func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { err = s.transaction(func(tx *txn) error { diff --git a/wallet/addresses.go b/wallet/addresses.go index 8a595c3..63cc478 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -6,6 +6,12 @@ import ( "go.sia.tech/core/types" ) +// CheckAddresses returns true if any of the addresses have been seen on the +// blockchain. This is a quick way to scan wallets for lookaheads. +func (m *Manager) CheckAddresses(address []types.Address) (bool, error) { + return m.store.CheckAddresses(address) +} + // AddressBalance returns the balance of a single address. func (m *Manager) AddressBalance(address types.Address) (balance Balance, err error) { return m.store.AddressBalance(address) diff --git a/wallet/manager.go b/wallet/manager.go index 0c1782b..4b22913 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -87,6 +87,15 @@ type ( AddressEvents(address types.Address, offset, limit int) (events []Event, err error) AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) + // CheckAddresses returns true if any of the addresses have been seen on the + // blockchain. This is a quick way to scan wallets for lookaheads. + // + // If index mode is full, this function returns true if any + // address has been seen on chain. + // + // In personal index mode, this function returns true only + // if the address is registered to a wallet. + CheckAddresses([]types.Address) (bool, error) Events(eventIDs []types.Hash256) ([]Event, error) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) From 717863ce6b66225949f4ffa739bf53b1f412c9ac Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 29 Apr 2025 12:04:20 -0700 Subject: [PATCH 426/630] fix lint --- api/api.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/api.go b/api/api.go index 185517d..0aa9c15 100644 --- a/api/api.go +++ b/api/api.go @@ -219,11 +219,12 @@ type AddressSiafundElementsResponse struct { Outputs []wallet.UnspentSiafundElement `json:"outputs"` } +// CheckAddressesRequest is the request type for [POST] /check/addresses. type CheckAddressesRequest struct { Addresses []types.Address `json:"addresses"` } -// AddressKnownResponse is the response type for /addresses/known. +// CheckAddressesResponse is the response type for [POST] /check/addresses. type CheckAddressesResponse struct { Known bool `json:"known"` } From d706e06ac64dd226d3a5bc9f3bd14afd6b09aa2a Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 29 Apr 2025 12:05:56 -0700 Subject: [PATCH 427/630] add missing client method --- api/client.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/client.go b/api/client.go index 1e9cd9b..ff7590e 100644 --- a/api/client.go +++ b/api/client.go @@ -258,6 +258,16 @@ func (c *Client) AddressSiafundOutputs(addr types.Address, useTpool bool, offset return resp.Outputs, resp.Basis, err } +// CheckAddresses checks whether the specified addresses are known to the wallet. +// In full index mode, this will return true if any of the addresses have been seen on chain. +func (c *Client) CheckAddresses(addresses []types.Address) (bool, error) { + var resp CheckAddressesResponse + err := c.c.POST(context.Background(), "/check/addresses", CheckAddressesRequest{ + Addresses: addresses, + }, &resp) + return resp.Known, err +} + // Event returns the event with the specified ID. func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/events/%v", id), &resp) From e791dc8d9768475b5fe865c3db2477584fffd809 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 29 Apr 2025 12:07:03 -0700 Subject: [PATCH 428/630] address comments --- persist/sqlite/address_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go index 2467d72..872bf10 100644 --- a/persist/sqlite/address_test.go +++ b/persist/sqlite/address_test.go @@ -15,7 +15,7 @@ func TestAddressesKnown(t *testing.T) { // generate a large number of random addresses addresses := make([]types.Address, 1000) - for i := range len(addresses) { + for i := range addresses { addresses[i] = frand.Entropy256() } From b5d6bc10cef688eb6b5e784bf3dd4972379be465 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 29 Apr 2025 16:51:19 -0700 Subject: [PATCH 429/630] consistent test name --- persist/sqlite/address_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go index 872bf10..ecbefea 100644 --- a/persist/sqlite/address_test.go +++ b/persist/sqlite/address_test.go @@ -10,7 +10,7 @@ import ( "lukechampine.com/frand" ) -func TestAddressesKnown(t *testing.T) { +func TestCheckAddresses(t *testing.T) { log := zaptest.NewLogger(t) // generate a large number of random addresses From d7b88a4faf3c0c546f1356a632f0a397b20256db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 16:54:50 +0000 Subject: [PATCH 430/630] build(deps): bump go.sia.tech/jape in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/jape](https://github.com/SiaFoundation/jape). Updates `go.sia.tech/jape` from 0.13.1 to 0.14.0 - [Release notes](https://github.com/SiaFoundation/jape/releases) - [Changelog](https://github.com/SiaFoundation/jape/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/jape/compare/v0.13.1...v0.14.0) --- updated-dependencies: - dependency-name: go.sia.tech/jape dependency-version: 0.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 758d83d..ff216b9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.12.0 go.sia.tech/coreutils v0.13.2 - go.sia.tech/jape v0.13.1 + go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 golang.org/x/term v0.31.0 diff --git a/go.sum b/go.sum index de0bd92..0212359 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ go.sia.tech/core v0.12.0 h1:nuHjUE3MnYPQ+BKo44DD64332jBGVZlH657c5RIomXw= go.sia.tech/core v0.12.0/go.mod h1:ycpNTb9Y7Vtnq6HQ3iqOxeywLnjDs0udD3ZVq6KE13Y= go.sia.tech/coreutils v0.13.2 h1:SlxQ6onhdjGrTB8I/TxFwIbFTM0mzH5amDoAQMkjmOI= go.sia.tech/coreutils v0.13.2/go.mod h1:RELgLyNq1oCRBAXnMvGUefASRJWjju6jgS6ioYbZGFE= -go.sia.tech/jape v0.13.1 h1:gBTQCIXVFzUTs6mFA3NcKo/55SzlFYCzb4WbvGeaAF4= -go.sia.tech/jape v0.13.1/go.mod h1:ZZe2SKMWfpV0mpZWld/Rv4GGXpr51liyfUkes2ikw54= +go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= +go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= From f7099504b8c0940a4c2ff7a39cb9b76458210631 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 6 May 2025 10:10:53 +0200 Subject: [PATCH 431/630] fix TestAPISecurity --- api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index c07c8cf..8ab3c75 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1304,7 +1304,7 @@ func TestAPISecurity(t *testing.T) { c = api.NewClient("http://"+httpListener.Addr().String(), "wrong") if _, err := c.ConsensusTip(); err == nil { t.Fatal("expected auth error") - } else if err.Error() == "unauthorized" { + } else if err.Error() != "unauthorized" { t.Fatal("expected auth error, got", err) } @@ -1337,7 +1337,7 @@ func TestAPISecurity(t *testing.T) { // check that a private endpoint is still protected if _, err := c.Wallets(); err == nil { t.Fatal("expected auth error") - } else if err.Error() == "unauthorized" { + } else if err.Error() != "unauthorized" { t.Fatal("expected auth error, got", err) } From d09f7ed46bb86d1244ca715d7afbaf343b1c2c5c Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 7 May 2025 10:51:28 -0700 Subject: [PATCH 432/630] add log cli flags --- ...for_configuring_logging_using_cli_flags.md | 5 +++ ...cli_flag_to_set_log_level_logleveldebug.md | 5 +++ cmd/walletd/main.go | 36 ++++++------------- config/config.go | 21 +++++------ 4 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 .changeset/add_support_for_configuring_logging_using_cli_flags.md create mode 100644 .changeset/added_cli_flag_to_set_log_level_logleveldebug.md diff --git a/.changeset/add_support_for_configuring_logging_using_cli_flags.md b/.changeset/add_support_for_configuring_logging_using_cli_flags.md new file mode 100644 index 0000000..915e9fa --- /dev/null +++ b/.changeset/add_support_for_configuring_logging_using_cli_flags.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Added CLI flag to disable log locations `--log.file.enabled=false` `--log.stdout.enabled=false` diff --git a/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md b/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md new file mode 100644 index 0000000..450731f --- /dev/null +++ b/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Added CLI flag to set log level `--log.level=debug` diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 617659d..eb78e7c 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -76,7 +76,7 @@ var cfg = config.Config{ BatchSize: 1000, }, Log: config.Log{ - Level: "info", + Level: zap.NewAtomicLevelAt(zap.InfoLevel), File: config.LogFile{ Enabled: true, Format: "json", @@ -162,25 +162,7 @@ func humanEncoder(showColors bool) zapcore.Encoder { return zapcore.NewConsoleEncoder(cfg) } -func parseLogLevel(level string) zap.AtomicLevel { - switch level { - case "debug": - return zap.NewAtomicLevelAt(zap.DebugLevel) - case "info": - return zap.NewAtomicLevelAt(zap.InfoLevel) - case "warn": - return zap.NewAtomicLevelAt(zap.WarnLevel) - case "error": - return zap.NewAtomicLevelAt(zap.ErrorLevel) - default: - fmt.Printf("invalid log level %q", level) - os.Exit(1) - } - panic("unreachable") -} - -func initStdoutLog(colored bool, levelStr string) *zap.Logger { - level := parseLogLevel(levelStr) +func initStdoutLog(colored bool, level zap.AtomicLevel) *zap.Logger { core := zapcore.NewCore(humanEncoder(colored), zapcore.Lock(os.Stdout), level) return zap.New(core, zap.AddCaller()) } @@ -218,6 +200,10 @@ func main() { rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (personal, full, none)") rootCmd.IntVar(&cfg.Index.BatchSize, "index.batch", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") + rootCmd.TextVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "log level (debug, info, warn, error)") + rootCmd.BoolVar(&cfg.Log.File.Enabled, "log.file.enabled", cfg.Log.File.Enabled, "enable file logging") + rootCmd.BoolVar(&cfg.Log.StdOut.Enabled, "log.stdout.enabled", cfg.Log.StdOut.Enabled, "enable stdout logging") + versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) configCmd := flagg.New("config", "interactively configure walletd") @@ -257,7 +243,7 @@ func main() { var logCores []zapcore.Core if cfg.Log.StdOut.Enabled { // if no log level is set for stdout, use the global log level - if cfg.Log.StdOut.Level == "" { + if cfg.Log.StdOut.Level == (zap.AtomicLevel{}) { cfg.Log.StdOut.Level = cfg.Log.Level } @@ -270,13 +256,12 @@ func main() { } // create the stdout logger - level := parseLogLevel(cfg.Log.StdOut.Level) - logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), level)) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), cfg.Log.StdOut.Level)) } if cfg.Log.File.Enabled { // if no log level is set for file, use the global log level - if cfg.Log.File.Level == "" { + if cfg.Log.File.Level == (zap.AtomicLevel{}) { cfg.Log.File.Level = cfg.Log.Level } @@ -299,8 +284,7 @@ func main() { defer closeFn() // create the file logger - level := parseLogLevel(cfg.Log.File.Level) - logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(fileWriter), level)) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(fileWriter), cfg.Log.File.Level)) } var log *zap.Logger diff --git a/config/config.go b/config/config.go index 0a0312e..e6caf31 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "os" "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap" "gopkg.in/yaml.v3" ) @@ -38,26 +39,26 @@ type ( // LogFile configures the file output of the logger. LogFile struct { - Enabled bool `yaml:"enabled,omitempty"` - Level string `yaml:"level,omitempty"` // override the file log level - Format string `yaml:"format,omitempty"` + Enabled bool `yaml:"enabled,omitempty"` + Level zap.AtomicLevel `yaml:"level,omitempty"` // override the file log level + Format string `yaml:"format,omitempty"` // Path is the path of the log file. Path string `yaml:"path,omitempty"` } // StdOut configures the standard output of the logger. StdOut struct { - Level string `yaml:"level,omitempty"` // override the stdout log level - Enabled bool `yaml:"enabled,omitempty"` - Format string `yaml:"format,omitempty"` - EnableANSI bool `yaml:"enableANSI,omitempty"` //nolint:tagliatelle + Level zap.AtomicLevel `yaml:"level,omitempty"` // override the stdout log level + Enabled bool `yaml:"enabled,omitempty"` + Format string `yaml:"format,omitempty"` + EnableANSI bool `yaml:"enableANSI,omitempty"` //nolint:tagliatelle } // Log contains the configuration for the logger. Log struct { - Level string `yaml:"level,omitempty"` // global log level - StdOut StdOut `yaml:"stdout,omitempty"` - File LogFile `yaml:"file,omitempty"` + Level zap.AtomicLevel `yaml:"level,omitempty"` // global log level + StdOut StdOut `yaml:"stdout,omitempty"` + File LogFile `yaml:"file,omitempty"` } // Config contains the configuration for the host. From 67dce7a59ed11ef354a139ad8a38e15f1e56356b Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 12 May 2025 08:50:41 -0700 Subject: [PATCH 433/630] update erravimus --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index ff216b9..ecff408 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,11 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.12.0 - go.sia.tech/coreutils v0.13.2 + go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 - golang.org/x/term v0.31.0 + golang.org/x/term v0.32.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -31,11 +31,11 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.37.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/tools v0.32.0 // indirect ) diff --git a/go.sum b/go.sum index 0212359..24a5799 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.12.0 h1:nuHjUE3MnYPQ+BKo44DD64332jBGVZlH657c5RIomXw= go.sia.tech/core v0.12.0/go.mod h1:ycpNTb9Y7Vtnq6HQ3iqOxeywLnjDs0udD3ZVq6KE13Y= -go.sia.tech/coreutils v0.13.2 h1:SlxQ6onhdjGrTB8I/TxFwIbFTM0mzH5amDoAQMkjmOI= -go.sia.tech/coreutils v0.13.2/go.mod h1:RELgLyNq1oCRBAXnMvGUefASRJWjju6jgS6ioYbZGFE= +go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2 h1:2J6dj8JK2dzLlaMzPmu/DOBCeSMhTYeYQl65wUMSqlY= +go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2/go.mod h1:10LIkoS//x5fn/dVuNUS/1xDyX4RjYNmGwMvwBpQEKg= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -61,22 +61,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= From 7954375dcde79069c86f546df783fe0ebcd21846 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 16:09:57 +0000 Subject: [PATCH 434/630] chore: prepare release 2.3.0 --- ...heck_for_addresses_that_have_been_seen_on_chain.md | 7 ------- ...support_for_configuring_logging_using_cli_flags.md | 5 ----- .../added_cli_flag_to_set_log_level_logleveldebug.md | 5 ----- CHANGELOG.md | 11 +++++++++++ go.mod | 2 +- 5 files changed, 12 insertions(+), 18 deletions(-) delete mode 100644 .changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md delete mode 100644 .changeset/add_support_for_configuring_logging_using_cli_flags.md delete mode 100644 .changeset/added_cli_flag_to_set_log_level_logleveldebug.md diff --git a/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md b/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md deleted file mode 100644 index 62df047..0000000 --- a/.changeset/add_endpoint_to_check_for_addresses_that_have_been_seen_on_chain.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Added `[POST] /check/addresses` to check for addresses that have been seen on chain - -This endpoint is useful for scanning the chain for look-aheads when in full index mode diff --git a/.changeset/add_support_for_configuring_logging_using_cli_flags.md b/.changeset/add_support_for_configuring_logging_using_cli_flags.md deleted file mode 100644 index 915e9fa..0000000 --- a/.changeset/add_support_for_configuring_logging_using_cli_flags.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Added CLI flag to disable log locations `--log.file.enabled=false` `--log.stdout.enabled=false` diff --git a/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md b/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md deleted file mode 100644 index 450731f..0000000 --- a/.changeset/added_cli_flag_to_set_log_level_logleveldebug.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Added CLI flag to set log level `--log.level=debug` diff --git a/CHANGELOG.md b/CHANGELOG.md index 74baed3..989a313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## 2.3.0 (2025-05-12) + +### Features + +- Added CLI flag to disable log locations `--log.file.enabled=false` `--log.stdout.enabled=false` +- Added CLI flag to set log level `--log.level=debug` + +#### Added `[POST] /check/addresses` to check for addresses that have been seen on chain + +This endpoint is useful for scanning the chain for look-aheads when in full index mode + ## 2.2.1 (2025-04-24) ### Fixes diff --git a/go.mod b/go.mod index ecff408..23ed246 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.2.1 +module go.sia.tech/walletd/v2 // v2.3.0 go 1.23.2 From 78d497081f9d3208d3d84a1b25778353b1cd7257 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 13 May 2025 09:40:39 -0700 Subject: [PATCH 435/630] add consensus state --- ...eturn_the_block_and_its_consensus_state.md | 5 +++ api/api.go | 6 +++ api/api_test.go | 41 +++++++++++++++++++ api/client.go | 11 +++++ api/server.go | 26 ++++++++++++ 5 files changed, 89 insertions(+) create mode 100644 .changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md diff --git a/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md b/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md new file mode 100644 index 0000000..27363fe --- /dev/null +++ b/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Add GET /consensus/state/:id endpoint to return a block and its consensus state diff --git a/api/api.go b/api/api.go index 0aa9c15..2be58c9 100644 --- a/api/api.go +++ b/api/api.go @@ -56,6 +56,12 @@ type TxpoolUpdateV2TransactionsRequest struct { Transactions []types.V2Transaction `json:"transactions"` } +// ConsensusStateResponse is the response type for GET /consensus/state/:id. +type ConsensusStateResponse struct { + State consensus.State `json:"state"` + Block types.Block `json:"block"` +} + // TxpoolUpdateV2TransactionsResponse is the response type for /txpool/transactions/v2/basis. type TxpoolUpdateV2TransactionsResponse struct { Basis types.ChainIndex `json:"basis"` diff --git a/api/api_test.go b/api/api_test.go index 8ab3c75..c978a4d 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -530,6 +530,47 @@ func TestConsensus(t *testing.T) { } } +func TestConsensusState(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + // mine a block + minedBlock, ok := coreutils.MineBlock(cn.Chain, types.Address{}, time.Minute) + if !ok { + t.Fatal("no block found") + } else if err := cn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { + t.Fatal(err) + } + + // block should be tip now + ci, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } else if ci.ID != minedBlock.ID() { + t.Fatalf("expected consensus tip to be %v, got %v", minedBlock.ID(), ci.ID) + } + + // fetch block + resp, err := c.ConsensusState(minedBlock.ID()) + if err != nil { + t.Fatal(err) + } else if resp.Block.ID() != minedBlock.ID() { + t.Fatal("mismatch") + } else if resp.State.Index != cn.Chain.Tip() { + t.Fatal("mismatch tip") + } +} + func TestConsensusUpdates(t *testing.T) { log := zaptest.NewLogger(t) diff --git a/api/client.go b/api/client.go index ff7590e..8c8fcb4 100644 --- a/api/client.go +++ b/api/client.go @@ -102,6 +102,17 @@ func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err err return } +// ConsensusState returns the consensus state of the specified block ID. +// The block must be in the best chain. +func (c *Client) ConsensusState(bid types.BlockID) (resp ConsensusStateResponse, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/state/%v", bid), &resp) + if err != nil { + return + } + resp.State.Network, err = c.getNetwork() + return +} + // ConsensusIndex returns the consensus index at the specified height. func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/index/%d", height), &resp) diff --git a/api/server.go b/api/server.go index 9fe7f7e..49c431e 100644 --- a/api/server.go +++ b/api/server.go @@ -62,6 +62,7 @@ type ( Tip() types.ChainIndex BestIndex(height uint64) (types.ChainIndex, bool) Block(id types.BlockID) (types.Block, bool) + State(id types.BlockID) (consensus.State, bool) TipState() consensus.State AddBlocks([]types.Block) error RecommendedFee() types.Currency @@ -184,6 +185,30 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { jc.Encode(s.cm.TipState()) } +func (s *server) consensusStateIDHandler(jc jape.Context) { + var bid types.BlockID + if jc.DecodeParam("id", &bid) != nil { + return + } + + block, found := s.cm.Block(bid) + if !found { + jc.Error(errors.New("couldn't find block"), http.StatusNotFound) + return + } + + state, found := s.cm.State(bid) + if !found { + jc.Error(errors.New("couldn't find state"), http.StatusNotFound) + return + } + + jc.Encode(ConsensusStateResponse{ + State: state, + Block: block, + }) +} + func (s *server) consensusBlocksIDHandler(jc jape.Context) { var bid types.BlockID if jc.DecodeParam("id", &bid) != nil { @@ -1442,6 +1467,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), "GET /consensus/tipstate": wrapPublicAuthHandler(srv.consensusTipStateHandler), + "GET /consensus/state/:id": wrapPublicAuthHandler(srv.consensusStateIDHandler), "GET /consensus/blocks/:id": wrapPublicAuthHandler(srv.consensusBlocksIDHandler), "GET /consensus/updates/:index": wrapPublicAuthHandler(srv.consensusUpdatesIndexHandler), "GET /consensus/index/:height": wrapPublicAuthHandler(srv.consensusIndexHeightHandler), From 2e9d32a40ac44a35b44db71ce146bc7c6b909746 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 13 May 2025 14:04:54 -0700 Subject: [PATCH 436/630] api state -> checkpoint --- ..._endpoint_to_return_the_block_and_its_consensus_state.md | 2 +- api/api.go | 4 ++-- api/api_test.go | 4 ++-- api/client.go | 6 +++--- api/server.go | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md b/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md index 27363fe..49ae972 100644 --- a/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md +++ b/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md @@ -2,4 +2,4 @@ default: minor --- -# Add GET /consensus/state/:id endpoint to return a block and its consensus state +# Added GET /consensus/checkpoint/:id which returns the block and its consensus state. diff --git a/api/api.go b/api/api.go index 2be58c9..91c211e 100644 --- a/api/api.go +++ b/api/api.go @@ -56,8 +56,8 @@ type TxpoolUpdateV2TransactionsRequest struct { Transactions []types.V2Transaction `json:"transactions"` } -// ConsensusStateResponse is the response type for GET /consensus/state/:id. -type ConsensusStateResponse struct { +// ConsensusCheckpointResponse is the response type for GET /consensus/state/:id. +type ConsensusCheckpointResponse struct { State consensus.State `json:"state"` Block types.Block `json:"block"` } diff --git a/api/api_test.go b/api/api_test.go index c978a4d..6a0f7d4 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -530,7 +530,7 @@ func TestConsensus(t *testing.T) { } } -func TestConsensusState(t *testing.T) { +func TestConsensusCheckpoint(t *testing.T) { log := zaptest.NewLogger(t) n, genesisBlock := testutil.V2Network() @@ -561,7 +561,7 @@ func TestConsensusState(t *testing.T) { } // fetch block - resp, err := c.ConsensusState(minedBlock.ID()) + resp, err := c.ConsensusCheckpoint(minedBlock.ID()) if err != nil { t.Fatal(err) } else if resp.Block.ID() != minedBlock.ID() { diff --git a/api/client.go b/api/client.go index 8c8fcb4..e78cf68 100644 --- a/api/client.go +++ b/api/client.go @@ -102,10 +102,10 @@ func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err err return } -// ConsensusState returns the consensus state of the specified block ID. +// ConsensusCheckpoint returns the consensus state of the specified block ID. // The block must be in the best chain. -func (c *Client) ConsensusState(bid types.BlockID) (resp ConsensusStateResponse, err error) { - err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/state/%v", bid), &resp) +func (c *Client) ConsensusCheckpoint(bid types.BlockID) (resp ConsensusCheckpointResponse, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/checkpoint/%v", bid), &resp) if err != nil { return } diff --git a/api/server.go b/api/server.go index 49c431e..bcbfc1b 100644 --- a/api/server.go +++ b/api/server.go @@ -185,7 +185,7 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { jc.Encode(s.cm.TipState()) } -func (s *server) consensusStateIDHandler(jc jape.Context) { +func (s *server) consensusCheckpointIDHandler(jc jape.Context) { var bid types.BlockID if jc.DecodeParam("id", &bid) != nil { return @@ -203,7 +203,7 @@ func (s *server) consensusStateIDHandler(jc jape.Context) { return } - jc.Encode(ConsensusStateResponse{ + jc.Encode(ConsensusCheckpointResponse{ State: state, Block: block, }) @@ -1467,7 +1467,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), "GET /consensus/tipstate": wrapPublicAuthHandler(srv.consensusTipStateHandler), - "GET /consensus/state/:id": wrapPublicAuthHandler(srv.consensusStateIDHandler), + "GET /consensus/checkpoint/:id": wrapPublicAuthHandler(srv.consensusCheckpointIDHandler), "GET /consensus/blocks/:id": wrapPublicAuthHandler(srv.consensusBlocksIDHandler), "GET /consensus/updates/:index": wrapPublicAuthHandler(srv.consensusUpdatesIndexHandler), "GET /consensus/index/:height": wrapPublicAuthHandler(srv.consensusIndexHeightHandler), From caed9863879f402b8c6e08177b90e1f7aabd7cf5 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 13 May 2025 14:05:46 -0700 Subject: [PATCH 437/630] state -> checkpoint --- api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api.go b/api/api.go index 91c211e..b459d2b 100644 --- a/api/api.go +++ b/api/api.go @@ -56,7 +56,7 @@ type TxpoolUpdateV2TransactionsRequest struct { Transactions []types.V2Transaction `json:"transactions"` } -// ConsensusCheckpointResponse is the response type for GET /consensus/state/:id. +// ConsensusCheckpointResponse is the response type for GET /consensus/checkpoint/:id. type ConsensusCheckpointResponse struct { State consensus.State `json:"state"` Block types.Block `json:"block"` From a5f78778f64b9ddc2c873c74f0c726991c1300c4 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 14 May 2025 14:49:33 +0200 Subject: [PATCH 438/630] return ids of broadcasted txns --- api/api.go | 6 ++++++ api/api_test.go | 36 ++++++++++++++++++++++-------------- api/client.go | 7 ++++--- api/server.go | 9 ++++++++- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/api/api.go b/api/api.go index b459d2b..e664e17 100644 --- a/api/api.go +++ b/api/api.go @@ -42,6 +42,12 @@ type TxpoolBroadcastRequest struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } +// TxpoolBroadcastRequest is the response type for /txpool/broadcast. +type TxpoolBroadcastResponse struct { + Transactions []types.TransactionID `json:"transactions"` + V2Transactions []types.TransactionID `json:"v2transactions"` +} + // TxpoolTransactionsResponse is the response type for /txpool/transactions. type TxpoolTransactionsResponse struct { Basis types.ChainIndex `json:"basis"` diff --git a/api/api_test.go b/api/api_test.go index 6a0f7d4..fc89f32 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -299,7 +299,7 @@ func TestWallet(t *testing.T) { txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + if _, _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } @@ -430,7 +430,7 @@ func TestAddresses(t *testing.T) { txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + if _, _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -711,8 +711,12 @@ func TestConstructSiacoins(t *testing.T) { resp.Transaction.Signatures[i].Signature = sig[:] } - if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + if v1IDs, v2IDs, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) + } else if len(v1IDs) != 1 || len(v2IDs) != 0 { + t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", v1IDs, v2IDs) + } else if v1IDs[0] != resp.ID { + t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v1IDs[0]) } unconfirmed, err := wc.UnconfirmedEvents() @@ -828,7 +832,7 @@ func TestConstructSiafunds(t *testing.T) { resp.Transaction.Signatures[i].Signature = sig[:] } - if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + if _, _, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) } @@ -971,8 +975,12 @@ func TestConstructV2Siacoins(t *testing.T) { resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + if v1IDs, v2IDs, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) + } else if len(v1IDs) != 1 || len(v2IDs) != 0 { + t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", v1IDs, v2IDs) + } else if v1IDs[0] != resp.ID { + t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v1IDs[0]) } unconfirmed, err := wc.UnconfirmedEvents() @@ -1074,7 +1082,7 @@ func TestConstructV2Siafunds(t *testing.T) { resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + if _, _, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) } @@ -1183,7 +1191,7 @@ func TestSpentElement(t *testing.T) { senderPrivateKey.SignHash(cs.InputSigHash(txn)), } - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1251,7 +1259,7 @@ func TestSpentElement(t *testing.T) { senderPrivateKey.SignHash(cs.InputSigHash(txn)), } - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1475,7 +1483,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } // broadcast the transaction - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1516,7 +1524,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { t.Fatalf("expected basis to be %v, got %v", tip, basis) } - if err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1599,7 +1607,7 @@ func TestAddressTPool(t *testing.T) { pk.SignHash(sigHash), } - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } @@ -1661,7 +1669,7 @@ func TestEphemeralTransactions(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} expectedOutputID := txn.SiacoinOutputID(txn.ID(), 1) - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } @@ -1709,7 +1717,7 @@ func TestEphemeralTransactions(t *testing.T) { t.Fatalf("expected siacoin element to have leaf index, got %v", sces[0].StateElement.LeafIndex) } - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn2}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn2}); err != nil { t.Fatal(err) } @@ -1794,7 +1802,7 @@ func TestBroadcastRace(t *testing.T) { sigHash := cs.InputSigHash(txn) txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} t.Log("broadcasting", txn.ID(), cs.Index) - if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } } diff --git a/api/client.go b/api/client.go index e78cf68..4d5648b 100644 --- a/api/client.go +++ b/api/client.go @@ -47,13 +47,14 @@ func (c *Client) State() (resp StateResponse, err error) { } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (err error) { +func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (v1IDs, v2IDs []types.TransactionID, err error) { + var resp TxpoolBroadcastResponse err = c.c.POST(context.Background(), "/txpool/broadcast", TxpoolBroadcastRequest{ Basis: basis, Transactions: txns, V2Transactions: v2txns, - }, nil) - return + }, &resp) + return resp.Transactions, resp.V2Transactions, err } // TxpoolTransactions returns all transactions in the transaction pool. diff --git a/api/server.go b/api/server.go index bcbfc1b..61920ab 100644 --- a/api/server.go +++ b/api/server.go @@ -354,6 +354,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Decode(&tbr) != nil { return } + var resp TxpoolBroadcastResponse if len(tbr.Transactions) != 0 { if len(tbr.Transactions) == 1 { // if there's only one transaction, best-effort check for parents @@ -368,6 +369,9 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { return } + for _, txn := range tbr.Transactions { + resp.Transactions = append(resp.Transactions, txn.ID()) + } } if len(tbr.V2Transactions) != 0 { if len(tbr.V2Transactions) == 1 { @@ -386,9 +390,12 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { return } + for _, txn := range tbr.V2Transactions { + resp.V2Transactions = append(resp.V2Transactions, txn.ID()) + } } - jc.Encode(nil) + jc.Encode(resp) } func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { From dd236bce9593254011eb33fffa1abb01f130d5e3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 14 May 2025 14:51:45 +0200 Subject: [PATCH 439/630] document change --- .../return_ids_of_transaction_from_txn_broadcast_endpoint.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md diff --git a/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md b/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md new file mode 100644 index 0000000..de3de3f --- /dev/null +++ b/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Return IDs of transaction from txn broadcast endpoint. From a6d95b59748efdcc3a69ce38eae66d56580a3ed4 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 14 May 2025 14:52:27 +0200 Subject: [PATCH 440/630] fix build --- api/construct_test.go | 2 +- api/construct_v2_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/construct_test.go b/api/construct_test.go index 2c43efe..f6d287e 100644 --- a/api/construct_test.go +++ b/api/construct_test.go @@ -77,7 +77,7 @@ func ExampleWalletClient_Construct() { } // broadcast the transaction - if err := client.TxpoolBroadcast(resp.Basis, []types.Transaction{txn}, nil); err != nil { + if _, _, err := client.TxpoolBroadcast(resp.Basis, []types.Transaction{txn}, nil); err != nil { panic(err) } } diff --git a/api/construct_v2_test.go b/api/construct_v2_test.go index e3c9d78..01f6a33 100644 --- a/api/construct_v2_test.go +++ b/api/construct_v2_test.go @@ -77,7 +77,7 @@ func ExampleWalletClient_ConstructV2() { } // broadcast the transaction - if err := client.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{txn}); err != nil { + if _, _, err := client.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{txn}); err != nil { panic(err) } } From 348000542ca1a27fa50031c33c76bbc5702dcce2 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 14 May 2025 14:53:41 +0200 Subject: [PATCH 441/630] fix lint --- api/api.go | 2 +- api/api_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/api.go b/api/api.go index e664e17..b266362 100644 --- a/api/api.go +++ b/api/api.go @@ -42,7 +42,7 @@ type TxpoolBroadcastRequest struct { V2Transactions []types.V2Transaction `json:"v2transactions"` } -// TxpoolBroadcastRequest is the response type for /txpool/broadcast. +// TxpoolBroadcastResponse is the response type for /txpool/broadcast. type TxpoolBroadcastResponse struct { Transactions []types.TransactionID `json:"transactions"` V2Transactions []types.TransactionID `json:"v2transactions"` diff --git a/api/api_test.go b/api/api_test.go index fc89f32..af97db8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -977,10 +977,10 @@ func TestConstructV2Siacoins(t *testing.T) { if v1IDs, v2IDs, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) - } else if len(v1IDs) != 1 || len(v2IDs) != 0 { + } else if len(v1IDs) != 0 || len(v2IDs) != 1 { t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", v1IDs, v2IDs) - } else if v1IDs[0] != resp.ID { - t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v1IDs[0]) + } else if v2IDs[0] != resp.ID { + t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v2IDs[0]) } unconfirmed, err := wc.UnconfirmedEvents() From 2c190a32d8ea50a57576a457b08b1d0e7a97eb43 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 14 May 2025 15:01:44 +0200 Subject: [PATCH 442/630] fix TestAPINoContent --- api/api_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index af97db8..f0667ed 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1408,14 +1408,11 @@ func TestAPINoContent(t *testing.T) { cn := testutil.NewConsensusNode(t, n, genesisBlock, log) c := startWalletServer(t, cn, log) - buf, err := json.Marshal(api.TxpoolBroadcastRequest{ - Transactions: []types.Transaction{}, - V2Transactions: []types.V2Transaction{}, - }) + buf, err := json.Marshal(cn.Chain.Tip().Height) if err != nil { t.Fatal(err) } - req, err := http.NewRequest(http.MethodPost, c.BaseURL()+"/txpool/broadcast", bytes.NewReader(buf)) + req, err := http.NewRequest(http.MethodPost, c.BaseURL()+"/rescan", bytes.NewReader(buf)) if err != nil { t.Fatal(err) } From 0dc0fa80f893669676af657bdeea9287b369bfe8 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 10:05:38 -0700 Subject: [PATCH 443/630] update broadcast response --- ...transaction_from_txn_broadcast_endpoint.md | 4 +- ...te_core_to_v0122_and_coreutils_to_v0134.md | 7 +++ api/api.go | 5 +-- api/api_test.go | 44 +++++++++---------- api/client.go | 5 +-- api/construct_test.go | 2 +- api/construct_v2_test.go | 2 +- api/server.go | 12 ++--- go.mod | 8 ++-- go.sum | 16 +++---- wallet/addresses.go | 2 + 11 files changed, 55 insertions(+), 52 deletions(-) create mode 100644 .changeset/update_core_to_v0122_and_coreutils_to_v0134.md diff --git a/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md b/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md index de3de3f..ac349b2 100644 --- a/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md +++ b/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md @@ -2,4 +2,6 @@ default: minor --- -# Return IDs of transaction from txn broadcast endpoint. +# Return transaction sets from broadcast endpoint + +This lets integrators get the IDs of the created UTXOs, addresses of inputs, and IDs of the transactions diff --git a/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md b/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md new file mode 100644 index 0000000..be007ea --- /dev/null +++ b/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md @@ -0,0 +1,7 @@ +--- +default: patch +--- + +# Update core to v0.12.2 and coreutils to v0.13.4 + +These releases include additional JSON convenience fields \ No newline at end of file diff --git a/api/api.go b/api/api.go index b266362..d9142fe 100644 --- a/api/api.go +++ b/api/api.go @@ -43,10 +43,7 @@ type TxpoolBroadcastRequest struct { } // TxpoolBroadcastResponse is the response type for /txpool/broadcast. -type TxpoolBroadcastResponse struct { - Transactions []types.TransactionID `json:"transactions"` - V2Transactions []types.TransactionID `json:"v2transactions"` -} +type TxpoolBroadcastResponse TxpoolBroadcastRequest // TxpoolTransactionsResponse is the response type for /txpool/transactions. type TxpoolTransactionsResponse struct { diff --git a/api/api_test.go b/api/api_test.go index f0667ed..17ef6a2 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -299,7 +299,7 @@ func TestWallet(t *testing.T) { txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if _, _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + if _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } @@ -430,7 +430,7 @@ func TestAddresses(t *testing.T) { txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool - if _, _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + if _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -711,12 +711,12 @@ func TestConstructSiacoins(t *testing.T) { resp.Transaction.Signatures[i].Signature = sig[:] } - if v1IDs, v2IDs, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + if broadcastResp, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) - } else if len(v1IDs) != 1 || len(v2IDs) != 0 { - t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", v1IDs, v2IDs) - } else if v1IDs[0] != resp.ID { - t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v1IDs[0]) + } else if len(broadcastResp.Transactions) != 1 || len(broadcastResp.V2Transactions) != 0 { + t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", len(broadcastResp.Transactions), len(broadcastResp.V2Transactions)) + } else if broadcastResp.Transactions[0].ID() != resp.ID { + t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, broadcastResp.Transactions[0].ID()) } unconfirmed, err := wc.UnconfirmedEvents() @@ -832,7 +832,7 @@ func TestConstructSiafunds(t *testing.T) { resp.Transaction.Signatures[i].Signature = sig[:] } - if _, _, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + if _, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) } @@ -975,12 +975,12 @@ func TestConstructV2Siacoins(t *testing.T) { resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - if v1IDs, v2IDs, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + if broadcastResp, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) - } else if len(v1IDs) != 0 || len(v2IDs) != 1 { - t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", v1IDs, v2IDs) - } else if v2IDs[0] != resp.ID { - t.Fatalf("expected v1 ID to be %v, got %v", resp.ID, v2IDs[0]) + } else if len(broadcastResp.Transactions) != 0 || len(broadcastResp.V2Transactions) != 1 { + t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", len(broadcastResp.Transactions), len(broadcastResp.V2Transactions)) + } else if broadcastResp.V2Transactions[0].ID() != resp.ID { + t.Fatalf("expected v2 ID to be %v, got %v", resp.ID, broadcastResp.V2Transactions[0].ID()) } unconfirmed, err := wc.UnconfirmedEvents() @@ -1082,7 +1082,7 @@ func TestConstructV2Siafunds(t *testing.T) { resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} } - if _, _, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + if _, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) } @@ -1191,7 +1191,7 @@ func TestSpentElement(t *testing.T) { senderPrivateKey.SignHash(cs.InputSigHash(txn)), } - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1259,7 +1259,7 @@ func TestSpentElement(t *testing.T) { senderPrivateKey.SignHash(cs.InputSigHash(txn)), } - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1480,7 +1480,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } // broadcast the transaction - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1521,7 +1521,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { t.Fatalf("expected basis to be %v, got %v", tip, basis) } - if _, _, err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) @@ -1604,7 +1604,7 @@ func TestAddressTPool(t *testing.T) { pk.SignHash(sigHash), } - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } @@ -1666,7 +1666,7 @@ func TestEphemeralTransactions(t *testing.T) { txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} expectedOutputID := txn.SiacoinOutputID(txn.ID(), 1) - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } @@ -1714,7 +1714,7 @@ func TestEphemeralTransactions(t *testing.T) { t.Fatalf("expected siacoin element to have leaf index, got %v", sces[0].StateElement.LeafIndex) } - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn2}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn2}); err != nil { t.Fatal(err) } @@ -1799,7 +1799,7 @@ func TestBroadcastRace(t *testing.T) { sigHash := cs.InputSigHash(txn) txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} t.Log("broadcasting", txn.ID(), cs.Index) - if _, _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } } diff --git a/api/client.go b/api/client.go index 4d5648b..7bed003 100644 --- a/api/client.go +++ b/api/client.go @@ -47,14 +47,13 @@ func (c *Client) State() (resp StateResponse, err error) { } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (v1IDs, v2IDs []types.TransactionID, err error) { - var resp TxpoolBroadcastResponse +func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (resp TxpoolBroadcastResponse, err error) { err = c.c.POST(context.Background(), "/txpool/broadcast", TxpoolBroadcastRequest{ Basis: basis, Transactions: txns, V2Transactions: v2txns, }, &resp) - return resp.Transactions, resp.V2Transactions, err + return } // TxpoolTransactions returns all transactions in the transaction pool. diff --git a/api/construct_test.go b/api/construct_test.go index f6d287e..1ad5047 100644 --- a/api/construct_test.go +++ b/api/construct_test.go @@ -77,7 +77,7 @@ func ExampleWalletClient_Construct() { } // broadcast the transaction - if _, _, err := client.TxpoolBroadcast(resp.Basis, []types.Transaction{txn}, nil); err != nil { + if _, err := client.TxpoolBroadcast(resp.Basis, []types.Transaction{txn}, nil); err != nil { panic(err) } } diff --git a/api/construct_v2_test.go b/api/construct_v2_test.go index 01f6a33..0090888 100644 --- a/api/construct_v2_test.go +++ b/api/construct_v2_test.go @@ -77,7 +77,7 @@ func ExampleWalletClient_ConstructV2() { } // broadcast the transaction - if _, _, err := client.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{txn}); err != nil { + if _, err := client.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{txn}); err != nil { panic(err) } } diff --git a/api/server.go b/api/server.go index 61920ab..aecfdcb 100644 --- a/api/server.go +++ b/api/server.go @@ -354,7 +354,6 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Decode(&tbr) != nil { return } - var resp TxpoolBroadcastResponse if len(tbr.Transactions) != 0 { if len(tbr.Transactions) == 1 { // if there's only one transaction, best-effort check for parents @@ -369,9 +368,6 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { return } - for _, txn := range tbr.Transactions { - resp.Transactions = append(resp.Transactions, txn.ID()) - } } if len(tbr.V2Transactions) != 0 { if len(tbr.V2Transactions) == 1 { @@ -390,12 +386,12 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { return } - for _, txn := range tbr.V2Transactions { - resp.V2Transactions = append(resp.V2Transactions, txn.ID()) - } } - jc.Encode(resp) + // the transactions are sent back to the client because the + // transaction set may have been modified and the transactions + // include additional convenience fields when being marshalled + jc.Encode(TxpoolBroadcastResponse(tbr)) } func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { diff --git a/go.mod b/go.mod index 23ed246..87886af 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.12.0 - go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2 + go.sia.tech/core v0.12.2 + go.sia.tech/coreutils v0.13.4 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 @@ -33,9 +33,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.38.0 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.39.0 // indirect + golang.org/x/net v0.40.0 // indirect golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.32.0 // indirect + golang.org/x/tools v0.33.0 // indirect ) diff --git a/go.sum b/go.sum index 24a5799..f5a2d67 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.12.0 h1:nuHjUE3MnYPQ+BKo44DD64332jBGVZlH657c5RIomXw= -go.sia.tech/core v0.12.0/go.mod h1:ycpNTb9Y7Vtnq6HQ3iqOxeywLnjDs0udD3ZVq6KE13Y= -go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2 h1:2J6dj8JK2dzLlaMzPmu/DOBCeSMhTYeYQl65wUMSqlY= -go.sia.tech/coreutils v0.13.4-0.20250512154444-5fc127e81fc2/go.mod h1:10LIkoS//x5fn/dVuNUS/1xDyX4RjYNmGwMvwBpQEKg= +go.sia.tech/core v0.12.2 h1:G/4FPc5tULVsJ5pqUtz7mAwS3L3CP1ST0Y6DCXJMB30= +go.sia.tech/core v0.12.2/go.mod h1:dvApqsjl43EoYC2yKVYVrlFtgbR0WKE1rI1eMklEL/4= +go.sia.tech/coreutils v0.13.4 h1:yf2U1Qf4Ki+JYdyekNk74lZ/qcPPff6Z5uTPSv2g91U= +go.sia.tech/coreutils v0.13.4/go.mod h1:CF+GxKuORSshfQb4VOEt+YetIRvpx4HP+3sAl2aKOXg= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -67,8 +67,8 @@ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJ golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= @@ -77,8 +77,8 @@ golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/wallet/addresses.go b/wallet/addresses.go index 63cc478..a96cc4d 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -33,6 +33,7 @@ func (m *Manager) AddressSiacoinOutputs(address types.Address, usePool bool, off continue } + sce.StateElement = sce.StateElement.Copy() created = append(created, UnspentSiacoinElement{ SiacoinElement: sce, }) @@ -62,6 +63,7 @@ func (m *Manager) AddressSiafundOutputs(address types.Address, usePool bool, off if sfe.SiafundOutput.Address != address { continue } + sfe.StateElement = sfe.StateElement.Copy() created = append(created, UnspentSiafundElement{ SiafundElement: sfe, }) From cc7edb57c965e2b344fe6747ce1caa9795693567 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 10:21:26 -0700 Subject: [PATCH 444/630] update core and coreutils --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 87886af..51b708b 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.12.2 - go.sia.tech/coreutils v0.13.4 + go.sia.tech/core v0.12.3 + go.sia.tech/coreutils v0.13.5 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index f5a2d67..3ebac80 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.12.2 h1:G/4FPc5tULVsJ5pqUtz7mAwS3L3CP1ST0Y6DCXJMB30= -go.sia.tech/core v0.12.2/go.mod h1:dvApqsjl43EoYC2yKVYVrlFtgbR0WKE1rI1eMklEL/4= -go.sia.tech/coreutils v0.13.4 h1:yf2U1Qf4Ki+JYdyekNk74lZ/qcPPff6Z5uTPSv2g91U= -go.sia.tech/coreutils v0.13.4/go.mod h1:CF+GxKuORSshfQb4VOEt+YetIRvpx4HP+3sAl2aKOXg= +go.sia.tech/core v0.12.3 h1:p0BfsKfc7jVRKRDm2K9udxqBEqIGU0An54vP4WL+SH8= +go.sia.tech/core v0.12.3/go.mod h1:woUSWQdrRFXkYdk2g2532rguQ1EyMU+2/kcC4JnHSx4= +go.sia.tech/coreutils v0.13.5 h1:C3rzdb0sGQAhplKAZ8yDBIJkWnSxKe4Jeobhu8j8FSA= +go.sia.tech/coreutils v0.13.5/go.mod h1:dYnjkSU6qzztNDy8Sx/OS1wa0tyi4zjrtbRkVOKaDH4= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 82a57ae74b951e5af8f9c3404fe05d65e39cb41c Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 12:02:37 -0700 Subject: [PATCH 445/630] update deps --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 51b708b..9a768d7 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.12.3 - go.sia.tech/coreutils v0.13.5 + go.sia.tech/coreutils v0.13.6 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 3ebac80..1930b3a 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.12.3 h1:p0BfsKfc7jVRKRDm2K9udxqBEqIGU0An54vP4WL+SH8= go.sia.tech/core v0.12.3/go.mod h1:woUSWQdrRFXkYdk2g2532rguQ1EyMU+2/kcC4JnHSx4= -go.sia.tech/coreutils v0.13.5 h1:C3rzdb0sGQAhplKAZ8yDBIJkWnSxKe4Jeobhu8j8FSA= -go.sia.tech/coreutils v0.13.5/go.mod h1:dYnjkSU6qzztNDy8Sx/OS1wa0tyi4zjrtbRkVOKaDH4= +go.sia.tech/coreutils v0.13.6 h1:RNCrMRO2QA0f/0OMlV5ULjyKYK1KUH1qN0zu/SLvIQk= +go.sia.tech/coreutils v0.13.6/go.mod h1:JR8onVt1R9wz4kg/bzyGg75HuZyTbBVmt3OKcUac6Qo= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From b1ce31dba1e85aa3a9ffbd7da96b6d6ff21c860a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 14 May 2025 19:08:49 +0000 Subject: [PATCH 446/630] chore: prepare release 2.4.0 --- ...o_return_the_block_and_its_consensus_state.md | 5 ----- ...of_transaction_from_txn_broadcast_endpoint.md | 7 ------- ...pdate_core_to_v0122_and_coreutils_to_v0134.md | 7 ------- CHANGELOG.md | 16 ++++++++++++++++ go.mod | 2 +- 5 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 .changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md delete mode 100644 .changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md delete mode 100644 .changeset/update_core_to_v0122_and_coreutils_to_v0134.md diff --git a/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md b/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md deleted file mode 100644 index 49ae972..0000000 --- a/.changeset/add_get_consensusstateid_endpoint_to_return_the_block_and_its_consensus_state.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Added GET /consensus/checkpoint/:id which returns the block and its consensus state. diff --git a/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md b/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md deleted file mode 100644 index ac349b2..0000000 --- a/.changeset/return_ids_of_transaction_from_txn_broadcast_endpoint.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Return transaction sets from broadcast endpoint - -This lets integrators get the IDs of the created UTXOs, addresses of inputs, and IDs of the transactions diff --git a/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md b/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md deleted file mode 100644 index be007ea..0000000 --- a/.changeset/update_core_to_v0122_and_coreutils_to_v0134.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: patch ---- - -# Update core to v0.12.2 and coreutils to v0.13.4 - -These releases include additional JSON convenience fields \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 989a313..83f155a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## 2.4.0 (2025-05-14) + +### Features + +- Added GET /consensus/checkpoint/:id which returns the block and its consensus state. + +#### Return transaction sets from broadcast endpoint + +This lets integrators get the IDs of the created UTXOs, addresses of inputs, and IDs of the transactions + +### Fixes + +#### Update core to v0.12.2 and coreutils to v0.13.4 + +These releases include additional JSON convenience fields + ## 2.3.0 (2025-05-12) ### Features diff --git a/go.mod b/go.mod index 9a768d7..a4cf63a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.3.0 +module go.sia.tech/walletd/v2 // v2.4.0 go 1.23.2 From bbde2639d00a9d9734f3068cc60415e61e3bd9f5 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 12:39:53 -0700 Subject: [PATCH 447/630] fix race --- .changeset/fix_race_in_txpool_broadcast.md | 5 ++++ api/api.go | 6 ++++- api/api_test.go | 1 - api/server.go | 31 +++++++++++++++------- 4 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 .changeset/fix_race_in_txpool_broadcast.md diff --git a/.changeset/fix_race_in_txpool_broadcast.md b/.changeset/fix_race_in_txpool_broadcast.md new file mode 100644 index 0000000..8995e84 --- /dev/null +++ b/.changeset/fix_race_in_txpool_broadcast.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fix race in txpool broadcast diff --git a/api/api.go b/api/api.go index d9142fe..ba17421 100644 --- a/api/api.go +++ b/api/api.go @@ -43,7 +43,11 @@ type TxpoolBroadcastRequest struct { } // TxpoolBroadcastResponse is the response type for /txpool/broadcast. -type TxpoolBroadcastResponse TxpoolBroadcastRequest +type TxpoolBroadcastResponse struct { + Basis types.ChainIndex `json:"basis"` + Transactions []types.Transaction `json:"transactions"` + V2Transactions []types.V2Transaction `json:"v2transactions"` +} // TxpoolTransactionsResponse is the response type for /txpool/transactions. type TxpoolTransactionsResponse struct { diff --git a/api/api_test.go b/api/api_test.go index 17ef6a2..03b8cd3 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1798,7 +1798,6 @@ func TestBroadcastRace(t *testing.T) { } sigHash := cs.InputSigHash(txn) txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} - t.Log("broadcasting", txn.ID(), cs.Index) if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } diff --git a/api/server.go b/api/server.go index aecfdcb..8f77fe0 100644 --- a/api/server.go +++ b/api/server.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/pprof" "runtime" + "slices" "sync" "time" @@ -354,18 +355,27 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if jc.Decode(&tbr) != nil { return } + + // the transactions are sent back to the client because the + // transaction set may have been modified and the transactions + // include additional convenience fields when being marshalled + resp := TxpoolBroadcastResponse{ + Basis: tbr.Basis, + } if len(tbr.Transactions) != 0 { if len(tbr.Transactions) == 1 { // if there's only one transaction, best-effort check for parents tbr.Transactions = append(s.cm.UnconfirmedParents(tbr.Transactions[0]), tbr.Transactions...) } + // prevents a race condition when encoding the transactions + // TODO: fix this race + resp.Transactions = slices.Clone(resp.Transactions) _, err := s.cm.AddPoolTransactions(tbr.Transactions) if err != nil { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return - } - if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { + } else if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { return } } @@ -379,19 +389,22 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } } + // prevents a race condition when encoding the transactions + // TODO: fix this race + resp.V2Transactions = make([]types.V2Transaction, 0, len(tbr.V2Transactions)) + for _, txn := range tbr.V2Transactions { + resp.V2Transactions = append(resp.V2Transactions, txn.DeepCopy()) + } + if _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions); err != nil { jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return - } - if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { + } else if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { return } - } - // the transactions are sent back to the client because the - // transaction set may have been modified and the transactions - // include additional convenience fields when being marshalled - jc.Encode(TxpoolBroadcastResponse(tbr)) + } + jc.Encode(resp) } func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { From 4502472e96ee65145822bb5ec09a465d1ec45b0b Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 12:41:40 -0700 Subject: [PATCH 448/630] clone proper slice --- api/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server.go b/api/server.go index 8f77fe0..48acbd7 100644 --- a/api/server.go +++ b/api/server.go @@ -370,7 +370,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { // prevents a race condition when encoding the transactions // TODO: fix this race - resp.Transactions = slices.Clone(resp.Transactions) + resp.Transactions = slices.Clone(tbr.Transactions) _, err := s.cm.AddPoolTransactions(tbr.Transactions) if err != nil { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) From 7abe6c17c097f013c6934e62e0861630969d8657 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 May 2025 12:42:24 -0700 Subject: [PATCH 449/630] fix lint --- api/server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/server.go b/api/server.go index 48acbd7..ad0264a 100644 --- a/api/server.go +++ b/api/server.go @@ -402,7 +402,6 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } else if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { return } - } jc.Encode(resp) } From 084338d70d8d4d2064d138d1f5cb0f02491496b8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 14 May 2025 19:45:23 +0000 Subject: [PATCH 450/630] chore: prepare release 2.4.1 --- .changeset/fix_race_in_txpool_broadcast.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/fix_race_in_txpool_broadcast.md diff --git a/.changeset/fix_race_in_txpool_broadcast.md b/.changeset/fix_race_in_txpool_broadcast.md deleted file mode 100644 index 8995e84..0000000 --- a/.changeset/fix_race_in_txpool_broadcast.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fix race in txpool broadcast diff --git a/CHANGELOG.md b/CHANGELOG.md index 83f155a..5f6db92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.4.1 (2025-05-14) + +### Fixes + +- Fix race in txpool broadcast + ## 2.4.0 (2025-05-14) ### Features diff --git a/go.mod b/go.mod index a4cf63a..7bdd9a9 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.4.0 +module go.sia.tech/walletd/v2 // v2.4.1 go 1.23.2 From 4ccdfc5fd6864d3b7f65696fc7fcb75261592fad Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 15 May 2025 11:11:23 -0700 Subject: [PATCH 451/630] lookup by height --- ...endpoints_now_support_lookups_by_height.md | 5 ++ api/server.go | 46 ++++++++++++++++--- 2 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 .changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md diff --git a/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md b/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md new file mode 100644 index 0000000..9719b5c --- /dev/null +++ b/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Consensus checkpoint and block endpoints now support lookups by height diff --git a/api/server.go b/api/server.go index ad0264a..443a0b3 100644 --- a/api/server.go +++ b/api/server.go @@ -8,6 +8,7 @@ import ( "net/http/pprof" "runtime" "slices" + "strconv" "sync" "time" @@ -187,18 +188,33 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { } func (s *server) consensusCheckpointIDHandler(jc jape.Context) { - var bid types.BlockID - if jc.DecodeParam("id", &bid) != nil { + var param string + if jc.DecodeParam("id", ¶m) != nil { return } - block, found := s.cm.Block(bid) + var id types.BlockID + if height, err := strconv.ParseUint(param, 10, 64); err == nil { + index, ok := s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + id = index.ID + } else { + if err := id.UnmarshalText([]byte(param)); err != nil { + jc.Error(fmt.Errorf("invalid block ID: %w", err), http.StatusBadRequest) + return + } + } + + block, found := s.cm.Block(id) if !found { jc.Error(errors.New("couldn't find block"), http.StatusNotFound) return } - state, found := s.cm.State(bid) + state, found := s.cm.State(id) if !found { jc.Error(errors.New("couldn't find state"), http.StatusNotFound) return @@ -211,11 +227,27 @@ func (s *server) consensusCheckpointIDHandler(jc jape.Context) { } func (s *server) consensusBlocksIDHandler(jc jape.Context) { - var bid types.BlockID - if jc.DecodeParam("id", &bid) != nil { + var param string + if jc.DecodeParam("id", ¶m) != nil { return } - block, found := s.cm.Block(bid) + + var id types.BlockID + if height, err := strconv.ParseUint(param, 10, 64); err == nil { + index, ok := s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + id = index.ID + } else { + if err := id.UnmarshalText([]byte(param)); err != nil { + jc.Error(fmt.Errorf("invalid block ID: %w", err), http.StatusBadRequest) + return + } + } + + block, found := s.cm.Block(id) if !found { jc.Error(errors.New("couldn't find block"), http.StatusNotFound) return From 71630e5f1fce14a3c61e79719742d01be1a9d9cc Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 15 May 2025 11:13:49 -0700 Subject: [PATCH 452/630] add client methods for height lookups --- api/api_test.go | 11 ++++++++++- api/client.go | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 03b8cd3..de47668 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -561,7 +561,7 @@ func TestConsensusCheckpoint(t *testing.T) { } // fetch block - resp, err := c.ConsensusCheckpoint(minedBlock.ID()) + resp, err := c.ConsensusCheckpointID(minedBlock.ID()) if err != nil { t.Fatal(err) } else if resp.Block.ID() != minedBlock.ID() { @@ -569,6 +569,15 @@ func TestConsensusCheckpoint(t *testing.T) { } else if resp.State.Index != cn.Chain.Tip() { t.Fatal("mismatch tip") } + + heightResp, err := c.ConsensusCheckpointHeight(cn.Chain.Tip().Height) + if err != nil { + t.Fatal(err) + } else if heightResp.Block.ID() != minedBlock.ID() { + t.Fatal("mismatch") + } else if heightResp.State.Index != cn.Chain.Tip() { + t.Fatal("mismatch tip") + } } func TestConsensusUpdates(t *testing.T) { diff --git a/api/client.go b/api/client.go index 7bed003..2779e80 100644 --- a/api/client.go +++ b/api/client.go @@ -102,9 +102,15 @@ func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err err return } -// ConsensusCheckpoint returns the consensus state of the specified block ID. +// ConsensusBlocksHeight returns the block with the given height. +func (c *Client) ConsensusBlocksHeight(height uint64) (resp types.Block, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/blocks/%d", height), &resp) + return +} + +// ConsensusCheckpointID returns the consensus state of the specified block ID. // The block must be in the best chain. -func (c *Client) ConsensusCheckpoint(bid types.BlockID) (resp ConsensusCheckpointResponse, err error) { +func (c *Client) ConsensusCheckpointID(bid types.BlockID) (resp ConsensusCheckpointResponse, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/checkpoint/%v", bid), &resp) if err != nil { return @@ -113,6 +119,16 @@ func (c *Client) ConsensusCheckpoint(bid types.BlockID) (resp ConsensusCheckpoin return } +// ConsensusCheckpointHeight returns the consensus state and block at the specified height. +func (c *Client) ConsensusCheckpointHeight(height uint64) (resp ConsensusCheckpointResponse, err error) { + err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/checkpoint/%d", height), &resp) + if err != nil { + return + } + resp.State.Network, err = c.getNetwork() + return +} + // ConsensusIndex returns the consensus index at the specified height. func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/index/%d", height), &resp) From b4561e158c9a1b8cefdf6771ce72af437be5fb00 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 16 May 2025 13:42:18 -0700 Subject: [PATCH 453/630] add health check --- ...to_check_the_health_of_the_walletd_node.md | 5 ++ api/server.go | 13 ++++- go.mod | 1 + wallet/manager.go | 47 +++++++++++++++++++ wallet/manager_test.go | 41 ++++++++++++---- 5 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 .changeset/added_get_health_to_check_the_health_of_the_walletd_node.md diff --git a/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md b/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md new file mode 100644 index 0000000..5d3afcb --- /dev/null +++ b/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Added [GET] /health to check the health of the walletd node diff --git a/api/server.go b/api/server.go index 443a0b3..39a4862 100644 --- a/api/server.go +++ b/api/server.go @@ -91,6 +91,8 @@ type ( // A WalletManager manages wallets, keyed by name. WalletManager interface { + Health() error + IndexMode() wallet.IndexMode Tip() (types.ChainIndex, error) Scan(_ context.Context, index types.ChainIndex) error @@ -175,6 +177,14 @@ func (s *server) stateHandler(jc jape.Context) { }) } +func (s *server) healthHandler(jc jape.Context) { + if err := s.wm.Health(); err != nil { + jc.Error(err, http.StatusInternalServerError) + return + } + jc.Encode(nil) +} + func (s *server) consensusNetworkHandler(jc jape.Context) { jc.Encode(*s.cm.TipState().Network) } @@ -1509,7 +1519,8 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption } handlers := map[string]jape.Handler{ - "GET /state": wrapPublicAuthHandler(srv.stateHandler), + "GET /state": wrapPublicAuthHandler(srv.stateHandler), + "GET /health": wrapPublicAuthHandler(srv.healthHandler), "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), diff --git a/go.mod b/go.mod index 7bdd9a9..51448bf 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.32.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 diff --git a/wallet/manager.go b/wallet/manager.go index 4b22913..3e25c47 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -1,6 +1,7 @@ package wallet import ( + "cmp" "context" "errors" "fmt" @@ -9,12 +10,16 @@ import ( "sync" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/walletd/v2/internal/threadgroup" "go.uber.org/zap" + "golang.org/x/exp/constraints" ) +const maxReorgPeriod = 3 * time.Hour + // IndexMode represents the index mode of the wallet manager. The index mode // determines how the wallet manager stores the consensus state. // @@ -44,6 +49,11 @@ var ( // ErrAlreadyReserved is returned when trying to reserve an output that is // already reserved. ErrAlreadyReserved = errors.New("output already reserved") + // ErrNotSyncing is returned when the consensus has not + // had a block change within the last 3 hours. + ErrNotSyncing = errors.New("not syncing") + // ErrNotSynced is returned when the wallet has not + ErrNotSynced = errors.New("not synced") ) type ( @@ -56,6 +66,7 @@ type ( V2PoolTransactions() []types.V2Transaction Tip() types.ChainIndex + TipState() consensus.State BestIndex(height uint64) (types.ChainIndex, bool) OnReorg(func(types.ChainIndex)) (cancel func()) @@ -197,6 +208,35 @@ func (m *Manager) utxosLocked(ids ...types.Hash256) error { return nil } +// Health checks if the wallet manager is healthy. It checks if the +// last block in the chain manager is recent enough and if the last indexed block +// is not too far behind the chain manager. If either of these checks fail, an +// error is returned. +func (m *Manager) Health() error { + cs := m.chain.TipState() + lastBlockTimestamp := cs.PrevTimestamps[0] + if time.Since(lastBlockTimestamp) > maxReorgPeriod { + return fmt.Errorf("last block timestamp %s is too old: %w", lastBlockTimestamp, ErrNotSyncing) + } + + maxSyncedDelta := uint64(maxReorgPeriod / cs.Network.BlockInterval) + indexedTip, err := m.store.LastCommittedIndex() + if err != nil { + return fmt.Errorf("failed to get tip: %w", err) + } else if n := delta(indexedTip.Height, cs.Index.Height); n > maxSyncedDelta { + return fmt.Errorf("last indexed block %q is too far behind tip %q: %w", indexedTip, cs.Index, ErrNotSynced) + } + return nil +} + +// SyncPool forces a sync of the transaction pool for testing +// purposes. +func (m *Manager) SyncPool() { + m.mu.Lock() + defer m.mu.Unlock() + m.resetPool() +} + // Tip returns the last scanned chain index of the manager. func (m *Manager) Tip() (types.ChainIndex, error) { return m.store.LastCommittedIndex() @@ -626,6 +666,13 @@ func (m *Manager) resetPool() { } } +func delta[T constraints.Integer | constraints.Float](a, b T) T { + if cmp.Compare(a, b) > 0 { + return a - b + } + return b - a +} + // NewManager creates a new wallet manager. func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { m := &Manager{ diff --git a/wallet/manager_test.go b/wallet/manager_test.go index bf7bd1a..a1d8846 100644 --- a/wallet/manager_test.go +++ b/wallet/manager_test.go @@ -1,9 +1,34 @@ -package wallet - -// SyncPool forces a sync of the transaction pool for testing -// purposes. -func (m *Manager) SyncPool() { - m.mu.Lock() - defer m.mu.Unlock() - m.resetPool() +package wallet_test + +import ( + "errors" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/v2/internal/testutil" + "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap/zaptest" +) + +func TestHealth(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesis := testutil.V2Network() + cn := testutil.NewConsensusNode(t, n, genesis, log) + cm := cn.Chain + + wm, err := wallet.NewManager(cm, cn.Store) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + if err := wm.Health(); !errors.Is(err, wallet.ErrNotSyncing) { + t.Fatalf("expected error %q, got %q", wallet.ErrNotSyncing, err) + } + + cn.MineBlocks(t, types.VoidAddress, 1) + + if err := wm.Health(); err != nil { + t.Fatalf("expected no error, got %v", err) + } } From 1f153d18035e3b3f35154a5d1b697422a26ea9f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 16 May 2025 21:14:13 +0000 Subject: [PATCH 454/630] chore: prepare release 2.5.0 --- ...d_get_health_to_check_the_health_of_the_walletd_node.md | 5 ----- ...nt_and_block_endpoints_now_support_lookups_by_height.md | 5 ----- CHANGELOG.md | 7 +++++++ go.mod | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .changeset/added_get_health_to_check_the_health_of_the_walletd_node.md delete mode 100644 .changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md diff --git a/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md b/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md deleted file mode 100644 index 5d3afcb..0000000 --- a/.changeset/added_get_health_to_check_the_health_of_the_walletd_node.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Added [GET] /health to check the health of the walletd node diff --git a/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md b/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md deleted file mode 100644 index 9719b5c..0000000 --- a/.changeset/consensus_checkpoint_and_block_endpoints_now_support_lookups_by_height.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Consensus checkpoint and block endpoints now support lookups by height diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f6db92..8663151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.5.0 (2025-05-16) + +### Features + +- Added [GET] /health to check the health of the walletd node +- Consensus checkpoint and block endpoints now support lookups by height + ## 2.4.1 (2025-05-14) ### Fixes diff --git a/go.mod b/go.mod index 51448bf..d6a23cd 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.4.1 +module go.sia.tech/walletd/v2 // v2.5.0 go 1.23.2 From d341cb356af2d74d11599560a371871bdcd6ce6c Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 19 May 2025 16:28:59 -0700 Subject: [PATCH 455/630] update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d6a23cd..93db7ec 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.12.3 - go.sia.tech/coreutils v0.13.6 + go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 1930b3a..e421635 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.12.3 h1:p0BfsKfc7jVRKRDm2K9udxqBEqIGU0An54vP4WL+SH8= go.sia.tech/core v0.12.3/go.mod h1:woUSWQdrRFXkYdk2g2532rguQ1EyMU+2/kcC4JnHSx4= -go.sia.tech/coreutils v0.13.6 h1:RNCrMRO2QA0f/0OMlV5ULjyKYK1KUH1qN0zu/SLvIQk= -go.sia.tech/coreutils v0.13.6/go.mod h1:JR8onVt1R9wz4kg/bzyGg75HuZyTbBVmt3OKcUac6Qo= +go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d h1:WOC0PB4oYTtL/ovSXAHdcOwnTIiEO/gO4hE5BNpyJDw= +go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d/go.mod h1:JR8onVt1R9wz4kg/bzyGg75HuZyTbBVmt3OKcUac6Qo= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 1506e86e5416174beb0588b3e0d9098d7526cfb2 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 19 May 2025 14:25:56 -0700 Subject: [PATCH 456/630] overwrite proofs when in full index mode --- ...nsaction_proofs_when_in_full_index_mode.md | 5 + api/api_test.go | 223 ++++++++++++++++++ api/server.go | 25 +- persist/sqlite/utxo.go | 90 +++---- persist/sqlite/wallet.go | 35 +++ wallet/manager.go | 6 + 6 files changed, 337 insertions(+), 47 deletions(-) create mode 100644 .changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md diff --git a/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md b/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md new file mode 100644 index 0000000..f0d8ed5 --- /dev/null +++ b/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Implicitly fill v2 transaction proofs when in full index mode diff --git a/api/api_test.go b/api/api_test.go index de47668..c596d17 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/jape" @@ -1813,3 +1814,225 @@ func TestBroadcastRace(t *testing.T) { } } } + +func TestTxPoolOverwriteProofs(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + cm := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cm, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + // add an address + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i := range resp.Transaction.SiacoinInputs { + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{senderPrivateKey.SignHash(sigHash)} + } + + // assert the transaction is valid + cs, ok := cm.Chain.State(resp.Basis.ID) + if !ok { + t.Fatal("failed to get state") + } + ms := consensus.NewMidState(cs) + if err := consensus.ValidateV2Transaction(ms, resp.Transaction); err != nil { + t.Fatal(err) + } + + // corrupt the proof + resp.Transaction.SiacoinInputs[0].Parent.StateElement.MerkleProof[frand.Intn(len(resp.Transaction.SiacoinInputs[0].Parent.StateElement.MerkleProof))] = frand.Entropy256() + + // assert the transaction is invalid + ms = consensus.NewMidState(cs) + if err := consensus.ValidateV2Transaction(ms, resp.Transaction); !strings.Contains(err.Error(), "not present in the accumulator") { + t.Fatalf("expected error to contain %q, got %v", "not present in the accumulator", err) + } + + if broadcastResp, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } else if len(broadcastResp.Transactions) != 0 || len(broadcastResp.V2Transactions) != 1 { + t.Fatalf("expected 1 v1 ID and 0 v2 IDs, got %v and %v", len(broadcastResp.Transactions), len(broadcastResp.V2Transactions)) + } else if broadcastResp.V2Transactions[0].ID() != resp.ID { + t.Fatalf("expected v2 ID to be %v, got %v", resp.ID, broadcastResp.V2Transactions[0].ID()) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + expectedValue := types.Siacoins(1).Add(resp.EstimatedFee) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected confirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected confirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected confirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } +} + +func TestTxPoolOverwriteProofsEphemeral(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + cm := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cm, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + // add an address + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: senderAddr}, + }, nil, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i := range resp.Transaction.SiacoinInputs { + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{senderPrivateKey.SignHash(sigHash)} + } + + basis := resp.Basis + txnset := []types.V2Transaction{resp.Transaction, { + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: resp.Transaction.EphemeralSiacoinOutput(0), + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: receiverAddr, Value: types.Siacoins(1)}, + }, + }} + sigHash = cs.InputSigHash(txnset[1]) + for i := range txnset[1].SiacoinInputs { + txnset[1].SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{senderPrivateKey.SignHash(sigHash)} + } + + // corrupt the proof + txnset[0].SiacoinInputs[0].Parent.StateElement.MerkleProof[frand.Intn(len(resp.Transaction.SiacoinInputs[0].Parent.StateElement.MerkleProof))] = frand.Entropy256() + + if broadcastResp, err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { + t.Fatal(err) + } else if len(broadcastResp.Transactions) != 0 || len(broadcastResp.V2Transactions) != 2 { + t.Fatalf("expected 0 v1 ID and 2 v2 IDs, got %v and %v", len(broadcastResp.Transactions), len(broadcastResp.V2Transactions)) + } else if broadcastResp.V2Transactions[0].ID() != txnset[0].ID() { + t.Fatalf("expected v2 ID to be %v, got %v", txnset[0].ID(), broadcastResp.V2Transactions[0].ID()) + } else if broadcastResp.V2Transactions[1].ID() != txnset[1].ID() { + t.Fatalf("expected v2 ID to be %v, got %v", txnset[1].ID(), broadcastResp.V2Transactions[1].ID()) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 2 { + t.Fatalf("expected 2 unconfirmed events, got %v", len(unconfirmed)) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 3 { + t.Fatalf("expected 3 confirmed events, got %v", len(confirmed)) // initial gift + setup + sent + } +} diff --git a/api/server.go b/api/server.go index 39a4862..0843258 100644 --- a/api/server.go +++ b/api/server.go @@ -147,6 +147,7 @@ type ( // In personal index mode, this returns true only if the address // is registered to a wallet. CheckAddresses([]types.Address) (bool, error) + OverwriteElementProofs([]types.V2Transaction) (types.ChainIndex, []types.V2Transaction, error) } ) @@ -422,20 +423,29 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } } if len(tbr.V2Transactions) != 0 { + var err error + if s.wm.IndexMode() == wallet.IndexModeFull { + // when in full index mode overwriting the proofs makes it slightly more convenient + // and less error-prone for users to broadcast v2 transactions since the correct + // proof can be filled implicitly. Unfortunately, that hides bad implementations + // from the implementor. In practice, this trade off is worth it. + tbr.Basis, tbr.V2Transactions, err = s.wm.OverwriteElementProofs(tbr.V2Transactions) + if jc.Check("couldn't overwrite proofs", err) != nil { + return + } + } + if len(tbr.V2Transactions) == 1 { // if there's only one transaction, best-effort check for parents - var err error tbr.Basis, tbr.V2Transactions, err = s.cm.V2TransactionSet(tbr.Basis, tbr.V2Transactions[0]) - if jc.Check("couldn't create v2 transaction set", err) != nil { + if jc.Check("couldn't get transaction set", err) != nil { return } } - // prevents a race condition when encoding the transactions - // TODO: fix this race - resp.V2Transactions = make([]types.V2Transaction, 0, len(tbr.V2Transactions)) - for _, txn := range tbr.V2Transactions { - resp.V2Transactions = append(resp.V2Transactions, txn.DeepCopy()) + resp.V2Transactions = slices.Clone(tbr.V2Transactions) + for i := range resp.V2Transactions { + resp.V2Transactions[i] = resp.V2Transactions[i].DeepCopy() } if _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions); err != nil { @@ -445,6 +455,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { return } } + resp.Basis = tbr.Basis jc.Encode(resp) } diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go index bf32f1f..55f428e 100644 --- a/persist/sqlite/utxo.go +++ b/persist/sqlite/utxo.go @@ -9,65 +9,75 @@ import ( "go.sia.tech/walletd/v2/wallet" ) -// SiacoinElement returns an unspent Siacoin UTXO by its ID. -func (s *Store) SiacoinElement(id types.SiacoinOutputID) (ele types.SiacoinElement, err error) { - err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address +func getSiacoinElement(tx *txn, id types.SiacoinOutputID, indexMode wallet.IndexMode) (ele types.SiacoinElement, err error) { + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1 AND spent_index_id IS NULL` - ele, err = scanSiacoinElement(tx.QueryRow(query, encode(id))) - if err != nil { - return err - } + ele, err = scanSiacoinElement(tx.QueryRow(query, encode(id))) + if err != nil { + return types.SiacoinElement{}, err + } - // retrieve the merkle proofs for the siacoin element - if s.indexMode == wallet.IndexModeFull { - proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) - if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) - } else if len(proof) != 1 { - panic("expected exactly one proof") // should never happen - } - ele.StateElement.MerkleProof = proof[0] + // retrieve the merkle proofs for the siacoin element + if indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) + if err != nil { + return types.SiacoinElement{}, fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen } - return nil - }) - if errors.Is(err, sql.ErrNoRows) { - err = wallet.ErrNotFound + ele.StateElement.MerkleProof = proof[0] } return } -// SiafundElement returns an unspent Siafund UTXO by its ID. -func (s *Store) SiafundElement(id types.SiafundOutputID) (ele types.SiafundElement, err error) { - err = s.transaction(func(tx *txn) error { - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address +func getSiafundElement(tx *txn, id types.SiafundOutputID, indexMode wallet.IndexMode) (ele types.SiafundElement, err error) { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address FROM siafund_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1 AND spent_index_id IS NULL` - ele, err = scanSiafundElement(tx.QueryRow(query, encode(id))) + ele, err = scanSiafundElement(tx.QueryRow(query, encode(id))) + if err != nil { + return types.SiafundElement{}, err + } + + // retrieve the merkle proofs for the siafund element + if indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) if err != nil { - return err + return types.SiafundElement{}, fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen + } + ele.StateElement.MerkleProof = proof[0] + } + return +} + +// SiacoinElement returns an unspent Siacoin UTXO by its ID. +func (s *Store) SiacoinElement(id types.SiacoinOutputID) (ele types.SiacoinElement, err error) { + err = s.transaction(func(tx *txn) error { + ele, err = getSiacoinElement(tx, id, s.indexMode) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound } + return err + }) + return +} - // retrieve the merkle proofs for the siafund element - if s.indexMode == wallet.IndexModeFull { - proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) - if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) - } else if len(proof) != 1 { - panic("expected exactly one proof") // should never happen - } - ele.StateElement.MerkleProof = proof[0] +// SiafundElement returns an unspent Siafund UTXO by its ID. +func (s *Store) SiafundElement(id types.SiafundOutputID) (ele types.SiafundElement, err error) { + err = s.transaction(func(tx *txn) error { + ele, err = getSiafundElement(tx, id, s.indexMode) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound } - return nil + return err }) - if errors.Is(err, sql.ErrNoRows) { - err = wallet.ErrNotFound - } return } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 35fe994..57f3599 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -757,3 +757,38 @@ func walletExists(tx *txn, id wallet.ID) error { } return err } + +// OverwriteElementProofs overwrites the element proofs for the given transactions. +func (s *Store) OverwriteElementProofs(txns []types.V2Transaction) (basis types.ChainIndex, updated []types.V2Transaction, err error) { + err = s.transaction(func(tx *txn) error { + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + for _, txn := range txns { + txn = txn.DeepCopy() + for i, sci := range txn.SiacoinInputs { + ele, err := getSiacoinElement(tx, sci.Parent.ID, s.indexMode) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return fmt.Errorf("failed to get siacoin element: %w", err) + } + txn.SiacoinInputs[i].Parent = ele + } + for i, sfi := range txn.SiafundInputs { + ele, err := getSiafundElement(tx, sfi.Parent.ID, s.indexMode) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return fmt.Errorf("failed to get siafund element: %w", err) + } + txn.SiafundInputs[i].Parent = ele + } + updated = append(updated, txn) + } + return nil + }) + return +} diff --git a/wallet/manager.go b/wallet/manager.go index 3e25c47..9f51a84 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -107,6 +107,7 @@ type ( // In personal index mode, this function returns true only // if the address is registered to a wallet. CheckAddresses([]types.Address) (bool, error) + OverwriteElementProofs(txns []types.V2Transaction) (basis types.ChainIndex, updated []types.V2Transaction, err error) Events(eventIDs []types.Hash256) ([]Event, error) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) @@ -518,6 +519,11 @@ top: return selected, basis, inputSum - amount, nil } +// OverwriteElementProofs overwrites the proofs of the given transactions. +func (m *Manager) OverwriteElementProofs(txns []types.V2Transaction) (types.ChainIndex, []types.V2Transaction, error) { + return m.store.OverwriteElementProofs(txns) +} + // Scan rescans the chain starting from the given index. The scan will complete // when the chain manager reaches the current tip or the context is canceled. func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { From 94478476b97d9036b6b4ae4617e2a6fb3db1ff12 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 19 May 2025 16:36:20 -0700 Subject: [PATCH 457/630] api always overwrite proofs --- api/server.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/server.go b/api/server.go index 0843258..445f0ba 100644 --- a/api/server.go +++ b/api/server.go @@ -424,15 +424,15 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { } if len(tbr.V2Transactions) != 0 { var err error - if s.wm.IndexMode() == wallet.IndexModeFull { - // when in full index mode overwriting the proofs makes it slightly more convenient - // and less error-prone for users to broadcast v2 transactions since the correct - // proof can be filled implicitly. Unfortunately, that hides bad implementations - // from the implementor. In practice, this trade off is worth it. - tbr.Basis, tbr.V2Transactions, err = s.wm.OverwriteElementProofs(tbr.V2Transactions) - if jc.Check("couldn't overwrite proofs", err) != nil { - return - } + // Overwrites the proofs for siacoin elements that are tracked in the database. Makes it slightly + // more convenient and less error-prone for users to broadcast v2 transactions since the correct + // proof can be filled implicitly. Unfortunately, that hides bad implementations from the + // implementor. In practice, this trade off is worth it. + // In full index mode, any UTXO can have its proofs overwritten. + // In personal index mode, only UTXOs that are registered to a wallet can have its proofs overwritten. + tbr.Basis, tbr.V2Transactions, err = s.wm.OverwriteElementProofs(tbr.V2Transactions) + if jc.Check("couldn't overwrite proofs", err) != nil { + return } if len(tbr.V2Transactions) == 1 { From 8f3a9a1c679e936dd268e8e4b55f2dc813936a06 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 20 May 2025 23:27:18 -0700 Subject: [PATCH 458/630] return unspent outputs --- ...also_return_the_number_of_confirmations.md | 27 ++++ api/api_test.go | 122 ++++++++++++++++++ api/client.go | 8 +- api/server.go | 16 +-- persist/sqlite/wallet.go | 14 +- wallet/manager.go | 25 ++-- wallet/wallet_test.go | 14 +- 7 files changed, 190 insertions(+), 36 deletions(-) create mode 100644 .changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md diff --git a/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md b/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md new file mode 100644 index 0000000..f6f216c --- /dev/null +++ b/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md @@ -0,0 +1,27 @@ +--- +default: minor +--- + +# Changed all endpoints that to return Siacoin or Siafund elements to also return the number of confirmations + +```json +{ + { + "id": "5fb7f9ef38dfeeeb4d8c0c1f105452511f0e966dec1ce545e490f5eee46d166f", + "stateElement": { + "leafIndex": 25490, + "merkleProof": [ + "9175d0ea4dbdecd0517bd275afd98250438193429d0dc7493672217464f3bfa3", + "ab87ecba97723b67e42027dd8d2ad5a51ab48c3cd1b38dc461805b266b1fa728", + "6cb7dcc6300e8344b17012b36fe64a0d7e1678d54736d1fd910c7c9665b273b9" + ] + }, + "siacoinOutput": { + "value": "344000", + "address": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "maturityHeight": 7437, + "confirmations": 6 + } +} +``` \ No newline at end of file diff --git a/api/api_test.go b/api/api_test.go index de47668..b4664e8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -345,6 +345,8 @@ func TestWallet(t *testing.T) { t.Fatal("should have two UTXOs, got", len(outputs)) } else if basis != cn.Chain.Tip() { t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if outputs[0].Confirmations != 1 { + t.Fatalf("expected 1 confirmation, got %v", outputs[0].Confirmations) } // mine a block to add an immature balance @@ -1813,3 +1815,123 @@ func TestBroadcastRace(t *testing.T) { } } } + +func TestWalletConfirmations(t *testing.T) { + log := zaptest.NewLogger(t) + + // create syncer + syncerListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer syncerListener.Close() + + // create chain manager + n, genesisBlock := testutil.V1Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = giftAddress + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + wc := c.Wallet(w.ID) + + // create and add an address + sk2 := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(sk2.PublicKey()) + err = wc.AddAddress(wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(sk2.PublicKey())), + }, + }) + if err != nil { + t.Fatal(err) + } + c.Rescan(0) + + // send gift to wallet + giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: giftSCOID, + UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1)}, + }, + SiafundInputs: []types.SiafundInput{{ + ParentID: genesisBlock.Transactions[0].SiafundOutputID(0), + UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + }}, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: genesisBlock.Transactions[0].SiafundOutputs[0].Value}, + }, + Signatures: []types.TransactionSignature{{ + ParentID: types.Hash256(giftSCOID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, { + ParentID: types.Hash256(genesisBlock.Transactions[0].SiafundOutputID(0)), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + sig2 := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(genesisBlock.Transactions[0].SiafundOutputID(0)), 0, 0, nil)) + txn.Signatures[1].Signature = sig2[:] + + // broadcast the transaction to the transaction pool + if _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + t.Fatal(err) + } + + // confirm the transaction + cn.MineBlocks(t, types.VoidAddress, 1) + + assertConfirmations := func(t *testing.T, n uint64) { + t.Helper() + + outputs, basis, err := wc.SiacoinOutputs(0, 100) + if err != nil { + t.Fatal(err) + } else if len(outputs) != 1 { + t.Fatal("should have one UTXOs, got", len(outputs)) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if outputs[0].Confirmations != n { + t.Fatalf("expected %d confirmation, got %v", n, outputs[0].Confirmations) + } + + sfe, basis, err := wc.SiafundOutputs(0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfe) != 1 { + t.Fatal("should have one siafund output, got", len(sfe)) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if sfe[0].Confirmations != n { + t.Fatalf("expected %d confirmation, got %v", n, sfe[0].Confirmations) + } + } + + assertConfirmations(t, 1) + cn.MineBlocks(t, types.VoidAddress, 10) + assertConfirmations(t, 11) +} diff --git a/api/client.go b/api/client.go index 2779e80..13c7ef9 100644 --- a/api/client.go +++ b/api/client.go @@ -360,15 +360,15 @@ func (c *WalletClient) UnconfirmedEvents() (resp []wallet.Event, err error) { } // SiacoinOutputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) SiacoinOutputs(offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { - var resp SiacoinElementsResponse +func (c *WalletClient) SiacoinOutputs(offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + var resp UnspentSiacoinElementsResponse err := c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &resp) return resp.Outputs, resp.Basis, err } // SiafundOutputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) SiafundOutputs(offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { - var resp SiafundElementsResponse +func (c *WalletClient) SiafundOutputs(offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + var resp UnspentSiafundElementsResponse err := c.c.GET(context.Background(), fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &resp) return resp.Outputs, resp.Basis, err } diff --git a/api/server.go b/api/server.go index 39a4862..7d116b4 100644 --- a/api/server.go +++ b/api/server.go @@ -108,10 +108,10 @@ type ( WalletAddress(wallet.ID, types.Address) (wallet.Address, error) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) - SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) - SelectSiafundElements(walletID wallet.ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) - UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) - UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]wallet.UnspentSiacoinElement, types.ChainIndex, types.Currency, error) + SelectSiafundElements(walletID wallet.ID, amount uint64) ([]wallet.UnspentSiafundElement, types.ChainIndex, uint64, error) + UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) + UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) WalletBalance(id wallet.ID) (wallet.Balance, error) AddressBalance(address types.Address) (wallet.Balance, error) @@ -691,7 +691,7 @@ func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { return } - jc.Encode(SiacoinElementsResponse{ + jc.Encode(UnspentSiacoinElementsResponse{ Basis: basis, Outputs: scos, }) @@ -712,7 +712,7 @@ func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { if jc.Check("couldn't load siacoin outputs", err) != nil { return } - jc.Encode(SiafundElementsResponse{ + jc.Encode(UnspentSiafundElementsResponse{ Basis: basis, Outputs: sfos, }) @@ -1214,7 +1214,7 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } sfi := types.V2SiafundInput{ - Parent: sfe, + Parent: sfe.SiafundElement, ClaimAddress: wcr.ChangeAddress, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: sp, @@ -1239,7 +1239,7 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { } sci := types.V2SiacoinInput{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: sp, }, diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 35fe994..2925b44 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -229,7 +229,7 @@ WHERE wa.wallet_id=$1` } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { +func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -240,8 +240,9 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins return fmt.Errorf("failed to get basis: %w", err) } - const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height FROM siacoin_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.spent_index_id IS NULL AND se.maturity_height <= $1 AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$2) LIMIT $3 OFFSET $4` @@ -253,7 +254,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins defer rows.Close() for rows.Next() { - siacoin, err := scanSiacoinElement(rows) + siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) if err != nil { return fmt.Errorf("failed to scan siacoin element: %w", err) } @@ -285,7 +286,7 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins } // WalletSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { +func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { err = s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err @@ -296,8 +297,9 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds return fmt.Errorf("failed to get basis: %w", err) } - const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height FROM siafund_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) LIMIT $2 OFFSET $3` @@ -309,7 +311,7 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds defer rows.Close() for rows.Next() { - siafund, err := scanSiafundElement(rows) + siafund, err := scanUnspentSiafundElement(rows, basis.Height) if err != nil { return fmt.Errorf("failed to scan siafund element: %w", err) } diff --git a/wallet/manager.go b/wallet/manager.go index 3e25c47..66b6b9c 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -86,8 +86,8 @@ type ( DeleteWallet(walletID ID) error WalletBalance(walletID ID) (Balance, error) WalletAddress(ID, types.Address) (Address, error) - WalletSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) - WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + WalletSiacoinOutputs(walletID ID, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) + WalletSiafundOutputs(walletID ID, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) @@ -284,13 +284,13 @@ func (m *Manager) WalletEvents(walletID ID, offset, limit int) ([]Event, error) // UnspentSiacoinOutputs returns a paginated list of matured siacoin outputs // relevant to the wallet -func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { +func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) { return m.store.WalletSiacoinOutputs(walletID, offset, limit) } // UnspentSiafundOutputs returns a paginated list of siafund outputs relevant to // the wallet -func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { +func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) { return m.store.WalletSiafundOutputs(walletID, offset, limit) } @@ -369,7 +369,7 @@ func (m *Manager) WalletAddress(id ID, addr types.Address) (Address, error) { // SelectSiacoinElements selects siacoin elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. -func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { +func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]UnspentSiacoinElement, types.ChainIndex, types.Currency, error) { // sanity check that the wallet exists if _, err := m.WalletBalance(walletID); err != nil { return nil, types.ChainIndex{}, types.ZeroCurrency, err @@ -407,13 +407,13 @@ func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useU inPool := m.poolSCSpent var inputSum types.Currency - var selected []types.SiacoinElement + var selected []UnspentSiacoinElement var utxoIDs []types.Hash256 var basis types.ChainIndex const utxoBatchSize = 100 top: for i := 0; ; i += utxoBatchSize { - var utxos []types.SiacoinElement + var utxos []UnspentSiacoinElement var err error // extra large wallets may need to paginate through utxos // to find enough to cover the amount @@ -448,7 +448,10 @@ top: continue } - selected = append(selected, sce) + selected = append(selected, UnspentSiacoinElement{ + SiacoinElement: sce, + Confirmations: 0, + }) inputSum = inputSum.Add(sce.SiacoinOutput.Value) if inputSum.Cmp(amount) >= 0 { break @@ -466,7 +469,7 @@ top: // SelectSiafundElements selects siafund elements from the wallet that sum to // at least the given amount. Returns the elements, the element basis, and the // change amount. -func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { +func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]UnspentSiafundElement, types.ChainIndex, uint64, error) { // sanity check that the wallet exists if _, err := m.WalletBalance(walletID); err != nil { return nil, types.ChainIndex{}, 0, err @@ -480,13 +483,13 @@ func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.Sia } var inputSum uint64 - var selected []types.SiafundElement + var selected []UnspentSiafundElement var utxoIDs []types.Hash256 var basis types.ChainIndex const utxoBatchSize = 100 top: for i := 0; ; i += utxoBatchSize { - var utxos []types.SiafundElement + var utxos []UnspentSiafundElement var err error utxos, basis, err = m.store.WalletSiafundOutputs(walletID, i, utxoBatchSize) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 2180e82..4ce184e 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -2497,7 +2497,7 @@ func TestV2(t *testing.T) { policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{Type: policy}, }, @@ -2925,7 +2925,7 @@ func TestReorgV2(t *testing.T) { policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{Type: policy}, }, @@ -3053,7 +3053,7 @@ func TestOrphansV2(t *testing.T) { policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) txn := types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{Type: policy}, }, @@ -3139,7 +3139,7 @@ func TestOrphansV2(t *testing.T) { // spend the payout txn = types.V2Transaction{ SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: types.SpendPolicy{Type: policy}, }, @@ -4172,7 +4172,7 @@ func TestV2SiafundClaims(t *testing.T) { } for _, sfe := range siafunds { txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ - Parent: sfe, + Parent: sfe.SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: sp, }, @@ -4246,7 +4246,7 @@ func TestV2SiafundClaims(t *testing.T) { for _, sce := range siacoins { fcTxn.SiacoinInputs = append(fcTxn.SiacoinInputs, types.V2SiacoinInput{ - Parent: sce, + Parent: sce.SiacoinElement, SatisfiedPolicy: types.SatisfiedPolicy{ Policy: sp, }, @@ -4285,7 +4285,7 @@ func TestV2SiafundClaims(t *testing.T) { } for _, sfe := range siafunds { txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ - Parent: sfe, + Parent: sfe.SiafundElement, SatisfiedPolicy: types.SatisfiedPolicy{Policy: sp}, ClaimAddress: addr, }) From c86e55d2449b0904ffd0ba4c5aadb0e061207c61 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 20 May 2025 23:48:03 -0700 Subject: [PATCH 459/630] Update .changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md Co-authored-by: Peter-Jan Brone --- ...afund_elements_to_also_return_the_number_of_confirmations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md b/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md index f6f216c..4d548e7 100644 --- a/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md +++ b/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md @@ -2,7 +2,7 @@ default: minor --- -# Changed all endpoints that to return Siacoin or Siafund elements to also return the number of confirmations +# Changed all endpoints that return Siacoin or Siafund elements to also return the number of confirmations ```json { From d8e83d8d30cf10feec3096f9a47990b896bc5364 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 06:58:41 +0000 Subject: [PATCH 460/630] chore: prepare release 2.6.0 --- ...also_return_the_number_of_confirmations.md | 27 ------------------ CHANGELOG.md | 28 +++++++++++++++++++ go.mod | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) delete mode 100644 .changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md diff --git a/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md b/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md deleted file mode 100644 index 4d548e7..0000000 --- a/.changeset/changed_all_endpoints_that_to_return_siacoin_or_siafund_elements_to_also_return_the_number_of_confirmations.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -default: minor ---- - -# Changed all endpoints that return Siacoin or Siafund elements to also return the number of confirmations - -```json -{ - { - "id": "5fb7f9ef38dfeeeb4d8c0c1f105452511f0e966dec1ce545e490f5eee46d166f", - "stateElement": { - "leafIndex": 25490, - "merkleProof": [ - "9175d0ea4dbdecd0517bd275afd98250438193429d0dc7493672217464f3bfa3", - "ab87ecba97723b67e42027dd8d2ad5a51ab48c3cd1b38dc461805b266b1fa728", - "6cb7dcc6300e8344b17012b36fe64a0d7e1678d54736d1fd910c7c9665b273b9" - ] - }, - "siacoinOutput": { - "value": "344000", - "address": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" - }, - "maturityHeight": 7437, - "confirmations": 6 - } -} -``` \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 8663151..6b7db8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ +## 2.6.0 (2025-05-21) + +### Features + +#### Changed all endpoints that return Siacoin or Siafund elements to also return the number of confirmations + +```json +{ + { + "id": "5fb7f9ef38dfeeeb4d8c0c1f105452511f0e966dec1ce545e490f5eee46d166f", + "stateElement": { + "leafIndex": 25490, + "merkleProof": [ + "9175d0ea4dbdecd0517bd275afd98250438193429d0dc7493672217464f3bfa3", + "ab87ecba97723b67e42027dd8d2ad5a51ab48c3cd1b38dc461805b266b1fa728", + "6cb7dcc6300e8344b17012b36fe64a0d7e1678d54736d1fd910c7c9665b273b9" + ] + }, + "siacoinOutput": { + "value": "344000", + "address": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69" + }, + "maturityHeight": 7437, + "confirmations": 6 + } +} +``` + ## 2.5.0 (2025-05-16) ### Features diff --git a/go.mod b/go.mod index 93db7ec..ac2682f 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.5.0 +module go.sia.tech/walletd/v2 // v2.6.0 go 1.23.2 From b1250df79c9909a1e405963c51e9a188c16b260d Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 21 May 2025 10:55:30 -0700 Subject: [PATCH 461/630] update core and coreutils --- api/mine.go | 2 +- cmd/walletd/miner.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- wallet/wallet_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/mine.go b/api/mine.go index d2b6bd1..6e43658 100644 --- a/api/mine.go +++ b/api/mine.go @@ -45,7 +45,7 @@ func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types. b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) } if b.V2 != nil { - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), addr) + b.V2.Commitment = cs.Commitment(addr, b.Transactions, b.V2.Transactions) } b.Nonce = 0 diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go index 7267602..78f187d 100644 --- a/cmd/walletd/miner.go +++ b/cmd/walletd/miner.go @@ -48,7 +48,7 @@ func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { Height: cs.Index.Height + 1, Transactions: v2txns, } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + b.V2.Commitment = cs.Commitment(b.MinerPayouts[0].Address, b.Transactions, b.V2Transactions()) } if !coreutils.FindBlockNonce(cs, &b, time.Minute) { continue diff --git a/go.mod b/go.mod index ac2682f..5c4780c 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.12.3 - go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d + go.sia.tech/core v0.12.4 + go.sia.tech/coreutils v0.14.0 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index e421635..3a54302 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.12.3 h1:p0BfsKfc7jVRKRDm2K9udxqBEqIGU0An54vP4WL+SH8= -go.sia.tech/core v0.12.3/go.mod h1:woUSWQdrRFXkYdk2g2532rguQ1EyMU+2/kcC4JnHSx4= -go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d h1:WOC0PB4oYTtL/ovSXAHdcOwnTIiEO/gO4hE5BNpyJDw= -go.sia.tech/coreutils v0.13.7-0.20250519232338-480bcda7534d/go.mod h1:JR8onVt1R9wz4kg/bzyGg75HuZyTbBVmt3OKcUac6Qo= +go.sia.tech/core v0.12.4 h1:q427SKscvhRrbwQLyydMb/uva8CFfwi5Kpxc8UkNK6o= +go.sia.tech/core v0.12.4/go.mod h1:NpWoU+7q9X+lPfqlkW4Xjyhwt9+DZpvf2WWmo4Ui4Ig= +go.sia.tech/coreutils v0.14.0 h1:hBXu3ZlaGOV7BFRJXshR5lvt8pAu4o0VCSunisNT2Tg= +go.sia.tech/coreutils v0.14.0/go.mod h1:8vXG2nIz9zzrP9QjocwAebcVPhKXQvt0GGfmhGX5vow= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 4ce184e..d32c1a5 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -102,7 +102,7 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty Height: state.Index.Height + 1, }, } - b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + b.V2.Commitment = state.Commitment(b.MinerPayouts[0].Address, b.Transactions, b.V2Transactions()) for b.ID().CmpWork(state.ChildTarget) < 0 { b.Nonce += state.NonceFactor() } From 28c4e784001d3a4599b52dbfb627bc0579bb1cf0 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 16:29:41 -0700 Subject: [PATCH 462/630] add changeset files --- .changeset/update_coreutils_to_v0140.md | 5 +++++ .changeset/updated_core_to_v0124.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/update_coreutils_to_v0140.md create mode 100644 .changeset/updated_core_to_v0124.md diff --git a/.changeset/update_coreutils_to_v0140.md b/.changeset/update_coreutils_to_v0140.md new file mode 100644 index 0000000..92d9019 --- /dev/null +++ b/.changeset/update_coreutils_to_v0140.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Updated coreutils to v0.14.0 diff --git a/.changeset/updated_core_to_v0124.md b/.changeset/updated_core_to_v0124.md new file mode 100644 index 0000000..6accca9 --- /dev/null +++ b/.changeset/updated_core_to_v0124.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Updated core to v0.12.4 From 5e547e5fb85dce407a624d827e90fd02c71848eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 23:29:54 +0000 Subject: [PATCH 463/630] chore: prepare release 2.7.0 --- .changeset/update_coreutils_to_v0140.md | 5 ----- .changeset/updated_core_to_v0124.md | 5 ----- CHANGELOG.md | 7 +++++++ go.mod | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .changeset/update_coreutils_to_v0140.md delete mode 100644 .changeset/updated_core_to_v0124.md diff --git a/.changeset/update_coreutils_to_v0140.md b/.changeset/update_coreutils_to_v0140.md deleted file mode 100644 index 92d9019..0000000 --- a/.changeset/update_coreutils_to_v0140.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Updated coreutils to v0.14.0 diff --git a/.changeset/updated_core_to_v0124.md b/.changeset/updated_core_to_v0124.md deleted file mode 100644 index 6accca9..0000000 --- a/.changeset/updated_core_to_v0124.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Updated core to v0.12.4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7db8e..e63b54e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.7.0 (2025-05-23) + +### Features + +- Updated coreutils to v0.14.0 +- Updated core to v0.12.4 + ## 2.6.0 (2025-05-21) ### Features diff --git a/go.mod b/go.mod index 5c4780c..5e1897a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.6.0 +module go.sia.tech/walletd/v2 // v2.7.0 go 1.23.2 From 8147939f5bf9a8bf8887cb9ad6fc14eac7ee3ba7 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 16:44:51 -0700 Subject: [PATCH 464/630] ci: automate releases --- .github/workflows/publish.yml | 48 ++++++++++++++++++++++++++++------- knope.toml | 13 +++++----- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5849c36..b8688b9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,21 +1,31 @@ name: Publish -# Controls when the action will run. on: - # Triggers the workflow on new SemVer tags - push: - branches: - - master - tags: - - "v[0-9]+.[0-9]+.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-**" + pull_request: + types: [closed] + branches: [master] + workflow_dispatch: concurrency: group: ${{ github.workflow }} cancel-in-progress: false jobs: - publish: + get-tag: + if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + - run: echo "tag_name=$(gh release list --json 'isDraft,tagName' --jq '.[] | select(.isDraft) | .tagName')" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + id: get-tag + outputs: + tag_name: ${{ steps.get-tag.outputs.tag_name }} + build: + needs: + - get-tag + if: needs.get-tag.outputs.tag_name != '' uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master secrets: inherit with: @@ -26,3 +36,23 @@ jobs: project: walletd project-desc: "walletd: The new Sia wallet" version-tag: ${{ github.ref_name }} + release: + needs: + - build + - get-tag + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4.2.2 + - uses: actions/download-artifact@v4.3.0 + with: + path: artifacts + - name: Upload artifacts to release + run: | + cd artifacts + gh release upload ${{ needs.get-tag.outputs.tag_name }} * + gh release edit ${{ needs.get-tag.outputs.tag_name }} --draft=false --latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/knope.toml b/knope.toml index 1034ad0..fad9b3d 100644 --- a/knope.toml +++ b/knope.toml @@ -1,7 +1,7 @@ [package] changelog = "CHANGELOG.md" versioned_files = ["go.mod"] -ignore_go_major_versioning = true +assets = "marker" [[workflows]] name = "document-change" @@ -44,12 +44,11 @@ variables = { "$version" = "Version" } template = "This PR was created automatically. Merging it will finalize the changelog for $version\n\n$changelog" variables = { "$changelog" = "ChangelogEntry", "$version" = "Version" } -# Do not enable releases, just changelogs for now. -# [[workflows]] -# name = "release" -# -# [[workflows.steps]] -# type = "Release" +[[workflows]] +name = "release" + +[[workflows.steps]] +type = "Release" [github] owner = "SiaFoundation" From f1cddf7a675d957dca864bddf95ecd199b893e7f Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 16:50:13 -0700 Subject: [PATCH 465/630] ci: automate releases --- .github/workflows/publish.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b8688b9..4280ca8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,6 +45,16 @@ jobs: contents: write steps: - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: actions/download-artifact@v4.3.0 with: path: artifacts @@ -52,7 +62,6 @@ jobs: run: | cd artifacts gh release upload ${{ needs.get-tag.outputs.tag_name }} * - gh release edit ${{ needs.get-tag.outputs.tag_name }} --draft=false --latest env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From bcf865389db4d2e457f7ae828771e0a9188f0fd4 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 16:59:17 -0700 Subject: [PATCH 466/630] ci: split release and publish --- .github/workflows/publish.yml | 53 +++++++++++------------------------ .github/workflows/release.yml | 43 ++++++++++++++++++++++++++++ knope.toml | 3 ++ 3 files changed, 63 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4280ca8..b3e7e42 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,31 +1,21 @@ name: Publish +# Controls when the action will run. on: - pull_request: - types: [closed] - branches: [master] - workflow_dispatch: + # Triggers the workflow on new SemVer tags + push: + branches: + - master + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-**" concurrency: group: ${{ github.workflow }} cancel-in-progress: false jobs: - get-tag: - if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.2.2 - - run: echo "tag_name=$(gh release list --json 'isDraft,tagName' --jq '.[] | select(.isDraft) | .tagName')" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - id: get-tag - outputs: - tag_name: ${{ steps.get-tag.outputs.tag_name }} - build: - needs: - - get-tag - if: needs.get-tag.outputs.tag_name != '' + publish: uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master secrets: inherit with: @@ -36,32 +26,23 @@ jobs: project: walletd project-desc: "walletd: The new Sia wallet" version-tag: ${{ github.ref_name }} - release: - needs: - - build - - get-tag + upload: + if: github.event_name == 'push' && startsWith(github.ref_name, 'v') runs-on: ubuntu-latest - permissions: - contents: write + needs: + - publish steps: - uses: actions/checkout@v4.2.2 with: fetch-depth: 0 - - name: Configure Git - run: | - git config --global user.name github-actions[bot] - git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 - - run: knope release --verbose - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/download-artifact@v4.3.0 + - name: Download artifacts + uses: actions/download-artifact@v4.3.0 with: path: artifacts - name: Upload artifacts to release run: | cd artifacts - gh release upload ${{ needs.get-tag.outputs.tag_name }} * + gh release upload ${{ github.ref_name }} * env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..59bdb79 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Publish + +on: + pull_request: + types: [closed] + branches: [master] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + get-tag: + if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + - run: echo "tag_name=$(gh release list --json 'isDraft,tagName' --jq '.[] | select(.isDraft) | .tagName')" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + id: get-tag + outputs: + tag_name: ${{ steps.get-tag.outputs.tag_name }} + release: + needs: + - get-tag + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/knope.toml b/knope.toml index fad9b3d..4962299 100644 --- a/knope.toml +++ b/knope.toml @@ -3,6 +3,9 @@ changelog = "CHANGELOG.md" versioned_files = ["go.mod"] assets = "marker" +[bot.releases] +enabled = true + [[workflows]] name = "document-change" From 0fe167bfe3ef18529d01802a075c4d1119bfe957 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 17:00:34 -0700 Subject: [PATCH 467/630] ci: fix duplicate job names --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59bdb79..5a45caa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Publish +name: Release on: pull_request: From c4722156aa64acdeb80c3507f08dbbb00e05b54d Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 23 May 2025 17:13:31 -0700 Subject: [PATCH 468/630] ci: fix automation permissions --- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 20 ++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b3e7e42..4304162 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -44,5 +44,5 @@ jobs: cd artifacts gh release upload ${{ github.ref_name }} * env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a45caa..00c04ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,20 +11,8 @@ concurrency: cancel-in-progress: false jobs: - get-tag: - if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4.2.2 - - run: echo "tag_name=$(gh release list --json 'isDraft,tagName' --jq '.[] | select(.isDraft) | .tagName')" >> $GITHUB_OUTPUT - env: - GH_TOKEN: ${{ github.token }} - id: get-tag - outputs: - tag_name: ${{ steps.get-tag.outputs.tag_name }} release: - needs: - - get-tag + if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest permissions: contents: write @@ -32,12 +20,8 @@ jobs: - uses: actions/checkout@v4.2.2 with: fetch-depth: 0 - - name: Configure Git - run: | - git config --global user.name github-actions[bot] - git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 - run: knope release --verbose env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} From 9e11173134650058931e3a41dba0d32e92305766 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 25 May 2025 18:41:30 -0700 Subject: [PATCH 469/630] update core --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5e1897a..0cf4ec1 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.12.4 - go.sia.tech/coreutils v0.14.0 + go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230 + go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 3a54302..cb28f33 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.12.4 h1:q427SKscvhRrbwQLyydMb/uva8CFfwi5Kpxc8UkNK6o= -go.sia.tech/core v0.12.4/go.mod h1:NpWoU+7q9X+lPfqlkW4Xjyhwt9+DZpvf2WWmo4Ui4Ig= -go.sia.tech/coreutils v0.14.0 h1:hBXu3ZlaGOV7BFRJXshR5lvt8pAu4o0VCSunisNT2Tg= -go.sia.tech/coreutils v0.14.0/go.mod h1:8vXG2nIz9zzrP9QjocwAebcVPhKXQvt0GGfmhGX5vow= +go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230 h1:o8+TZHq1+1j1Ww8QECvFVHRZ35hgCOaKpS7X6OR+N2Y= +go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230/go.mod h1:NpWoU+7q9X+lPfqlkW4Xjyhwt9+DZpvf2WWmo4Ui4Ig= +go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f h1:VnaLaf4xrrVBWsMtiuDG7LVtbStHBMn3q+9qZYYxd0o= +go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f/go.mod h1:WsQjcHTvnCI10dU8anbH06kYggmyrS4+kxtkOD8KxE4= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 176ae864daaf74ce842e162b2a9302b598402354 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 26 May 2025 07:36:34 +0200 Subject: [PATCH 470/630] address chris' comments --- api/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/server.go b/api/server.go index 445f0ba..fcc38a6 100644 --- a/api/server.go +++ b/api/server.go @@ -402,9 +402,7 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { // the transactions are sent back to the client because the // transaction set may have been modified and the transactions // include additional convenience fields when being marshalled - resp := TxpoolBroadcastResponse{ - Basis: tbr.Basis, - } + var resp TxpoolBroadcastResponse if len(tbr.Transactions) != 0 { if len(tbr.Transactions) == 1 { // if there's only one transaction, best-effort check for parents @@ -430,9 +428,11 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { // implementor. In practice, this trade off is worth it. // In full index mode, any UTXO can have its proofs overwritten. // In personal index mode, only UTXOs that are registered to a wallet can have its proofs overwritten. - tbr.Basis, tbr.V2Transactions, err = s.wm.OverwriteElementProofs(tbr.V2Transactions) - if jc.Check("couldn't overwrite proofs", err) != nil { - return + if len(tbr.Transactions) == 0 && s.wm.IndexMode() == wallet.IndexModeFull { + tbr.Basis, tbr.V2Transactions, err = s.wm.OverwriteElementProofs(tbr.V2Transactions) + if jc.Check("couldn't overwrite proofs", err) != nil { + return + } } if len(tbr.V2Transactions) == 1 { From ea96db77f0b0717d6edc67b63a2fca7d69a2ea11 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 25 May 2025 09:58:04 -0700 Subject: [PATCH 471/630] fix localhost panic --- ...to_localhost_on_some_windows_11_systems.md | 5 +++ cmd/walletd/node.go | 32 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 .changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md diff --git a/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md b/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md new file mode 100644 index 0000000..26d8f16 --- /dev/null +++ b/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fixed a panic when listening to localhost on some Windows 11 systems. diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index c583f5a..eb7cbf3 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -98,6 +98,36 @@ func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error return d.ExternalIP() } +// startLocalhostListener https://github.com/SiaFoundation/hostd/issues/202 +func startLocalhostListener(listenAddr string, log *zap.Logger) (l net.Listener, err error) { + addr, port, err := net.SplitHostPort(listenAddr) + if err != nil { + return nil, fmt.Errorf("failed to parse API address: %w", err) + } + + // if the address is not localhost, listen on the address as-is + if addr != "localhost" { + return net.Listen("tcp", listenAddr) + } + + // localhost fails on some new installs of Windows 11, so try a few + // different addresses + tryAddresses := []string{ + net.JoinHostPort("localhost", port), // original address + net.JoinHostPort("127.0.0.1", port), // IPv4 loopback + net.JoinHostPort("::1", port), // IPv6 loopback + } + + for _, addr := range tryAddresses { + l, err = net.Listen("tcp", addr) + if err == nil { + return + } + log.Debug("failed to listen on fallback address", zap.String("address", addr), zap.Error(err)) + } + return +} + func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { f, err := os.Open(fp) if err != nil { @@ -161,7 +191,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } defer syncerListener.Close() - httpListener, err := net.Listen("tcp", cfg.HTTP.Address) + httpListener, err := startLocalhostListener(cfg.HTTP.Address, log) if err != nil { return fmt.Errorf("failed to listen on %q: %w", cfg.HTTP.Address, err) } From ac3016a871636de8c499b817ab601f05f62b6fa1 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 26 May 2025 08:17:08 -0700 Subject: [PATCH 472/630] update core --- .../update_core_to_v0130_and_coreutils_to_v0150.md | 5 +++++ go.mod | 6 +++--- go.sum | 12 ++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 .changeset/update_core_to_v0130_and_coreutils_to_v0150.md diff --git a/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md b/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md new file mode 100644 index 0000000..2918d11 --- /dev/null +++ b/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Update core to v0.13.0 and coreutils to v0.15.0 diff --git a/go.mod b/go.mod index 0cf4ec1..5c1cbd2 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230 - go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f + go.sia.tech/core v0.13.0 + go.sia.tech/coreutils v0.15.0 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 @@ -25,7 +25,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.51.0 // indirect + github.com/quic-go/quic-go v0.52.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect go.etcd.io/bbolt v1.4.0 // indirect go.sia.tech/mux v1.4.0 // indirect diff --git a/go.sum b/go.sum index cb28f33..e9145c5 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.51.0 h1:K8exxe9zXxeRKxaXxi/GpUqYiTrtdiWP8bo1KFya6Wc= -github.com/quic-go/quic-go v0.51.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= +github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= +github.com/quic-go/quic-go v0.52.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230 h1:o8+TZHq1+1j1Ww8QECvFVHRZ35hgCOaKpS7X6OR+N2Y= -go.sia.tech/core v0.12.5-0.20250526013225-8e48f53ca230/go.mod h1:NpWoU+7q9X+lPfqlkW4Xjyhwt9+DZpvf2WWmo4Ui4Ig= -go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f h1:VnaLaf4xrrVBWsMtiuDG7LVtbStHBMn3q+9qZYYxd0o= -go.sia.tech/coreutils v0.14.1-0.20250526014037-adeefa95f18f/go.mod h1:WsQjcHTvnCI10dU8anbH06kYggmyrS4+kxtkOD8KxE4= +go.sia.tech/core v0.13.0 h1:LulIZQe1A3DZ9/CyX1mcJvHy3zmw/0jEkZfuNp92D2w= +go.sia.tech/core v0.13.0/go.mod h1:hAsdf7uqD8+oBgg5pwxFFgW7Rv41kBKM1v6r3a7UZqc= +go.sia.tech/coreutils v0.15.0 h1:aL8K0beMaZ5vktfKNj605J+Rqlxy2Yc5pzcyWZcTpW8= +go.sia.tech/coreutils v0.15.0/go.mod h1:pdcuQatmqVc/rubyQgzl5k1qodqOmpMK7XTcpXhTbn0= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From c816964bbeed91f6775365dda526c3ee7f999054 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 26 May 2025 12:41:36 -0700 Subject: [PATCH 473/630] sqlite: fix database is locked --- ...valid_commitment_and_reset_if_necessary.md | 5 + .../fixed_sqlite_database_is_locked_error.md | 5 + .../reduced_batch_size_for_slower_hardware.md | 5 + internal/testutil/testutil.go | 2 +- persist/sqlite/address_test.go | 2 +- persist/sqlite/consensus.go | 4 +- persist/sqlite/consensus_test.go | 10 +- persist/sqlite/consts_default.go | 14 --- persist/sqlite/consts_testing.go | 14 --- persist/sqlite/events_test.go | 3 +- persist/sqlite/migrations_test.go | 7 +- persist/sqlite/options.go | 21 ++++ persist/sqlite/peers_test.go | 4 +- persist/sqlite/sql.go | 6 - persist/sqlite/store.go | 107 +++++++----------- wallet/wallet_test.go | 44 +++---- 16 files changed, 114 insertions(+), 139 deletions(-) create mode 100644 .changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md create mode 100644 .changeset/fixed_sqlite_database_is_locked_error.md create mode 100644 .changeset/reduced_batch_size_for_slower_hardware.md delete mode 100644 persist/sqlite/consts_default.go delete mode 100644 persist/sqlite/consts_testing.go create mode 100644 persist/sqlite/options.go diff --git a/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md b/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md new file mode 100644 index 0000000..c21bd15 --- /dev/null +++ b/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Check consensus database for invalid commitment and reset if necessary. diff --git a/.changeset/fixed_sqlite_database_is_locked_error.md b/.changeset/fixed_sqlite_database_is_locked_error.md new file mode 100644 index 0000000..ecfd15a --- /dev/null +++ b/.changeset/fixed_sqlite_database_is_locked_error.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fixed SQLite database is locked error. diff --git a/.changeset/reduced_batch_size_for_slower_hardware.md b/.changeset/reduced_batch_size_for_slower_hardware.md new file mode 100644 index 0000000..88a713e --- /dev/null +++ b/.changeset/reduced_batch_size_for_slower_hardware.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Reduced batch size for slower hardware. diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 9c90b02..7f01f2f 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -79,7 +79,7 @@ func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, } cm := chain.NewManager(dbstore, tipState) - store, err := sqlite.OpenDatabase(filepath.Join(tb.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + store, err := sqlite.OpenDatabase(filepath.Join(tb.TempDir(), "walletd.sqlite"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { tb.Fatal(err) } diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go index ecbefea..219f00a 100644 --- a/persist/sqlite/address_test.go +++ b/persist/sqlite/address_test.go @@ -20,7 +20,7 @@ func TestCheckAddresses(t *testing.T) { } // create a new database - db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 9aafb96..125f294 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -200,8 +200,8 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. return nil } - if state.Index.Height > spentElementRetentionBlocks { - pruneHeight := state.Index.Height - spentElementRetentionBlocks + if state.Index.Height > s.spentElementRetentionBlocks { + pruneHeight := state.Index.Height - s.spentElementRetentionBlocks siacoins, err := pruneSpentSiacoinElements(tx, pruneHeight) if err != nil { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 36adbf8..bb4e802 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -51,7 +51,7 @@ func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { func TestPruneSiacoins(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), WithLog(log.Named("sqlite3")), WithRetainSpentElements(20)) if err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestPruneSiacoins(t *testing.T) { assertUTXOs(0, 1) // mine until the payout matures - for i := 0; i < int(maturityHeight); i++ { + for range maturityHeight { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func TestPruneSiacoins(t *testing.T) { assertUTXOs(1, 0) // mine until the element is pruned - for i := 0; i < spentElementRetentionBlocks-1; i++ { + for i := uint64(0); i < db.spentElementRetentionBlocks-1; i++ { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } @@ -192,7 +192,7 @@ func TestPruneSiacoins(t *testing.T) { func TestPruneSiafunds(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -298,7 +298,7 @@ func TestPruneSiafunds(t *testing.T) { assertUTXOs(1, 0) // mine until the element is pruned - for i := 0; i < spentElementRetentionBlocks-1; i++ { + for i := uint64(0); i < db.spentElementRetentionBlocks-1; i++ { if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { t.Fatal(err) } diff --git a/persist/sqlite/consts_default.go b/persist/sqlite/consts_default.go deleted file mode 100644 index 19f84e5..0000000 --- a/persist/sqlite/consts_default.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !testing - -package sqlite - -import "time" - -const ( - busyTimeout = 10000 // 10 seconds - maxRetryAttempts = 30 // 30 attempts - factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds - maxBackoff = 15 * time.Second - - spentElementRetentionBlocks = 144 // 1 day -) diff --git a/persist/sqlite/consts_testing.go b/persist/sqlite/consts_testing.go deleted file mode 100644 index e4ade1e..0000000 --- a/persist/sqlite/consts_testing.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build testing - -package sqlite - -import "time" - -const ( - busyTimeout = 100 // 100ms - maxRetryAttempts = 10 // 10 attempts - factor = 2.0 // factor ^ retryAttempts = backoff time in milliseconds - maxBackoff = 15 * time.Second - - spentElementRetentionBlocks = 36 -) diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go index fd00c1d..eeddb72 100644 --- a/persist/sqlite/events_test.go +++ b/persist/sqlite/events_test.go @@ -7,13 +7,12 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap" "lukechampine.com/frand" ) func runBenchmarkWalletEvents(b *testing.B, name string, addresses, eventsPerAddress int) { b.Run(name, func(b *testing.B) { - db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3"), zap.NewNop()) + db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3")) if err != nil { b.Fatal(err) } diff --git a/persist/sqlite/migrations_test.go b/persist/sqlite/migrations_test.go index 369e05e..e01545d 100644 --- a/persist/sqlite/migrations_test.go +++ b/persist/sqlite/migrations_test.go @@ -7,7 +7,6 @@ import ( "testing" "go.sia.tech/core/types" - "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -149,7 +148,7 @@ func TestMigrationConsistency(t *testing.T) { expectedVersion := int64(len(migrations) + 1) log := zaptest.NewLogger(t) - store, err := OpenDatabase(fp, log) + store, err := OpenDatabase(fp, WithLog(log)) if err != nil { t.Fatal(err) } @@ -162,7 +161,7 @@ func TestMigrationConsistency(t *testing.T) { } // ensure the database does not change version when opened again - store, err = OpenDatabase(fp, log) + store, err = OpenDatabase(fp, WithLog(log)) if err != nil { t.Fatal(err) } @@ -173,7 +172,7 @@ func TestMigrationConsistency(t *testing.T) { } fp2 := filepath.Join(t.TempDir(), "walletd.sqlite3") - baseline, err := OpenDatabase(fp2, zap.NewNop()) + baseline, err := OpenDatabase(fp2) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/options.go b/persist/sqlite/options.go new file mode 100644 index 0000000..2497731 --- /dev/null +++ b/persist/sqlite/options.go @@ -0,0 +1,21 @@ +package sqlite + +import "go.uber.org/zap" + +// An Option is a function that configures the Store. +type Option func(*Store) + +// WithLog sets the logger for the store. +func WithLog(log *zap.Logger) Option { + return func(s *Store) { + s.log = log + } +} + +// WithRetainSpentElements sets the number of blocks to retain +// spent elements. +func WithRetainSpentElements(blocks uint64) Option { + return func(s *Store) { + s.spentElementRetentionBlocks = blocks + } +} diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index 0ef91dc..a6ec068 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -12,7 +12,7 @@ import ( func TestAddPeer(t *testing.T) { log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestAddPeer(t *testing.T) { func TestBanPeer(t *testing.T) { log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index 5b54163..a0d6541 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -3,7 +3,6 @@ package sqlite import ( "context" "database/sql" - "math/rand" "strings" "time" @@ -185,11 +184,6 @@ func setDBVersion(tx *txn, version int64) error { return tx.QueryRow(query, version).Scan(&dbID) } -// jitterSleep sleeps for a random duration between t and t*1.5. -func jitterSleep(t time.Duration) { - time.Sleep(t + time.Duration(rand.Int63n(int64(t/2)))) -} - func queryPlaceHolders(n int) string { if n == 0 { return "" diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index f73d559..d282389 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -2,23 +2,21 @@ package sqlite import ( "database/sql" - "encoding/hex" "errors" "fmt" - "math" "strings" "time" "github.com/mattn/go-sqlite3" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" - "lukechampine.com/frand" ) type ( // A Store is a persistent store that uses a SQL database as its backend. Store struct { - indexMode wallet.IndexMode + indexMode wallet.IndexMode + spentElementRetentionBlocks uint64 // number of blocks to retain spent elements db *sql.DB log *zap.Logger @@ -32,95 +30,72 @@ func (s *Store) Close() error { // transaction executes a function within a database transaction. If the // function returns an error, the transaction is rolled back. Otherwise, the -// transaction is committed. If the transaction fails due to a busy error, it is -// retried up to 10 times before returning. +// transaction is committed. func (s *Store) transaction(fn func(*txn) error) error { - var err error - txnID := hex.EncodeToString(frand.Bytes(4)) - log := s.log.Named("transaction").With(zap.String("id", txnID)) - start := time.Now() - attempt := 1 - for ; attempt < maxRetryAttempts; attempt++ { - attemptStart := time.Now() - log := log.With(zap.Int("attempt", attempt)) - err = doTransaction(s.db, log, fn) - if err == nil { - // no error, break out of the loop - return nil - } - - // return immediately if the error is not a busy error - if !strings.Contains(err.Error(), "database is locked") { - break - } - // exponential backoff - sleep := time.Duration(math.Pow(factor, float64(attempt))) * time.Millisecond - if sleep > maxBackoff { - sleep = maxBackoff - } - log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) - jitterSleep(sleep) - } - return fmt.Errorf("transaction failed (attempt %d): %w", attempt, err) -} + log := s.log.Named("transaction") -func sqliteFilepath(fp string) string { - params := []string{ - fmt.Sprintf("_busy_timeout=%d", busyTimeout), - "_foreign_keys=true", - "_journal_mode=WAL", - "_secure_delete=false", - "_cache_size=-65536", // 64MiB - } - return "file:" + fp + "?" + strings.Join(params, "&") -} - -// doTransaction is a helper function to execute a function within a transaction. If fn returns -// an error, the transaction is rolled back. Otherwise, the transaction is -// committed. -func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { - dbtx, err := db.Begin() + start := time.Now() + tx, err := s.db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } - start := time.Now() defer func() { - if err := dbtx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { log.Error("failed to rollback transaction", zap.Error(err)) } - // log the transaction if it took longer than txn duration - if time.Since(start) > longTxnDuration { - log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) - } }() - - tx := &txn{ - Tx: dbtx, + if err := fn(&txn{ + Tx: tx, log: log, - } - if err := fn(tx); err != nil { + }); err != nil { return err - } else if err := tx.Commit(); err != nil { + } + // log the transaction if it took longer than txn duration + if time.Since(start) > longTxnDuration { + log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) + } + // commit the transaction + if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } return nil } +func sqliteFilepath(fp string) string { + params := []string{ + fmt.Sprintf("_busy_timeout=%d", 300000), // 300 seconds + "_foreign_keys=true", + "_journal_mode=WAL", + "_secure_delete=false", + "_cache_size=-65536", // 64MiB + } + return "file:" + fp + "?" + strings.Join(params, "&") +} + // OpenDatabase creates a new SQLite store and initializes the database. If the // database does not exist, it is created. -func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { +func OpenDatabase(fp string, opts ...Option) (*Store, error) { db, err := sql.Open("sqlite3", sqliteFilepath(fp)) if err != nil { return nil, err } + // set the number of open connections to 1 to prevent "database is locked" + // errors + db.SetMaxOpenConns(1) + store := &Store{ - db: db, - log: log, + db: db, + + log: zap.NewNop(), + spentElementRetentionBlocks: 144, // default to 144 blocks (1 day) + } + for _, opt := range opts { + opt(store) } if err := store.init(); err != nil { return nil, err } sqliteVersion, _, _ := sqlite3.Version() - log.Debug("database initialized", zap.String("sqliteVersion", sqliteVersion), zap.Int("schemaVersion", len(migrations)+1), zap.String("path", fp)) + store.log.Debug("database initialized", zap.String("sqliteVersion", sqliteVersion), zap.Int("schemaVersion", len(migrations)+1), zap.String("path", fp)) return store, nil } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index d32c1a5..107fba6 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -112,7 +112,7 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty func TestReserve(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -186,7 +186,7 @@ func TestReserve(t *testing.T) { func TestSelectSiacoins(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -355,7 +355,7 @@ func TestSelectSiacoins(t *testing.T) { func TestSelectSiafunds(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -502,7 +502,7 @@ func TestReorg(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -710,7 +710,7 @@ func TestEphemeralBalance(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -906,7 +906,7 @@ func TestEphemeralBalance(t *testing.T) { func TestWalletAddresses(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -1033,7 +1033,7 @@ func TestWalletAddresses(t *testing.T) { func TestScan(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -1200,7 +1200,7 @@ func TestScan(t *testing.T) { func TestSiafunds(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -1361,7 +1361,7 @@ func TestOrphans(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -1558,7 +1558,7 @@ func TestFullIndex(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -1782,7 +1782,7 @@ func TestEvents(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2034,7 +2034,7 @@ func TestEvents(t *testing.T) { func TestWalletUnconfirmedEvents(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2222,7 +2222,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { func TestAddressUnconfirmedEvents(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2430,7 +2430,7 @@ func TestV2(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2538,7 +2538,7 @@ func TestV2(t *testing.T) { func TestScanV2(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2733,7 +2733,7 @@ func TestReorgV2(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -2961,7 +2961,7 @@ func TestOrphansV2(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -3175,7 +3175,7 @@ func TestDeleteWallet(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -3273,7 +3273,7 @@ func TestEventTypes(t *testing.T) { log := zap.NewNop() dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -3859,7 +3859,7 @@ func TestSiafundClaims(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -4105,7 +4105,7 @@ func TestV2SiafundClaims(t *testing.T) { log := zaptest.NewLogger(t) dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -4392,7 +4392,7 @@ func TestReset(t *testing.T) { } } - db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) + db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } From 35ca08b5e0b697854aaea2099c7e1fe8d1722661 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 26 May 2025 12:41:45 -0700 Subject: [PATCH 474/630] cmd: reduce batch size for slower hardware --- cmd/walletd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index eb78e7c..5a3d554 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -73,7 +73,7 @@ var cfg = config.Config{ }, Index: config.Index{ Mode: wallet.IndexModePersonal, - BatchSize: 1000, + BatchSize: 10, }, Log: config.Log{ Level: zap.NewAtomicLevelAt(zap.InfoLevel), From 8e977c392c3aeecb2ca38512f3bea5ed1c6af8e8 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 26 May 2025 12:42:29 -0700 Subject: [PATCH 475/630] cmd: migrate commitments --- cmd/walletd/node.go | 65 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index eb7cbf3..497da3f 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -146,7 +146,59 @@ func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { return &network.Network, network.Genesis, nil } +// migrateConsensusDB checks if the consensus database needs to be migrated +// to match the new v2 commitment. +func migrateConsensusDB(fp string, n *consensus.Network, genesis types.Block, log *zap.Logger) error { + bdb, err := coreutils.OpenBoltChainDB(fp) + if err != nil { + return fmt.Errorf("failed to open consensus database: %w", err) + } + defer bdb.Close() + + dbstore, tipState, err := chain.NewDBStore(bdb, n, genesis, chain.NewZapMigrationLogger(log.Named("chaindb"))) + if err != nil { + return fmt.Errorf("failed to create chain store: %w", err) + } else if tipState.Index.Height < n.HardforkV2.AllowHeight { + return nil // no migration needed, the chain is still on v1 + } + + log.Debug("checking for v2 commitment migration") + b, _, ok := dbstore.Block(tipState.Index.ID) + if !ok { + return fmt.Errorf("failed to get tip block %q", tipState.Index) + } else if b.V2 == nil { + log.Debug("tip block is not a v2 block, skipping commitment migration") + return nil + } + + parentState, ok := dbstore.State(b.ParentID) + if !ok { + return fmt.Errorf("failed to get parent state for tip block %q", b.ParentID) + } + commitment := parentState.Commitment(b.MinerPayouts[0].Address, b.Transactions, b.V2Transactions()) + log = log.With(zap.Stringer("tip", b.ID()), zap.Stringer("commitment", b.V2.Commitment), zap.Stringer("expected", commitment)) + if b.V2.Commitment == commitment { + log.Debug("tip block commitment matches parent state, no migration needed") + return nil + } + // reset the database if the commitment is not a merkle root + log.Debug("resetting consensus database for new v2 commitment") + if err := bdb.Close(); err != nil { + return fmt.Errorf("failed to close old consensus database: %w", err) + } else if err := os.RemoveAll(fp); err != nil { + return fmt.Errorf("failed to remove old consensus database: %w", err) + } + log.Debug("consensus database reset") + return nil +} + func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { + store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) + if err != nil { + return fmt.Errorf("failed to open wallet database: %w", err) + } + defer store.Close() + var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string @@ -173,7 +225,12 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } } - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) + consensusPath := filepath.Join(cfg.Directory, "consensus.db") + if err := migrateConsensusDB(consensusPath, network, genesisBlock, log.Named("migrate")); err != nil { + return fmt.Errorf("failed to open consensus database: %w", err) + } + + bdb, err := coreutils.OpenBoltChainDB(consensusPath) if err != nil { return fmt.Errorf("failed to open consensus database: %w", err) } @@ -219,12 +276,6 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { syncerAddr = net.JoinHostPort("127.0.0.1", port) } - store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), log.Named("sqlite3")) - if err != nil { - return fmt.Errorf("failed to open wallet database: %w", err) - } - defer store.Close() - if cfg.Syncer.Bootstrap { for _, peer := range bootstrapPeers { if err := store.AddPeer(peer); err != nil { From 105420f5fc8c30a7878c3e32d187a5c98b687e88 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 26 May 2025 12:52:01 -0700 Subject: [PATCH 476/630] seconds -> milliseconds --- persist/sqlite/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index d282389..557beb1 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -63,7 +63,7 @@ func (s *Store) transaction(fn func(*txn) error) error { func sqliteFilepath(fp string) string { params := []string{ - fmt.Sprintf("_busy_timeout=%d", 300000), // 300 seconds + fmt.Sprintf("_busy_timeout=%d", time.Minute.Milliseconds()), "_foreign_keys=true", "_journal_mode=WAL", "_secure_delete=false", From ea6b79ef011bdf6d156dcb735feee64045770e79 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 19:54:11 +0000 Subject: [PATCH 477/630] chore: prepare release 2.8.0 --- ...or_invalid_commitment_and_reset_if_necessary.md | 5 ----- ...ning_to_localhost_on_some_windows_11_systems.md | 5 ----- .../fixed_sqlite_database_is_locked_error.md | 5 ----- ...2_transaction_proofs_when_in_full_index_mode.md | 5 ----- .../reduced_batch_size_for_slower_hardware.md | 5 ----- .../update_core_to_v0130_and_coreutils_to_v0150.md | 5 ----- CHANGELOG.md | 14 ++++++++++++++ go.mod | 2 +- 8 files changed, 15 insertions(+), 31 deletions(-) delete mode 100644 .changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md delete mode 100644 .changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md delete mode 100644 .changeset/fixed_sqlite_database_is_locked_error.md delete mode 100644 .changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md delete mode 100644 .changeset/reduced_batch_size_for_slower_hardware.md delete mode 100644 .changeset/update_core_to_v0130_and_coreutils_to_v0150.md diff --git a/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md b/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md deleted file mode 100644 index c21bd15..0000000 --- a/.changeset/check_consensus_database_for_invalid_commitment_and_reset_if_necessary.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Check consensus database for invalid commitment and reset if necessary. diff --git a/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md b/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md deleted file mode 100644 index 26d8f16..0000000 --- a/.changeset/fixed_a_panic_when_listening_to_localhost_on_some_windows_11_systems.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fixed a panic when listening to localhost on some Windows 11 systems. diff --git a/.changeset/fixed_sqlite_database_is_locked_error.md b/.changeset/fixed_sqlite_database_is_locked_error.md deleted file mode 100644 index ecfd15a..0000000 --- a/.changeset/fixed_sqlite_database_is_locked_error.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fixed SQLite database is locked error. diff --git a/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md b/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md deleted file mode 100644 index f0d8ed5..0000000 --- a/.changeset/implicitly_fill_v2_transaction_proofs_when_in_full_index_mode.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Implicitly fill v2 transaction proofs when in full index mode diff --git a/.changeset/reduced_batch_size_for_slower_hardware.md b/.changeset/reduced_batch_size_for_slower_hardware.md deleted file mode 100644 index 88a713e..0000000 --- a/.changeset/reduced_batch_size_for_slower_hardware.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Reduced batch size for slower hardware. diff --git a/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md b/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md deleted file mode 100644 index 2918d11..0000000 --- a/.changeset/update_core_to_v0130_and_coreutils_to_v0150.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Update core to v0.13.0 and coreutils to v0.15.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index e63b54e..8ea8509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 2.8.0 (2025-05-26) + +### Features + +- Implicitly fill v2 transaction proofs when in full index mode +- Update core to v0.13.0 and coreutils to v0.15.0 + +### Fixes + +- Check consensus database for invalid commitment and reset if necessary. +- Fixed a panic when listening to localhost on some Windows 11 systems. +- Fixed SQLite database is locked error. +- Reduced batch size for slower hardware. + ## 2.7.0 (2025-05-23) ### Features diff --git a/go.mod b/go.mod index 5c1cbd2..c297523 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.7.0 +module go.sia.tech/walletd/v2 // v2.8.0 go 1.23.2 From 8c8472446b9f4313831f7becae23e6a064b59a16 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 28 May 2025 21:34:01 -0700 Subject: [PATCH 478/630] update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c297523..01c21f9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.0 - go.sia.tech/coreutils v0.15.0 + go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index e9145c5..381c98f 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.13.0 h1:LulIZQe1A3DZ9/CyX1mcJvHy3zmw/0jEkZfuNp92D2w= go.sia.tech/core v0.13.0/go.mod h1:hAsdf7uqD8+oBgg5pwxFFgW7Rv41kBKM1v6r3a7UZqc= -go.sia.tech/coreutils v0.15.0 h1:aL8K0beMaZ5vktfKNj605J+Rqlxy2Yc5pzcyWZcTpW8= -go.sia.tech/coreutils v0.15.0/go.mod h1:pdcuQatmqVc/rubyQgzl5k1qodqOmpMK7XTcpXhTbn0= +go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0 h1:QMCP115mCR45o2vQVBDeBELbWcPf7oFJRu31F78mmPc= +go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0/go.mod h1:pdcuQatmqVc/rubyQgzl5k1qodqOmpMK7XTcpXhTbn0= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 929d535a9486ccde10da8df63017290ce6a6f531 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 12:41:52 -0700 Subject: [PATCH 479/630] update coreutils --- .changeset/update_coreutils_to_v0151.md | 5 +++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changeset/update_coreutils_to_v0151.md diff --git a/.changeset/update_coreutils_to_v0151.md b/.changeset/update_coreutils_to_v0151.md new file mode 100644 index 0000000..31b6d2e --- /dev/null +++ b/.changeset/update_coreutils_to_v0151.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils to v0.15.1 diff --git a/go.mod b/go.mod index 01c21f9..25b89f6 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.24.1 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.0 - go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0 + go.sia.tech/coreutils v0.15.1 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 381c98f..48fd34c 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.13.0 h1:LulIZQe1A3DZ9/CyX1mcJvHy3zmw/0jEkZfuNp92D2w= go.sia.tech/core v0.13.0/go.mod h1:hAsdf7uqD8+oBgg5pwxFFgW7Rv41kBKM1v6r3a7UZqc= -go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0 h1:QMCP115mCR45o2vQVBDeBELbWcPf7oFJRu31F78mmPc= -go.sia.tech/coreutils v0.15.1-0.20250529042731-b4b068aceaa0/go.mod h1:pdcuQatmqVc/rubyQgzl5k1qodqOmpMK7XTcpXhTbn0= +go.sia.tech/coreutils v0.15.1 h1:Oz8VChGjWvf4XpaxMU2uTfv24TWULNsBucJZoFbqp+o= +go.sia.tech/coreutils v0.15.1/go.mod h1:3FlGuTHbPMxZhYYm4l0j7Ch78Ag0cvpHdmSYlygcvyM= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From df071b8895027825fe4a09da98347e9a125b3086 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 May 2025 19:43:11 +0000 Subject: [PATCH 480/630] chore: prepare release 2.8.1 --- .changeset/update_coreutils_to_v0151.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_coreutils_to_v0151.md diff --git a/.changeset/update_coreutils_to_v0151.md b/.changeset/update_coreutils_to_v0151.md deleted file mode 100644 index 31b6d2e..0000000 --- a/.changeset/update_coreutils_to_v0151.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to v0.15.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ea8509..c49e9e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.8.1 (2025-05-29) + +### Fixes + +- Update coreutils to v0.15.1 + ## 2.8.0 (2025-05-26) ### Features diff --git a/go.mod b/go.mod index 25b89f6..d77e58d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.8.0 +module go.sia.tech/walletd/v2 // v2.8.1 go 1.23.2 From a8023d8b8f12370b64cf1a2b7c45dee547e23b3b Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 12:49:25 -0700 Subject: [PATCH 481/630] update core and coreutils --- .../updated_core_to_v0131_and_coreutils_to_v0152.md | 5 +++++ .changeset/updated_dockerfile_to_go_124.md | 5 +++++ Dockerfile | 2 +- go.mod | 8 +++----- go.sum | 8 ++++---- 5 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 .changeset/updated_core_to_v0131_and_coreutils_to_v0152.md create mode 100644 .changeset/updated_dockerfile_to_go_124.md diff --git a/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md b/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md new file mode 100644 index 0000000..c17e819 --- /dev/null +++ b/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Updated core to v0.13.1 and coreutils to v0.15.2. diff --git a/.changeset/updated_dockerfile_to_go_124.md b/.changeset/updated_dockerfile_to_go_124.md new file mode 100644 index 0000000..ff11335 --- /dev/null +++ b/.changeset/updated_dockerfile_to_go_124.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Updated build to use Go 1.24 diff --git a/Dockerfile b/Dockerfile index 3ad2c4d..a28e642 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/golang:1.23 AS builder +FROM docker.io/library/golang:1.24 AS builder WORKDIR /walletd diff --git a/go.mod b/go.mod index d77e58d..67eed18 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,11 @@ module go.sia.tech/walletd/v2 // v2.8.1 -go 1.23.2 - -toolchain go1.24.1 +go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.13.0 - go.sia.tech/coreutils v0.15.1 + go.sia.tech/core v0.13.1 + go.sia.tech/coreutils v0.15.2 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.2 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 48fd34c..03992ce 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.13.0 h1:LulIZQe1A3DZ9/CyX1mcJvHy3zmw/0jEkZfuNp92D2w= -go.sia.tech/core v0.13.0/go.mod h1:hAsdf7uqD8+oBgg5pwxFFgW7Rv41kBKM1v6r3a7UZqc= -go.sia.tech/coreutils v0.15.1 h1:Oz8VChGjWvf4XpaxMU2uTfv24TWULNsBucJZoFbqp+o= -go.sia.tech/coreutils v0.15.1/go.mod h1:3FlGuTHbPMxZhYYm4l0j7Ch78Ag0cvpHdmSYlygcvyM= +go.sia.tech/core v0.13.1 h1:dBKzZBhWZsgdV7qZa6qiaZtTDj5evvSqYWpeGNenlRI= +go.sia.tech/core v0.13.1/go.mod h1:oMOgHT4bf9VSXUCOgtt9w4MFns/pY0LRUgwyMXdxW5w= +go.sia.tech/coreutils v0.15.2 h1:2oEe8wpsmU5WVNfe0x75URhno+lPSXc+ozRtZNgjzu4= +go.sia.tech/coreutils v0.15.2/go.mod h1:Kz/VQViqymnR1EW7DDdKrQru8dMFxiY44/qmjriilWs= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 75a7d799c7fca7ed87e88944adc5a993dc8a8751 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 May 2025 10:57:08 +0200 Subject: [PATCH 482/630] api: allow adding addresses in batch --- api/api_test.go | 22 ++++++++++++++++++++++ api/client.go | 7 +++++++ api/server.go | 17 ++++++++++++++++- persist/sqlite/wallet.go | 28 +++++++++++++++++----------- wallet/manager.go | 8 ++++---- 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index c73d5ca..3e21c84 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -492,6 +492,28 @@ func TestAddresses(t *testing.T) { } else if !balance.ImmatureSiacoins.IsZero() { t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } + + // create new wallet + w, err = c.AddWallet(api.WalletUpdateRequest{Name: t.Name()}) + if err != nil { + t.Fatal(err) + } + wc = c.Wallet(w.ID) + + // create two addresses + pk1 := types.GeneratePrivateKey() + pk2 := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk1.PublicKey()) + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // assert multpile addresses can be added to a wallet + if err := wc.AddAddresses([]wallet.Address{{Address: addr1}, {Address: addr2}}); err != nil { + t.Fatal(err) + } else if addrs, err := wc.Addresses(); err != nil { + t.Fatal(err) + } else if len(addrs) != 2 { + t.Fatalf("expected 2 addresses, got %d", len(addrs)) + } } func TestConsensus(t *testing.T) { diff --git a/api/client.go b/api/client.go index 13c7ef9..98404cd 100644 --- a/api/client.go +++ b/api/client.go @@ -329,6 +329,13 @@ func (c *WalletClient) AddAddress(a wallet.Address) (err error) { return } +// AddAddresses adds the specified batch of addresses and associated metadata to +// the wallet. +func (c *WalletClient) AddAddresses(addrs []wallet.Address) (err error) { + err = c.c.PUT(context.Background(), fmt.Sprintf("/wallets/%v/addresses/batch", c.id), addrs) + return +} + // RemoveAddress removes the specified address from the wallet. func (c *WalletClient) RemoveAddress(addr types.Address) (err error) { err = c.c.DELETE(context.Background(), fmt.Sprintf("/wallets/%v/addresses/%v", c.id, addr)) diff --git a/api/server.go b/api/server.go index c24bccc..8dd6a2b 100644 --- a/api/server.go +++ b/api/server.go @@ -102,7 +102,7 @@ type ( DeleteWallet(wallet.ID) error Wallets() ([]wallet.Wallet, error) - AddAddress(id wallet.ID, addr wallet.Address) error + AddAddress(id wallet.ID, addrs ...wallet.Address) error RemoveAddress(id wallet.ID, addr types.Address) error Addresses(id wallet.ID) ([]wallet.Address, error) WalletAddress(wallet.ID, types.Address) (wallet.Address, error) @@ -609,6 +609,20 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { jc.Encode(nil) } +func (s *server) walletsAddressBatchHandlerPUT(jc jape.Context) { + var id wallet.ID + var addrs []wallet.Address + if jc.DecodeParam("id", &id) != nil || jc.Decode(&addrs) != nil { + return + } else if len(addrs) > 1000 { + jc.Error(fmt.Errorf("number of addresses exceeds the maximum batch size of 1000"), http.StatusBadRequest) + return + } else if jc.Check("couldn't add addresses", s.wm.AddAddress(id, addrs...)) != nil { + return + } + jc.Encode(nil) +} + func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { var id wallet.ID var addr types.Address @@ -1574,6 +1588,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), + "PUT /wallets/:id/addresses/batch": wrapAuthHandler(srv.walletsAddressBatchHandlerPUT), "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index c8c8ca5..9bc8692 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -144,25 +144,31 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { return } -// AddWalletAddress adds an address to a wallet. -func (s *Store) AddWalletAddress(id wallet.ID, addr wallet.Address) error { +// AddWalletAddress adds the given addresses to a wallet. +func (s *Store) AddWalletAddress(id wallet.ID, addr ...wallet.Address) error { return s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err } - addressID, err := insertAddress(tx, addr.Address) - if err != nil { - return fmt.Errorf("failed to insert address: %w", err) - } + for _, addr := range addr { + addressID, err := insertAddress(tx, addr.Address) + if err != nil { + return fmt.Errorf("failed to insert address %q: %w", addr.Address, err) + } + + var encodedPolicy any + if addr.SpendPolicy != nil { + encodedPolicy = encode(*addr.SpendPolicy) + } - var encodedPolicy any - if addr.SpendPolicy != nil { - encodedPolicy = encode(*addr.SpendPolicy) + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) + if err != nil { + return fmt.Errorf("failed to insert wallet address %q: %w", addr.Address, err) + } } - _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) - return err + return nil }) } diff --git a/wallet/manager.go b/wallet/manager.go index 2ab1d0e..d346505 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -91,7 +91,7 @@ type ( WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) - AddWalletAddress(walletID ID, address Address) error + AddWalletAddress(walletID ID, addresses ...Address) error RemoveWalletAddress(walletID ID, address types.Address) error AddressBalance(address types.Address) (balance Balance, err error) @@ -263,9 +263,9 @@ func (m *Manager) Wallets() ([]Wallet, error) { return m.store.Wallets() } -// AddAddress adds the given address to the given wallet. -func (m *Manager) AddAddress(walletID ID, addr Address) error { - return m.store.AddWalletAddress(walletID, addr) +// AddAddress adds the addresses to the given wallet. +func (m *Manager) AddAddress(walletID ID, addrs ...Address) error { + return m.store.AddWalletAddress(walletID, addrs...) } // RemoveAddress removes the given address from the given wallet. From 945ceee5fdc699cd855444086cd21ce653a7f73c Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 May 2025 11:02:33 +0200 Subject: [PATCH 483/630] docs: add changelog --- ..._allow_adding_multiple_addresses_to_a_wallet_at_a_time.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md diff --git a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md new file mode 100644 index 0000000..a936f6c --- /dev/null +++ b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Add batch endpoint to allow adding multiple addresses to a wallet at a time. From a6173d131566e8ea6c0fb20e5e814860d16a70fc Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 29 May 2025 11:14:36 +0200 Subject: [PATCH 484/630] docs: bump to minor --- ..._to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md index a936f6c..d516e87 100644 --- a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md +++ b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md @@ -1,5 +1,5 @@ --- -default: patch +default: minor --- # Add batch endpoint to allow adding multiple addresses to a wallet at a time. From 38a86fe639559d3c43a53a45429ddae3397be9d5 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:07:32 -0700 Subject: [PATCH 485/630] use prepared statement for batch insertion --- api/api_test.go | 2 + api/client.go | 2 +- api/server.go | 14 +++--- persist/sqlite/address_test.go | 2 +- persist/sqlite/consensus_test.go | 4 +- persist/sqlite/events_test.go | 2 +- persist/sqlite/wallet.go | 53 +++++++++++++++----- persist/sqlite/wallet_test.go | 84 ++++++++++++++++++++++++++++++++ wallet/manager.go | 8 +-- wallet/wallet_test.go | 50 +++++++++---------- 10 files changed, 168 insertions(+), 53 deletions(-) create mode 100644 persist/sqlite/wallet_test.go diff --git a/api/api_test.go b/api/api_test.go index 3e21c84..1766bc8 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1761,6 +1761,8 @@ func TestEphemeralTransactions(t *testing.T) { } func TestBroadcastRace(t *testing.T) { + t.Skip("NDF") // TODO: fix + log := zap.NewNop() pk := types.GeneratePrivateKey() sp := types.SpendPolicy{ diff --git a/api/client.go b/api/client.go index 98404cd..03f6bf0 100644 --- a/api/client.go +++ b/api/client.go @@ -332,7 +332,7 @@ func (c *WalletClient) AddAddress(a wallet.Address) (err error) { // AddAddresses adds the specified batch of addresses and associated metadata to // the wallet. func (c *WalletClient) AddAddresses(addrs []wallet.Address) (err error) { - err = c.c.PUT(context.Background(), fmt.Sprintf("/wallets/%v/addresses/batch", c.id), addrs) + err = c.c.PUT(context.Background(), fmt.Sprintf("/wallets/%v/batch/addresses", c.id), addrs) return } diff --git a/api/server.go b/api/server.go index 8dd6a2b..b896913 100644 --- a/api/server.go +++ b/api/server.go @@ -102,7 +102,7 @@ type ( DeleteWallet(wallet.ID) error Wallets() ([]wallet.Wallet, error) - AddAddress(id wallet.ID, addrs ...wallet.Address) error + AddAddresses(id wallet.ID, addrs ...wallet.Address) error RemoveAddress(id wallet.ID, addr types.Address) error Addresses(id wallet.ID) ([]wallet.Address, error) WalletAddress(wallet.ID, types.Address) (wallet.Address, error) @@ -603,21 +603,21 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { var addr wallet.Address if jc.DecodeParam("id", &id) != nil || jc.Decode(&addr) != nil { return - } else if jc.Check("couldn't add address", s.wm.AddAddress(id, addr)) != nil { + } else if jc.Check("couldn't add address", s.wm.AddAddresses(id, addr)) != nil { return } jc.Encode(nil) } -func (s *server) walletsAddressBatchHandlerPUT(jc jape.Context) { +func (s *server) walletsBatchAddressesHandlerPUT(jc jape.Context) { var id wallet.ID var addrs []wallet.Address if jc.DecodeParam("id", &id) != nil || jc.Decode(&addrs) != nil { return - } else if len(addrs) > 1000 { - jc.Error(fmt.Errorf("number of addresses exceeds the maximum batch size of 1000"), http.StatusBadRequest) + } else if len(addrs) > 10000 { + jc.Error(fmt.Errorf("number of addresses exceeds the maximum batch size of 10000"), http.StatusBadRequest) return - } else if jc.Check("couldn't add addresses", s.wm.AddAddress(id, addrs...)) != nil { + } else if jc.Check("couldn't add addresses", s.wm.AddAddresses(id, addrs...)) != nil { return } jc.Encode(nil) @@ -1588,9 +1588,9 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), - "PUT /wallets/:id/addresses/batch": wrapAuthHandler(srv.walletsAddressBatchHandlerPUT), "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), + "PUT /wallets/:id/batch/addresses": wrapAuthHandler(srv.walletsBatchAddressesHandlerPUT), "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), "GET /wallets/:id/events": wrapAuthHandler(srv.walletsEventsHandler), "POST /wallets/:id/construct/transaction": wrapAuthHandler(srv.walletsConstructHandler), diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go index 219f00a..135b718 100644 --- a/persist/sqlite/address_test.go +++ b/persist/sqlite/address_test.go @@ -38,7 +38,7 @@ func TestCheckAddresses(t *testing.T) { w, err := db.AddWallet(wallet.Wallet{}) if err != nil { t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{ + } else if err := db.AddWalletAddresses(w.ID, wallet.Address{ Address: address, }); err != nil { t.Fatal(err) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index bb4e802..6e70456 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -79,7 +79,7 @@ func TestPruneSiacoins(t *testing.T) { w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := db.AddWalletAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -222,7 +222,7 @@ func TestPruneSiafunds(t *testing.T) { w, err := db.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := db.AddWalletAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go index eeddb72..3197432 100644 --- a/persist/sqlite/events_test.go +++ b/persist/sqlite/events_test.go @@ -27,7 +27,7 @@ func runBenchmarkWalletEvents(b *testing.B, name string, addresses, eventsPerAdd for i := 0; i < addresses; i++ { addr := types.Address(frand.Entropy256()) - if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + if err := db.AddWalletAddresses(w.ID, wallet.Address{Address: addr}); err != nil { b.Fatal(err) } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 9bc8692..9075697 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -144,30 +144,44 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { return } -// AddWalletAddress adds the given addresses to a wallet. -func (s *Store) AddWalletAddress(id wallet.ID, addr ...wallet.Address) error { +// AddWalletAddresses adds the given addresses to a wallet. +func (s *Store) AddWalletAddresses(id wallet.ID, addr ...wallet.Address) error { return s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err + } else if len(addr) == 0 { + return errors.New("no addresses to add") } - for _, addr := range addr { - addressID, err := insertAddress(tx, addr.Address) - if err != nil { - return fmt.Errorf("failed to insert address %q: %w", addr.Address, err) - } + addresses := make([]types.Address, 0, len(addr)) + for _, a := range addr { + addresses = append(addresses, a.Address) + } + + addressDBIDs, err := insertAddress(tx, addresses...) + if err != nil { + return fmt.Errorf("failed to insert addresses: %w", err) + } + + stmt, err := tx.Prepare(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`) + if err != nil { + return fmt.Errorf("failed to prepare wallet address insert statement: %w", err) + } + defer stmt.Close() + + for i, addr := range addr { + addressDBID := addressDBIDs[i] var encodedPolicy any if addr.SpendPolicy != nil { encodedPolicy = encode(*addr.SpendPolicy) } - _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) + _, err = stmt.Exec(id, addressDBID, addr.Description, encodedPolicy, addr.Metadata) if err != nil { return fmt.Errorf("failed to insert wallet address %q: %w", addr.Address, err) } } - return nil }) } @@ -644,13 +658,28 @@ func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { return } -func insertAddress(tx *txn, addr types.Address) (id int64, err error) { +func insertAddress(tx *txn, addrs ...types.Address) (ids []int64, err error) { const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id` - err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency)).Scan(&id) - return + if len(addrs) == 0 { + return nil, errors.New("no addresses to insert") + } + + stmt, err := tx.Prepare(query) + if err != nil { + return nil, fmt.Errorf("failed to prepare address insert statement: %w", err) + } + defer stmt.Close() + for _, addr := range addrs { + var id int64 + if err := stmt.QueryRow(encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency)).Scan(&id); err != nil { + return nil, fmt.Errorf("failed to insert address %q: %w", addr, err) + } + ids = append(ids, id) + } + return ids, nil } func scanWalletAddress(s scanner) (wallet.Address, error) { diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go new file mode 100644 index 0000000..3a78dd1 --- /dev/null +++ b/persist/sqlite/wallet_test.go @@ -0,0 +1,84 @@ +package sqlite + +import ( + "fmt" + "path/filepath" + "reflect" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +func TestAddAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + + // generate a large number of random addresses + addresses := make([]wallet.Address, 1000) + for i := range addresses { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addresses[i].Address = sp.Address() + addresses[i].SpendPolicy = &sp + addresses[i].Description = fmt.Sprintf("address %d", i) + } + + // create a new database + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + w, err := db.AddWallet(wallet.Wallet{}) + if err != nil { + t.Fatal(err) + } + + if err := db.AddWalletAddresses(w.ID, addresses...); err != nil { + t.Fatal(err) + } + + walletAddresses, err := db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(walletAddresses) != len(addresses) { + t.Fatalf("expected %d addresses, got %d", len(addresses), len(walletAddresses)) + } + for i, addr := range walletAddresses { + if !reflect.DeepEqual(addr, addresses[i]) { + t.Fatalf("expected address %d to be %v, got %v", i, addresses[i], addr) + } + } +} + +func BenchmarkAddWalletAddresses(b *testing.B) { + db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3"), zap.NewNop()) + if err != nil { + b.Fatal(err) + } + defer db.Close() + + addresses := make([]wallet.Address, b.N) + for i := range addresses { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addresses[i].Address = sp.Address() + addresses[i].SpendPolicy = &sp + addresses[i].Description = fmt.Sprintf("address %d", i) + } + + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + b.ReportAllocs() + + if err := db.AddWalletAddresses(w.ID, addresses...); err != nil { + b.Fatal(err) + } +} diff --git a/wallet/manager.go b/wallet/manager.go index d346505..ca7261b 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -91,7 +91,7 @@ type ( WalletAddresses(walletID ID) ([]Address, error) Wallets() ([]Wallet, error) - AddWalletAddress(walletID ID, addresses ...Address) error + AddWalletAddresses(walletID ID, addresses ...Address) error RemoveWalletAddress(walletID ID, address types.Address) error AddressBalance(address types.Address) (balance Balance, err error) @@ -263,9 +263,9 @@ func (m *Manager) Wallets() ([]Wallet, error) { return m.store.Wallets() } -// AddAddress adds the addresses to the given wallet. -func (m *Manager) AddAddress(walletID ID, addrs ...Address) error { - return m.store.AddWalletAddress(walletID, addrs...) +// AddAddresses adds the addresses to the given wallet. +func (m *Manager) AddAddresses(walletID ID, addrs ...Address) error { + return m.store.AddWalletAddresses(walletID, addrs...) } // RemoveAddress removes the given address from the given wallet. diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 107fba6..0aefac2 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -146,7 +146,7 @@ func TestReserve(t *testing.T) { sp := types.SpendPolicy{Type: types.PolicyTypePublicKey(sk.PublicKey())} addr := sp.Address() - err = wm.AddAddress(w.ID, wallet.Address{ + err = wm.AddAddresses(w.ID, wallet.Address{ Address: addr, SpendPolicy: &sp, }) @@ -226,7 +226,7 @@ func TestSelectSiacoins(t *testing.T) { } addr := uc.UnlockHash() - err = wm.AddAddress(w.ID, wallet.Address{ + err = wm.AddAddresses(w.ID, wallet.Address{ Address: addr, SpendPolicy: &types.SpendPolicy{ Type: types.PolicyTypeUnlockConditions(uc), @@ -396,7 +396,7 @@ func TestSelectSiafunds(t *testing.T) { t.Fatal(err) } - err = wm.AddAddress(w.ID, wallet.Address{ + err = wm.AddAddresses(w.ID, wallet.Address{ Address: addr, SpendPolicy: &types.SpendPolicy{ Type: types.PolicyTypeUnlockConditions(uc), @@ -534,7 +534,7 @@ func TestReorg(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -739,7 +739,7 @@ func TestEphemeralBalance(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -972,7 +972,7 @@ func TestWalletAddresses(t *testing.T) { SpendPolicy: &spendPolicy, Description: "hello, world", } - err = wm.AddAddress(w.ID, addr) + err = wm.AddAddresses(w.ID, addr) if err != nil { t.Fatal(err) } @@ -995,7 +995,7 @@ func TestWalletAddresses(t *testing.T) { addr.Description = "goodbye, world" addr.Metadata = json.RawMessage(`{"foo": "bar"}`) - if err := wm.AddAddress(w.ID, addr); err != nil { + if err := wm.AddAddresses(w.ID, addr); err != nil { t.Fatal(err) } @@ -1076,7 +1076,7 @@ func TestScan(t *testing.T) { } // add the address to the wallet - if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } // rescan to get the genesis Siafund state @@ -1158,7 +1158,7 @@ func TestScan(t *testing.T) { } // add the second address to the wallet - if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { t.Fatal(err) @@ -1243,7 +1243,7 @@ func TestSiafunds(t *testing.T) { } // add the address to the wallet - if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + if err := wm.AddAddresses(w1.ID, wallet.Address{Address: addr1}); err != nil { t.Fatal(err) } @@ -1327,7 +1327,7 @@ func TestSiafunds(t *testing.T) { w2, err := wm.AddWallet(wallet.Wallet{Name: "test2"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr2}); err != nil { + } else if err := wm.AddAddresses(w2.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) } @@ -1346,7 +1346,7 @@ func TestSiafunds(t *testing.T) { } // add the first address to the second wallet - if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr1}); err != nil { + if err := wm.AddAddresses(w2.ID, wallet.Address{Address: addr1}); err != nil { t.Fatal(err) } // rescan shouldn't be necessary since the address was already scanned @@ -1392,7 +1392,7 @@ func TestOrphans(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -2071,7 +2071,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // add the address to the wallet - if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + if err := wm.AddAddresses(w1.ID, wallet.Address{Address: addr1}); err != nil { t.Fatal(err) } @@ -2141,7 +2141,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // add the second address to the wallet - if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + if err := wm.AddAddresses(w1.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) } @@ -2259,7 +2259,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // add the address to the wallet - if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + if err := wm.AddAddresses(w1.ID, wallet.Address{Address: addr1}); err != nil { t.Fatal(err) } @@ -2330,7 +2330,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // add the second address to the wallet - if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + if err := wm.AddAddresses(w1.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) } @@ -2459,7 +2459,7 @@ func TestV2(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -2578,7 +2578,7 @@ func TestScanV2(t *testing.T) { } // add the address to the wallet - if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } // rescan to get the genesis Siafund state @@ -2656,7 +2656,7 @@ func TestScanV2(t *testing.T) { } // add the second address to the wallet - if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr2}); err != nil { t.Fatal(err) } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { t.Fatal(err) @@ -2762,7 +2762,7 @@ func TestReorgV2(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -2989,7 +2989,7 @@ func TestOrphansV2(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -3204,7 +3204,7 @@ func TestDeleteWallet(t *testing.T) { w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) - } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + } else if err := wm.AddAddresses(w.ID, wallet.Address{Address: addr}); err != nil { t.Fatal(err) } @@ -3894,7 +3894,7 @@ func TestSiafundClaims(t *testing.T) { } uc := types.StandardUnlockConditions(pk.PublicKey()) - err = wm.AddAddress(w.ID, wallet.Address{ + err = wm.AddAddresses(w.ID, wallet.Address{ Address: addr, SpendPolicy: &types.SpendPolicy{ Type: types.PolicyTypeUnlockConditions(uc), @@ -4145,7 +4145,7 @@ func TestV2SiafundClaims(t *testing.T) { sp := types.SpendPolicy{ Type: types.PolicyTypePublicKey(pk.PublicKey()), } - err = wm.AddAddress(w.ID, wallet.Address{ + err = wm.AddAddresses(w.ID, wallet.Address{ Address: addr, SpendPolicy: &sp, }) From 2eb45edbd64d7384fdf23882a9d4738eb77cf2ee Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:08:00 -0700 Subject: [PATCH 486/630] sp --- api/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api_test.go b/api/api_test.go index 1766bc8..007e385 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -506,7 +506,7 @@ func TestAddresses(t *testing.T) { addr1 := types.StandardUnlockHash(pk1.PublicKey()) addr2 := types.StandardUnlockHash(pk2.PublicKey()) - // assert multpile addresses can be added to a wallet + // assert multiple addresses can be added to a wallet if err := wc.AddAddresses([]wallet.Address{{Address: addr1}, {Address: addr2}}); err != nil { t.Fatal(err) } else if addrs, err := wc.Addresses(); err != nil { From aa78a9ad1fe558555471ffbafb5ecde7a3de3029 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:09:00 -0700 Subject: [PATCH 487/630] use const for batch size --- api/server.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/server.go b/api/server.go index b896913..f0f55be 100644 --- a/api/server.go +++ b/api/server.go @@ -610,12 +610,14 @@ func (s *server) walletsAddressHandlerPUT(jc jape.Context) { } func (s *server) walletsBatchAddressesHandlerPUT(jc jape.Context) { + const maxBatchAddressSize = 10000 + var id wallet.ID var addrs []wallet.Address if jc.DecodeParam("id", &id) != nil || jc.Decode(&addrs) != nil { return - } else if len(addrs) > 10000 { - jc.Error(fmt.Errorf("number of addresses exceeds the maximum batch size of 10000"), http.StatusBadRequest) + } else if len(addrs) > maxBatchAddressSize { + jc.Error(fmt.Errorf("number of addresses exceeds the maximum batch size %d", maxBatchAddressSize), http.StatusBadRequest) return } else if jc.Check("couldn't add addresses", s.wm.AddAddresses(id, addrs...)) != nil { return From ce945050de099d3b93cf17ae38c7a19f206365dd Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:10:00 -0700 Subject: [PATCH 488/630] update changeset --- ...o_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md index d516e87..54ae5f5 100644 --- a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md +++ b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md @@ -2,4 +2,6 @@ default: minor --- -# Add batch endpoint to allow adding multiple addresses to a wallet at a time. +# Added `[POST] /api/wallet/:id/batch/addresses + +This new endpoint allows clients to add up to 10000 addresses in a single API call From 93b85910553157c15a34f77381f2d17cc88c8f47 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:12:23 -0700 Subject: [PATCH 489/630] fix sqlite tests --- persist/sqlite/wallet_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index 3a78dd1..b3a0ca8 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap" "go.uber.org/zap/zaptest" ) @@ -26,7 +25,7 @@ func TestAddAddresses(t *testing.T) { } // create a new database - db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), WithLog(log.Named("sqlite3"))) if err != nil { t.Fatal(err) } @@ -55,7 +54,7 @@ func TestAddAddresses(t *testing.T) { } func BenchmarkAddWalletAddresses(b *testing.B) { - db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3"), zap.NewNop()) + db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3")) if err != nil { b.Fatal(err) } From 053371f080992b8fed576fedf74504c30d796994 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:15:24 -0700 Subject: [PATCH 490/630] variable name clarity --- persist/sqlite/wallet.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 9075697..0bcc48a 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -145,17 +145,17 @@ func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { } // AddWalletAddresses adds the given addresses to a wallet. -func (s *Store) AddWalletAddresses(id wallet.ID, addr ...wallet.Address) error { +func (s *Store) AddWalletAddresses(id wallet.ID, walletAddresses ...wallet.Address) error { return s.transaction(func(tx *txn) error { if err := walletExists(tx, id); err != nil { return err - } else if len(addr) == 0 { + } else if len(walletAddresses) == 0 { return errors.New("no addresses to add") } - addresses := make([]types.Address, 0, len(addr)) - for _, a := range addr { - addresses = append(addresses, a.Address) + addresses := make([]types.Address, 0, len(walletAddresses)) + for _, wa := range walletAddresses { + addresses = append(addresses, wa.Address) } addressDBIDs, err := insertAddress(tx, addresses...) @@ -169,17 +169,17 @@ func (s *Store) AddWalletAddresses(id wallet.ID, addr ...wallet.Address) error { } defer stmt.Close() - for i, addr := range addr { + for i, wa := range walletAddresses { addressDBID := addressDBIDs[i] var encodedPolicy any - if addr.SpendPolicy != nil { - encodedPolicy = encode(*addr.SpendPolicy) + if wa.SpendPolicy != nil { + encodedPolicy = encode(*wa.SpendPolicy) } - _, err = stmt.Exec(id, addressDBID, addr.Description, encodedPolicy, addr.Metadata) + _, err = stmt.Exec(id, addressDBID, wa.Description, encodedPolicy, wa.Metadata) if err != nil { - return fmt.Errorf("failed to insert wallet address %q: %w", addr.Address, err) + return fmt.Errorf("failed to insert wallet address %q: %w", wa.Address, err) } } return nil From 07af9b196378f2cbecb3609682c17b29e3d02d29 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:18:30 -0700 Subject: [PATCH 491/630] update endpoint --- ..._to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md index 54ae5f5..e3b50aa 100644 --- a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md +++ b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md @@ -2,6 +2,6 @@ default: minor --- -# Added `[POST] /api/wallet/:id/batch/addresses +# Added `[POST] /api/wallets/:id/batch/addresses This new endpoint allows clients to add up to 10000 addresses in a single API call From f5b1a9afb989f0d06b2540b9f7f4a9abb1131689 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 13:35:20 -0700 Subject: [PATCH 492/630] extend addresses test --- persist/sqlite/wallet_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index b3a0ca8..51228f6 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -9,6 +9,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap/zaptest" + "lukechampine.com/frand" ) func TestAddAddresses(t *testing.T) { @@ -51,6 +52,39 @@ func TestAddAddresses(t *testing.T) { t.Fatalf("expected address %d to be %v, got %v", i, addresses[i], addr) } } + + // change random addresses' descriptions + for range 10 { + i := frand.Intn(len(addresses)) + addresses[i].Description = fmt.Sprintf("updated address %d", i) + } + + // add additional addresses + for range 10 { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addresses = append(addresses, wallet.Address{ + Address: sp.Address(), + SpendPolicy: &sp, + Description: fmt.Sprintf("address %d", len(addresses)), + }) + } + + // re-add the initial addresses and the new ones to ensure updates work + if err := db.AddWalletAddresses(w.ID, addresses...); err != nil { + t.Fatal(err) + } + walletAddresses, err = db.WalletAddresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(walletAddresses) != len(addresses) { + t.Fatalf("expected %d addresses, got %d", len(addresses), len(walletAddresses)) + } + for i, addr := range walletAddresses { + if !reflect.DeepEqual(addr, addresses[i]) { + t.Fatalf("expected address %d to be %v, got %v", i, addresses[i], addr) + } + } } func BenchmarkAddWalletAddresses(b *testing.B) { From 20db8c11681177635d112cce4420bf8cff7be6f5 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 29 May 2025 15:35:13 -0700 Subject: [PATCH 493/630] ci: remove release workflow --- .github/workflows/release.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 00c04ae..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Release - -on: - pull_request: - types: [closed] - branches: [master] - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -jobs: - release: - if: (github.head_ref == 'release' && github.event.pull_request.merged == true) || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4.2.2 - with: - fetch-depth: 0 - - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 - - run: knope release --verbose - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} - From 50568eb4bd3bad2b8c266bc4e47aa8921ac0f301 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 May 2025 22:35:27 +0000 Subject: [PATCH 494/630] chore: prepare release 2.9.0 --- ...ding_multiple_addresses_to_a_wallet_at_a_time.md | 7 ------- .../updated_core_to_v0131_and_coreutils_to_v0152.md | 5 ----- .changeset/updated_dockerfile_to_go_124.md | 5 ----- CHANGELOG.md | 13 +++++++++++++ go.mod | 2 +- 5 files changed, 14 insertions(+), 18 deletions(-) delete mode 100644 .changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md delete mode 100644 .changeset/updated_core_to_v0131_and_coreutils_to_v0152.md delete mode 100644 .changeset/updated_dockerfile_to_go_124.md diff --git a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md b/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md deleted file mode 100644 index e3b50aa..0000000 --- a/.changeset/add_batch_endpoint_to_allow_adding_multiple_addresses_to_a_wallet_at_a_time.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Added `[POST] /api/wallets/:id/batch/addresses - -This new endpoint allows clients to add up to 10000 addresses in a single API call diff --git a/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md b/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md deleted file mode 100644 index c17e819..0000000 --- a/.changeset/updated_core_to_v0131_and_coreutils_to_v0152.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Updated core to v0.13.1 and coreutils to v0.15.2. diff --git a/.changeset/updated_dockerfile_to_go_124.md b/.changeset/updated_dockerfile_to_go_124.md deleted file mode 100644 index ff11335..0000000 --- a/.changeset/updated_dockerfile_to_go_124.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Updated build to use Go 1.24 diff --git a/CHANGELOG.md b/CHANGELOG.md index c49e9e0..5b0bcb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 2.9.0 (2025-05-29) + +### Features + +#### Added `[POST] /api/wallets/:id/batch/addresses + +This new endpoint allows clients to add up to 10000 addresses in a single API call + +### Fixes + +- Updated core to v0.13.1 and coreutils to v0.15.2. +- Updated build to use Go 1.24 + ## 2.8.1 (2025-05-29) ### Fixes diff --git a/go.mod b/go.mod index 67eed18..5dddd55 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.8.1 +module go.sia.tech/walletd/v2 // v2.9.0 go 1.24.2 From e18f1f762f85a9d26d9ec5ff957d39997bdd8943 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:08:47 +0000 Subject: [PATCH 495/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.29.2 to 0.29.3 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.29.2...walletd@0.29.3) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-version: 0.29.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5dddd55..b7a0d74 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.13.1 go.sia.tech/coreutils v0.15.2 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.29.2 + go.sia.tech/web/walletd v0.29.3 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.32.0 diff --git a/go.sum b/go.sum index 03992ce..5773870 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.29.2 h1:bi0A4ZDROEAmh3CtJ+fJ/3dXfobc37kjqmnmJTNPmM4= -go.sia.tech/web/walletd v0.29.2/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.29.3 h1:65Jl/2qAH+BECam3rJ6bp/GIONMct6XjClO1f7IjfYo= +go.sia.tech/web/walletd v0.29.3/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From 878c187ea1c7730f5e87d2681f68319cdc9596e2 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 5 Jun 2025 21:28:59 -0700 Subject: [PATCH 496/630] add batch endpoints --- .changeset/add_batch_endpoints.md | 11 ++ api/api.go | 6 + api/server.go | 98 +++++++++- persist/sqlite/addresses.go | 294 +++++++++++++++++++++++++++--- wallet/addresses.go | 28 ++- wallet/addresses_test.go | 170 +++++++++++++++++ wallet/manager.go | 7 +- 7 files changed, 583 insertions(+), 31 deletions(-) create mode 100644 .changeset/add_batch_endpoints.md diff --git a/.changeset/add_batch_endpoints.md b/.changeset/add_batch_endpoints.md new file mode 100644 index 0000000..597c0b4 --- /dev/null +++ b/.changeset/add_batch_endpoints.md @@ -0,0 +1,11 @@ +--- +default: minor +--- + +# Add batch endpoints + +- `[POST] /batch/addresses/balance` +- `[POST] /batch/addresses/events` +- `[POST] /batch/addresses/unconfirmed` +- `[POST] /batch/addresses/outputs/siacoin` +- `[POST] /batch/addresses/outputs/siafund` \ No newline at end of file diff --git a/api/api.go b/api/api.go index ba17421..91b3102 100644 --- a/api/api.go +++ b/api/api.go @@ -248,3 +248,9 @@ type ElementSpentResponse struct { Spent bool `json:"spent"` Event *wallet.Event `json:"event,omitempty"` } + +// BatchAddressesRequest is the request type for batch +// address operations. +type BatchAddressesRequest struct { + Addresses []types.Address `json:"addresses"` +} diff --git a/api/server.go b/api/server.go index f0f55be..d4a7208 100644 --- a/api/server.go +++ b/api/server.go @@ -114,12 +114,16 @@ type ( UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) WalletBalance(id wallet.ID) (wallet.Balance, error) - AddressBalance(address types.Address) (wallet.Balance, error) + AddressBalance(address ...types.Address) (wallet.Balance, error) AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) AddressUnconfirmedEvents(address types.Address) ([]wallet.Event, error) AddressSiacoinOutputs(address types.Address, tpool bool, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) AddressSiafundOutputs(address types.Address, tpool bool, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) + BatchAddressEvents(addresses []types.Address, offset, limit int) ([]wallet.Event, error) + BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) + BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) + Events(eventIDs []types.Hash256) ([]wallet.Event, error) SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) @@ -1420,7 +1424,7 @@ func (s *server) checkAddressesHandlerPOST(jc jape.Context) { var req CheckAddressesRequest if jc.Decode(&req) != nil { return - } else if len(req.Addresses) > 10000 { + } else if len(req.Addresses) > 1000 { jc.Error(errors.New("too many addresses"), http.StatusBadRequest) return } @@ -1435,6 +1439,91 @@ func (s *server) checkAddressesHandlerPOST(jc jape.Context) { }) } +func (s *server) batchAddressesBalanceHandlerPOST(jc jape.Context) { + var req BatchAddressesRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Addresses) > 1000 { + jc.Error(errors.New("too many addresses"), http.StatusBadRequest) + return + } + + balance, err := s.wm.AddressBalance(req.Addresses...) + if jc.Check("couldn't get balances", err) != nil { + return + } + jc.Encode(BalanceResponse(balance)) +} + +func (s *server) batchAddressesEventsHandlerPOST(jc jape.Context) { + var req BatchAddressesRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Addresses) > 1000 { + jc.Error(errors.New("too many addresses"), http.StatusBadRequest) + return + } + + offset, limit := 0, 100 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + events, err := s.wm.BatchAddressEvents(req.Addresses, offset, limit) + if jc.Check("couldn't load events", err) != nil { + return + } + jc.Encode(events) +} + +func (s *server) batchAddressesOutputsSCHandlerPOST(jc jape.Context) { + var req BatchAddressesRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Addresses) > 1000 { + jc.Error(errors.New("too many addresses"), http.StatusBadRequest) + return + } + + offset, limit := 0, 100 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, basis, err := s.wm.BatchAddressSiacoinOutputs(req.Addresses, offset, limit) + if jc.Check("couldn't load siacoin outputs", err) != nil { + return + } + jc.Encode(AddressSiacoinElementsResponse{ + Basis: basis, + Outputs: utxos, + }) +} + +func (s *server) batchAddressesOutputsSFHandlerPOST(jc jape.Context) { + var req BatchAddressesRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Addresses) > 1000 { + jc.Error(errors.New("too many addresses"), http.StatusBadRequest) + return + } + + offset, limit := 0, 100 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, basis, err := s.wm.BatchAddressSiafundOutputs(req.Addresses, offset, limit) + if jc.Check("couldn't load siafund outputs", err) != nil { + return + } + jc.Encode(AddressSiafundElementsResponse{ + Basis: basis, + Outputs: utxos, + }) +} + func (s *server) debugMineHandler(jc jape.Context) { var req DebugMineRequest if jc.Decode(&req) != nil { @@ -1573,6 +1662,11 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /addresses/:addr/outputs/siacoin": wrapPublicAuthHandler(srv.addressesAddrOutputsSCHandler), "GET /addresses/:addr/outputs/siafund": wrapPublicAuthHandler(srv.addressesAddrOutputsSFHandler), + "POST /batch/addresses/balance": wrapPublicAuthHandler(srv.batchAddressesBalanceHandlerPOST), + "POST /batch/addresses/events": wrapPublicAuthHandler(srv.batchAddressesEventsHandlerPOST), + "POST /batch/addresses/outputs/siacoin": wrapPublicAuthHandler(srv.batchAddressesOutputsSCHandlerPOST), + "POST /batch/addresses/outputs/siafund": wrapPublicAuthHandler(srv.batchAddressesOutputsSFHandlerPOST), + "GET /outputs/siacoin/:id": wrapPublicAuthHandler(srv.outputsSiacoinHandlerGET), "GET /outputs/siacoin/:id/spent": wrapPublicAuthHandler(srv.outputsSiacoinSpentHandlerGET), "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 30de08e..c77f40b 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -39,42 +39,225 @@ func (s *Store) CheckAddresses(addresses []types.Address) (known bool, err error return } -// AddressBalance returns the balance of a single address. -func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { +// AddressBalance returns the aggregate balance of the addresses. +func (s *Store) AddressBalance(address ...types.Address) (balance wallet.Balance, err error) { + if len(address) == 0 { + return wallet.Balance{}, nil // no addresses, no balance + } err = s.transaction(func(tx *txn) error { const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - err := tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) - if errors.Is(err, sql.ErrNoRows) { - balance = wallet.Balance{} - return nil + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, addr := range address { + var siacoins, immatureSiacoins types.Currency + var siafunds uint64 + + if err := stmt.QueryRow(encode(addr)).Scan(decode(&siacoins), decode(&immatureSiacoins), &siafunds); err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to query address %q: %w", addr, err) + } + balance.Siacoins = balance.Siacoins.Add(siacoins) + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(immatureSiacoins) + balance.Siafunds += siafunds } - return err + return nil }) return } -func getAddressEvents(tx *txn, address types.Address, offset, limit int) (eventIDs []int64, err error) { - const query = `SELECT DISTINCT ea.event_id -FROM event_addresses ea -INNER JOIN sia_addresses sa ON ea.address_id = sa.id -WHERE sa.sia_address = $1 -ORDER BY ea.event_maturity_height DESC, ea.event_id DESC -LIMIT $2 OFFSET $3;` - - rows, err := tx.Query(query, encode(address), limit, offset) - if err != nil { - return nil, err +// BatchAddressEvents returns the events for a batch of addresses. +func (s *Store) BatchAddressEvents(addresses []types.Address, offset, limit int) (events []wallet.Event, err error) { + if len(addresses) == 0 { + return nil, nil // no addresses, no events } - defer rows.Close() + err = s.transaction(func(tx *txn) error { + dbIDs, err := s.getAddressesEvents(tx, addresses, offset, limit) + if err != nil { + return fmt.Errorf("failed to get events for addresses: %w", err) + } + if len(dbIDs) == 0 { + return nil // no events found + } - for rows.Next() { - var id int64 - if err := rows.Scan(&id); err != nil { - return nil, err + events, err = getEventsByID(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) } - eventIDs = append(eventIDs, id) - } - return eventIDs, rows.Err() + + addressMap := make(map[types.Address]bool) + for _, addr := range addresses { + addressMap[addr] = true + } + for i := range events { + seen := make(map[types.Address]bool) + switch ev := events[i].Data.(type) { + case wallet.EventV1Transaction: + for _, sci := range ev.Transaction.SiacoinInputs { + addr := sci.UnlockConditions.UnlockHash() + if addressMap[addr] && !seen[addr] { + seen[addr] = true + events[i].Relevant = append(events[i].Relevant, addr) + } + } + for _, sco := range ev.Transaction.SiacoinOutputs { + if addressMap[sco.Address] && !seen[sco.Address] { + seen[sco.Address] = true + events[i].Relevant = append(events[i].Relevant, sco.Address) + } + } + for _, sfi := range ev.Transaction.SiafundInputs { + addr := sfi.UnlockConditions.UnlockHash() + if addressMap[addr] && !seen[addr] { + seen[addr] = true + events[i].Relevant = append(events[i].Relevant, addr) + } + } + for _, sfo := range ev.Transaction.SiafundOutputs { + if addressMap[sfo.Address] && !seen[sfo.Address] { + seen[sfo.Address] = true + events[i].Relevant = append(events[i].Relevant, sfo.Address) + } + } + case wallet.EventV2Transaction: + for _, sci := range ev.SiacoinInputs { + if addressMap[sci.Parent.SiacoinOutput.Address] && !seen[sci.Parent.SiacoinOutput.Address] { + seen[sci.Parent.SiacoinOutput.Address] = true + events[i].Relevant = append(events[i].Relevant, sci.Parent.SiacoinOutput.Address) + } + } + for _, sco := range ev.SiacoinOutputs { + if addressMap[sco.Address] && !seen[sco.Address] { + seen[sco.Address] = true + events[i].Relevant = append(events[i].Relevant, sco.Address) + } + } + for _, sfi := range ev.SiafundInputs { + if addressMap[sfi.Parent.SiafundOutput.Address] && !seen[sfi.Parent.SiafundOutput.Address] { + seen[sfi.Parent.SiafundOutput.Address] = true + events[i].Relevant = append(events[i].Relevant, sfi.Parent.SiafundOutput.Address) + } + } + for _, sfo := range ev.SiafundOutputs { + if addressMap[sfo.Address] && !seen[sfo.Address] { + seen[sfo.Address] = true + events[i].Relevant = append(events[i].Relevant, sfo.Address) + } + } + case wallet.EventPayout: + events[i].Relevant = append(events[i].Relevant, ev.SiacoinElement.SiacoinOutput.Address) + } + } + return nil + }) + return +} + +// BatchAddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (s *Store) BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height + FROM siacoin_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address IN (` + queryPlaceHolders(len(addresses)) + `) AND se.maturity_height <= ? AND se.spent_index_id IS NULL + ORDER BY se.maturity_height DESC, se.id DESC + LIMIT ? OFFSET ?` + + rows, err := tx.Query(query, append(encodeSlice(addresses), basis.Height, limit, offset)...) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + + siacoins = append(siacoins, siacoin) + } + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siacoins)) + for i, se := range siacoins { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siacoins[i].StateElement.MerkleProof = proof + } + } + return nil + }) + return +} + +// BatchAddressSiafundOutputs returns the unspent siafund outputs for an address. +func (s *Store) BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + query := `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height + FROM siafund_elements se + INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address IN(` + queryPlaceHolders(len(addresses)) + `) AND se.spent_index_id IS NULL + ORDER BY se.id DESC + LIMIT ? OFFSET ?` + + rows, err := tx.Query(query, append(encodeSlice(addresses), limit, offset)...) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siafund, err := scanUnspentSiafundElement(rows, basis.Height) + if err != nil { + return fmt.Errorf("failed to scan siafund element: %w", err) + } + siafunds = append(siafunds, siafund) + } + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siafund elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siafunds)) + for i, se := range siafunds { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siafunds[i].StateElement.MerkleProof = proof + } + } + return nil + }) + return } // AddressEvents returns the events of a single address. @@ -347,3 +530,62 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 }) return } + +func getAddressEvents(tx *txn, address types.Address, offset, limit int) (eventIDs []int64, err error) { + const query = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +WHERE sa.sia_address = $1 +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT $2 OFFSET $3;` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + eventIDs = append(eventIDs, id) + } + return eventIDs, rows.Err() +} + +func (s *Store) getAddressesEvents(tx *txn, addresses []types.Address, offset, limit int) (eventIDs []int64, err error) { + if len(addresses) == 0 { + return nil, nil // no addresses, no events + } + + query := `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +WHERE sa.sia_address IN (` + queryPlaceHolders(len(addresses)) + `) +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT ? OFFSET ?;` + + params := make([]any, 0, len(addresses)+2) + for _, addr := range addresses { + params = append(params, encode(addr)) + } + params = append(params, limit, offset) + rows, err := tx.Query(query, params...) + if err != nil { + return nil, fmt.Errorf("failed to query address events: %w", err) + } + defer rows.Close() + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("failed to scan event ID: %w", err) + } + eventIDs = append(eventIDs, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over rows: %w", err) + } + return eventIDs, nil +} diff --git a/wallet/addresses.go b/wallet/addresses.go index a96cc4d..aabdb39 100644 --- a/wallet/addresses.go +++ b/wallet/addresses.go @@ -13,8 +13,8 @@ func (m *Manager) CheckAddresses(address []types.Address) (bool, error) { } // AddressBalance returns the balance of a single address. -func (m *Manager) AddressBalance(address types.Address) (balance Balance, err error) { - return m.store.AddressBalance(address) +func (m *Manager) AddressBalance(addresses ...types.Address) (balance Balance, err error) { + return m.store.AddressBalance(addresses...) } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. @@ -83,6 +83,30 @@ func (m *Manager) AddressEvents(address types.Address, offset, limit int) (event return m.store.AddressEvents(address, offset, limit) } +// BatchAddressEvents returns the events for a batch of addresses. +func (m *Manager) BatchAddressEvents(addresses []types.Address, offset, limit int) ([]Event, error) { + if len(addresses) == 0 { + return nil, nil // no addresses, no events + } + return m.store.BatchAddressEvents(addresses, offset, limit) +} + +// BatchAddressSiacoinOutputs returns the unspent siacoin outputs for a batch of addresses. +func (m *Manager) BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) { + if len(addresses) == 0 { + return nil, types.ChainIndex{}, nil // no addresses, no outputs + } + return m.store.BatchAddressSiacoinOutputs(addresses, offset, limit) +} + +// BatchAddressSiafundOutputs returns the unspent siafund outputs for a batch of addresses. +func (m *Manager) BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) { + if len(addresses) == 0 { + return nil, types.ChainIndex{}, nil // no addresses, no outputs + } + return m.store.BatchAddressSiafundOutputs(addresses, offset, limit) +} + // AddressUnconfirmedEvents returns the unconfirmed events for a single address. func (m *Manager) AddressUnconfirmedEvents(address types.Address) ([]Event, error) { index := m.chain.Tip() diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 6eec13a..63c0347 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -7,6 +7,7 @@ import ( "go.sia.tech/walletd/v2/internal/testutil" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap/zaptest" + "lukechampine.com/frand" ) func TestAddressUseTpool(t *testing.T) { @@ -97,3 +98,172 @@ func TestAddressUseTpool(t *testing.T) { cn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) } + +func TestBatchAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + + network, genesisBlock := testutil.V2Network() + cn := testutil.NewConsensusNode(t, network, genesisBlock, log) + cm := cn.Chain + db := cn.Store + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // mine a bunch of payouts to different addresses + addresses := make([]types.Address, 100) + for i := range addresses { + addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) + cn.MineBlocks(t, addresses[i], 1) + } + + events, err := wm.BatchAddressEvents(addresses, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(events) != 100 { + t.Fatalf("expected 100 events, got %d", len(events)) + } +} + +func TestBatchSiacoinOutputs(t *testing.T) { + log := zaptest.NewLogger(t) + + network, genesisBlock := testutil.V2Network() + cn := testutil.NewConsensusNode(t, network, genesisBlock, log) + cm := cn.Chain + db := cn.Store + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // mine a bunch of payouts to different addresses + addresses := make([]types.Address, 100) + for i := range addresses { + addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) + cn.MineBlocks(t, addresses[i], 1) + } + cn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) + + sces, _, err := wm.BatchAddressSiacoinOutputs(addresses, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(sces) != 100 { + t.Fatalf("expected 100 events, got %d", len(sces)) + } +} + +func TestBatchSiafundOutputs(t *testing.T) { + log := zaptest.NewLogger(t) + + giftAddr := types.AnyoneCanSpend().Address() + network, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiafundOutputs = []types.SiafundOutput{ + {Address: giftAddr, Value: 10000}, + } + cn := testutil.NewConsensusNode(t, network, genesisBlock, log) + cm := cn.Chain + db := cn.Store + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // distribute the siafund output to multiple addresses + var addresses []types.Address + outputID := genesisBlock.Transactions[0].SiafundOutputID(0) + outputValue := genesisBlock.Transactions[0].SiafundOutputs[0].Value + for range 100 { + txn := types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: types.SiafundElement{ + ID: outputID, + }, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.AnyoneCanSpend(), + }, + }, + }, + } + + for range 10 { + address := types.StandardAddress(types.GeneratePrivateKey().PublicKey()) + addresses = append(addresses, address) + txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ + Address: address, + Value: 1, + }) + outputValue-- + if outputValue == 0 { + break + } + } + + if outputValue > 0 { + txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ + Address: giftAddr, + Value: outputValue, + }) + } + outputID = txn.SiafundOutputID(txn.ID(), len(txn.SiafundOutputs)-1) + basis, txns, err := db.OverwriteElementProofs([]types.V2Transaction{txn}) + if err != nil { + t.Fatal(err) + } + if _, err := cm.AddV2PoolTransactions(basis, txns); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + cn.WaitForSync(t) + } + + sfes, _, err := wm.BatchAddressSiafundOutputs(addresses, 0, 10000) + if err != nil { + t.Fatal(err) + } else if len(sfes) != 1000 { + t.Fatalf("expected 1000 events, got %d", len(sfes)) + } +} + +func BenchmarkBatchAddresses(b *testing.B) { + log := zaptest.NewLogger(b) + + network, genesisBlock := testutil.V2Network() + cn := testutil.NewConsensusNode(b, network, genesisBlock, log) + cm := cn.Chain + db := cn.Store + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + b.Fatal(err) + } + defer wm.Close() + + // mine a bunch of payouts to different addresses + addresses := make([]types.Address, 10000) + for i := range addresses { + addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) + cn.MineBlocks(b, addresses[i], 1) + } + + b.ResetTimer() + b.ReportAllocs() + + for b.Loop() { + slice := addresses[frand.Intn(len(addresses)-1000):][:1000] + events, err := wm.BatchAddressEvents(slice, 0, 100) + if err != nil { + b.Fatal(err) + } else if len(events) != 100 { + b.Fatalf("expected 100 events, got %d", len(events)) + } + } +} diff --git a/wallet/manager.go b/wallet/manager.go index ca7261b..62accc3 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -94,10 +94,15 @@ type ( AddWalletAddresses(walletID ID, addresses ...Address) error RemoveWalletAddress(walletID ID, address types.Address) error - AddressBalance(address types.Address) (balance Balance, err error) + AddressBalance(address ...types.Address) (balance Balance, err error) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) + + BatchAddressEvents(addresses []types.Address, offset, limit int) ([]Event, error) + BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) ([]UnspentSiacoinElement, types.ChainIndex, error) + BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]UnspentSiafundElement, types.ChainIndex, error) + // CheckAddresses returns true if any of the addresses have been seen on the // blockchain. This is a quick way to scan wallets for lookaheads. // From 0aa28f00bf03571970d56000a1fd831fdb8e0dce Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 7 Jun 2025 12:06:24 -0700 Subject: [PATCH 497/630] api: add batch client methods --- api/api_test.go | 12 +++++++++++ api/client.go | 42 +++++++++++++++++++++++++++++++++++++ api/server.go | 10 +++++++++ persist/sqlite/addresses.go | 1 - wallet/addresses_test.go | 1 - 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 007e385..e9a3960 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1033,6 +1033,18 @@ func TestConstructV2Siacoins(t *testing.T) { case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } + + unconfirmed, err = c.TPoolEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } else if unconfirmed[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, unconfirmed[0].Type) + } else if unconfirmed[0].ID != sent.ID { + t.Fatalf("expected unconfirmed event to have ID %q, got %q", sent.ID, unconfirmed[0].ID) + } + cm.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) diff --git a/api/client.go b/api/client.go index 03f6bf0..333e491 100644 --- a/api/client.go +++ b/api/client.go @@ -285,6 +285,42 @@ func (c *Client) AddressSiafundOutputs(addr types.Address, useTpool bool, offset return resp.Outputs, resp.Basis, err } +// BatchAddressBalance returns the balance of a batch of addresses. +func (c *Client) BatchAddressBalance(addresses []types.Address) (BalanceResponse, error) { + var resp BalanceResponse + err := c.c.POST(context.Background(), "/batch/addresses/balance", BatchAddressesRequest{ + Addresses: addresses, + }, &resp) + return resp, err +} + +// BatchAddressSiacoinOutputs returns the unspent siacoin outputs for a batch of addresses. +func (c *Client) BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + var resp AddressSiacoinElementsResponse + err := c.c.POST(context.Background(), fmt.Sprintf("/batch/addresses/outputs/siacoin?offset=%d&limit=%d", offset, limit), BatchAddressesRequest{ + Addresses: addresses, + }, &resp) + return resp.Outputs, resp.Basis, err +} + +// BatchAddressSiafundOutputs returns the unspent siafund outputs for a batch of addresses. +func (c *Client) BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + var resp AddressSiafundElementsResponse + err := c.c.POST(context.Background(), fmt.Sprintf("/batch/addresses/outputs/siafund?offset=%d&limit=%d", offset, limit), BatchAddressesRequest{ + Addresses: addresses, + }, &resp) + return resp.Outputs, resp.Basis, err +} + +// BatchAddressEvents returns the events for a batch of addresses. +func (c *Client) BatchAddressEvents(addresses []types.Address, offset, limit int) ([]wallet.Event, error) { + var resp []wallet.Event + err := c.c.POST(context.Background(), fmt.Sprintf("/batch/addresses/events?offset=%d&limit=%d", offset, limit), BatchAddressesRequest{ + Addresses: addresses, + }, &resp) + return resp, err +} + // CheckAddresses checks whether the specified addresses are known to the wallet. // In full index mode, this will return true if any of the addresses have been seen on chain. func (c *Client) CheckAddresses(addresses []types.Address) (bool, error) { @@ -301,6 +337,12 @@ func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { return } +// TPoolEvents returns all unconfirmed events in the transaction pool. +func (c *Client) TPoolEvents() (resp []wallet.Event, err error) { + err = c.c.GET(context.Background(), "/txpool/events", &resp) + return +} + // SpentSiacoinElement returns whether a siacoin output has been spent and the // event that spent it. func (c *Client) SpentSiacoinElement(id types.SiacoinOutputID) (resp ElementSpentResponse, err error) { diff --git a/api/server.go b/api/server.go index d4a7208..72875fe 100644 --- a/api/server.go +++ b/api/server.go @@ -125,6 +125,7 @@ type ( BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) Events(eventIDs []types.Hash256) ([]wallet.Event, error) + UnconfirmedEvents() ([]wallet.Event, error) SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) @@ -1524,6 +1525,14 @@ func (s *server) batchAddressesOutputsSFHandlerPOST(jc jape.Context) { }) } +func (s *server) txpoolEventsUnconfirmedHandlerGET(jc jape.Context) { + events, err := s.wm.UnconfirmedEvents() + if jc.Check("couldn't load unconfirmed events", err) != nil { + return + } + jc.Encode(events) +} + func (s *server) debugMineHandler(jc jape.Context) { var req DebugMineRequest if jc.Decode(&req) != nil { @@ -1655,6 +1664,7 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption "GET /txpool/fee": wrapPublicAuthHandler(srv.txpoolFeeHandler), "POST /txpool/parents": wrapPublicAuthHandler(srv.txpoolParentsHandler), "POST /txpool/broadcast": wrapPublicAuthHandler(srv.txpoolBroadcastHandler), + "GET /txpool/events": wrapPublicAuthHandler(srv.txpoolEventsUnconfirmedHandlerGET), "GET /addresses/:addr/balance": wrapPublicAuthHandler(srv.addressesAddrBalanceHandler), "GET /addresses/:addr/events": wrapPublicAuthHandler(srv.addressesAddrEventsHandlerGET), diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index c77f40b..42e453b 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -168,7 +168,6 @@ func (s *Store) BatchAddressSiacoinOutputs(addresses []types.Address, offset, li INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE sa.sia_address IN (` + queryPlaceHolders(len(addresses)) + `) AND se.maturity_height <= ? AND se.spent_index_id IS NULL - ORDER BY se.maturity_height DESC, se.id DESC LIMIT ? OFFSET ?` rows, err := tx.Query(query, append(encodeSlice(addresses), basis.Height, limit, offset)...) diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 63c0347..4327d75 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -222,7 +222,6 @@ func TestBatchSiafundOutputs(t *testing.T) { t.Fatal(err) } cn.MineBlocks(t, types.VoidAddress, 1) - cn.WaitForSync(t) } sfes, _, err := wm.BatchAddressSiafundOutputs(addresses, 0, 10000) From c3cc9d9b3efba616d20baa2962474d73f872f2ba Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 7 Jun 2025 12:10:28 -0700 Subject: [PATCH 498/630] update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b7a0d74..33da4fe 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.1 - go.sia.tech/coreutils v0.15.2 + go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 5773870..e69fe79 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.13.1 h1:dBKzZBhWZsgdV7qZa6qiaZtTDj5evvSqYWpeGNenlRI= go.sia.tech/core v0.13.1/go.mod h1:oMOgHT4bf9VSXUCOgtt9w4MFns/pY0LRUgwyMXdxW5w= -go.sia.tech/coreutils v0.15.2 h1:2oEe8wpsmU5WVNfe0x75URhno+lPSXc+ozRtZNgjzu4= -go.sia.tech/coreutils v0.15.2/go.mod h1:Kz/VQViqymnR1EW7DDdKrQru8dMFxiY44/qmjriilWs= +go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4 h1:MDnsg/wLgs0Lslg764Twti0NNVYAFZMk3gupF6x8U/4= +go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4/go.mod h1:L2fdE/d/DbfUmwTx8htI2hN9ypunBX2Q/DRTYCtb+ck= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From aa94731dc23de8d1a58653a9d22042219f08cdef Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 7 Jun 2025 19:10:41 +0000 Subject: [PATCH 499/630] chore: prepare release 2.10.0 --- .changeset/add_batch_endpoints.md | 11 ----------- CHANGELOG.md | 12 ++++++++++++ go.mod | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) delete mode 100644 .changeset/add_batch_endpoints.md diff --git a/.changeset/add_batch_endpoints.md b/.changeset/add_batch_endpoints.md deleted file mode 100644 index 597c0b4..0000000 --- a/.changeset/add_batch_endpoints.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -default: minor ---- - -# Add batch endpoints - -- `[POST] /batch/addresses/balance` -- `[POST] /batch/addresses/events` -- `[POST] /batch/addresses/unconfirmed` -- `[POST] /batch/addresses/outputs/siacoin` -- `[POST] /batch/addresses/outputs/siafund` \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0bcb6..e811511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## 2.10.0 (2025-06-07) + +### Features + +#### Add batch endpoints + +- `[POST] /batch/addresses/balance` +- `[POST] /batch/addresses/events` +- `[POST] /batch/addresses/unconfirmed` +- `[POST] /batch/addresses/outputs/siacoin` +- `[POST] /batch/addresses/outputs/siafund` + ## 2.9.0 (2025-05-29) ### Features diff --git a/go.mod b/go.mod index 33da4fe..8163ecc 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.9.0 +module go.sia.tech/walletd/v2 // v2.10.0 go 1.24.2 From fc9351d1a8f8bad7efa03809f95d0134cc17b4fd Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 13 Jun 2025 21:43:45 -0700 Subject: [PATCH 500/630] update core and coreutils --- ...te_core_to_v0132_and_coreutils_to_v0161.md | 5 +++ go.mod | 16 +++++----- go.sum | 32 +++++++++---------- 3 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 .changeset/update_core_to_v0132_and_coreutils_to_v0161.md diff --git a/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md b/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md new file mode 100644 index 0000000..db7fa02 --- /dev/null +++ b/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core to v0.13.2 and coreutils to v0.16.1 diff --git a/go.mod b/go.mod index 8163ecc..d63f63f 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.13.1 - go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4 + go.sia.tech/core v0.13.2 + go.sia.tech/coreutils v0.16.1 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.3 go.uber.org/zap v1.27.0 @@ -30,11 +30,11 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sync v0.14.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/mod v0.25.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/tools v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index e69fe79..039491e 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.sia.tech/core v0.13.1 h1:dBKzZBhWZsgdV7qZa6qiaZtTDj5evvSqYWpeGNenlRI= -go.sia.tech/core v0.13.1/go.mod h1:oMOgHT4bf9VSXUCOgtt9w4MFns/pY0LRUgwyMXdxW5w= -go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4 h1:MDnsg/wLgs0Lslg764Twti0NNVYAFZMk3gupF6x8U/4= -go.sia.tech/coreutils v0.16.1-0.20250607184904-8c78c1233ed4/go.mod h1:L2fdE/d/DbfUmwTx8htI2hN9ypunBX2Q/DRTYCtb+ck= +go.sia.tech/core v0.13.2 h1:66ZYzN2+AiHZmRayt4idSfoEoQZ5LV541+2E0luiizA= +go.sia.tech/core v0.13.2/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= +go.sia.tech/coreutils v0.16.1 h1:FlicsokoxXI1WQAp2w9wGjDQAEb3dS7zY3vpumIpsoQ= +go.sia.tech/coreutils v0.16.1/go.mod h1:BPoX9f3/ViDi6STLMoeziZZKCJarVwGV3yXv5pl0opQ= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -61,24 +61,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f6ff5eb74f10e27216803eb41ef13483959cc076 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 14 Jun 2025 04:51:41 +0000 Subject: [PATCH 501/630] chore: prepare release 2.10.1 --- .changeset/update_core_to_v0132_and_coreutils_to_v0161.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_core_to_v0132_and_coreutils_to_v0161.md diff --git a/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md b/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md deleted file mode 100644 index db7fa02..0000000 --- a/.changeset/update_core_to_v0132_and_coreutils_to_v0161.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core to v0.13.2 and coreutils to v0.16.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e811511..dc460a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.10.1 (2025-06-14) + +### Fixes + +- Update core to v0.13.2 and coreutils to v0.16.1 + ## 2.10.0 (2025-06-07) ### Features diff --git a/go.mod b/go.mod index d63f63f..ea0a437 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.0 +module go.sia.tech/walletd/v2 // v2.10.1 go 1.24.2 From ff5ec12e42ecda5b5ee8a455c82414ba568c12f7 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 14 Jun 2025 08:49:40 -0700 Subject: [PATCH 502/630] update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ea0a437..b741f1d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.2 - go.sia.tech/coreutils v0.16.1 + go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.29.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 039491e..e3885b4 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.sia.tech/core v0.13.2 h1:66ZYzN2+AiHZmRayt4idSfoEoQZ5LV541+2E0luiizA= go.sia.tech/core v0.13.2/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= -go.sia.tech/coreutils v0.16.1 h1:FlicsokoxXI1WQAp2w9wGjDQAEb3dS7zY3vpumIpsoQ= -go.sia.tech/coreutils v0.16.1/go.mod h1:BPoX9f3/ViDi6STLMoeziZZKCJarVwGV3yXv5pl0opQ= +go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af h1:s32mW2KWeE4/HTTE2h9GUqVmr3bH8rTA9Sa7iTBlWrk= +go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af/go.mod h1:BPoX9f3/ViDi6STLMoeziZZKCJarVwGV3yXv5pl0opQ= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 7386e177c34b97d8bf122d4e5ac1c707f6925a60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 17:32:15 +0000 Subject: [PATCH 503/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.29.3 to 0.30.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.29.3...hostd@0.30.0) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-version: 0.30.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b741f1d..6323c7c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.13.2 go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.29.3 + go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.32.0 diff --git a/go.sum b/go.sum index e3885b4..39e0dbf 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.29.3 h1:65Jl/2qAH+BECam3rJ6bp/GIONMct6XjClO1f7IjfYo= -go.sia.tech/web/walletd v0.29.3/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.30.0 h1:nTDEmN7dWHT/gddBs6LSYzYGjF0MDofe26yxz56YH7k= +go.sia.tech/web/walletd v0.30.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= From b8c719f0c8c02d5ced82fdfe935fb9358f491282 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 17 Jun 2025 07:49:31 -0700 Subject: [PATCH 504/630] update coreutils --- .changeset/update_coreutils_to_v0162.md | 5 +++++ go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .changeset/update_coreutils_to_v0162.md diff --git a/.changeset/update_coreutils_to_v0162.md b/.changeset/update_coreutils_to_v0162.md new file mode 100644 index 0000000..6d2fc98 --- /dev/null +++ b/.changeset/update_coreutils_to_v0162.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils to v0.16.2 diff --git a/go.mod b/go.mod index 6323c7c..1261acf 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.2 - go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af + go.sia.tech/coreutils v0.16.2 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 @@ -25,7 +25,7 @@ require ( github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.52.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect - go.etcd.io/bbolt v1.4.0 // indirect + go.etcd.io/bbolt v1.4.1 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.0 // indirect diff --git a/go.sum b/go.sum index 39e0dbf..cf0ec8f 100644 --- a/go.sum +++ b/go.sum @@ -39,12 +39,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= -go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= +go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= go.sia.tech/core v0.13.2 h1:66ZYzN2+AiHZmRayt4idSfoEoQZ5LV541+2E0luiizA= go.sia.tech/core v0.13.2/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= -go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af h1:s32mW2KWeE4/HTTE2h9GUqVmr3bH8rTA9Sa7iTBlWrk= -go.sia.tech/coreutils v0.16.2-0.20250614154532-fc51637947af/go.mod h1:BPoX9f3/ViDi6STLMoeziZZKCJarVwGV3yXv5pl0opQ= +go.sia.tech/coreutils v0.16.2 h1:sAVwl7s9bBqBp9q1C93J9aY6ijdEtYHIhZkrde2GLsE= +go.sia.tech/coreutils v0.16.2/go.mod h1:egAAlR7vju4nMppZiIEfHcwr84abuW9wg/ytcu4kSnA= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From a1dc7bb04d0b9f1a11d0b881cf0cace01967d727 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 14:49:45 +0000 Subject: [PATCH 505/630] chore: prepare release 2.10.2 --- .changeset/update_coreutils_to_v0162.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_coreutils_to_v0162.md diff --git a/.changeset/update_coreutils_to_v0162.md b/.changeset/update_coreutils_to_v0162.md deleted file mode 100644 index 6d2fc98..0000000 --- a/.changeset/update_coreutils_to_v0162.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to v0.16.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index dc460a5..7d446f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.10.2 (2025-06-17) + +### Fixes + +- Update coreutils to v0.16.2 + ## 2.10.1 (2025-06-14) ### Fixes diff --git a/go.mod b/go.mod index 1261acf..991f4a4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.1 +module go.sia.tech/walletd/v2 // v2.10.2 go 1.24.2 From 6119f20576fd8fd1547b2ee6c0ef6a5d441420ee Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 18 Jun 2025 13:25:19 -0700 Subject: [PATCH 506/630] add chain log --- cmd/walletd/node.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 497da3f..7d017eb 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -240,7 +240,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } - cm := chain.NewManager(dbstore, tipState) + cm := chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) if err != nil { diff --git a/go.mod b/go.mod index 991f4a4..4d1204d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.13.2 - go.sia.tech/coreutils v0.16.2 + go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index cf0ec8f..143849c 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= go.sia.tech/core v0.13.2 h1:66ZYzN2+AiHZmRayt4idSfoEoQZ5LV541+2E0luiizA= go.sia.tech/core v0.13.2/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= -go.sia.tech/coreutils v0.16.2 h1:sAVwl7s9bBqBp9q1C93J9aY6ijdEtYHIhZkrde2GLsE= -go.sia.tech/coreutils v0.16.2/go.mod h1:egAAlR7vju4nMppZiIEfHcwr84abuW9wg/ytcu4kSnA= +go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd h1:i7dzaVDcDfHVvaXHkmzp0SaeOKnN/pmeb2TR+rdTo8I= +go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd/go.mod h1:egAAlR7vju4nMppZiIEfHcwr84abuW9wg/ytcu4kSnA= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 15ed6635c5454ee59c12709908dfaf37086cb002 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 18 Jun 2025 13:26:33 -0700 Subject: [PATCH 507/630] update coreutils --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4d1204d..abd8e70 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.13.2 - go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd + go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7 + go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 143849c..01e8790 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= -go.sia.tech/core v0.13.2 h1:66ZYzN2+AiHZmRayt4idSfoEoQZ5LV541+2E0luiizA= -go.sia.tech/core v0.13.2/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= -go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd h1:i7dzaVDcDfHVvaXHkmzp0SaeOKnN/pmeb2TR+rdTo8I= -go.sia.tech/coreutils v0.16.3-0.20250618171735-d2a733a1d4fd/go.mod h1:egAAlR7vju4nMppZiIEfHcwr84abuW9wg/ytcu4kSnA= +go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7 h1:4KCpwSuMMZuiWcSOpQ9GFLpGhIxVdaqOnd4Bz2U590c= +go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= +go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758 h1:uVnxQpv/JtThI01nb3jqDyLHJrCm0QONSyOW3WGYW4Q= +go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758/go.mod h1:tcO75XD7wSV+neJJiS036882NnWSfOXDcU9nlw1r6U8= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 51e5347379d2070e561bacf5237762c627c10a64 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 19 Jun 2025 07:16:45 -0700 Subject: [PATCH 508/630] api: prevent server error when mining blocks --- .changeset/fixed_debug_miner_error.md | 5 +++++ api/server.go | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .changeset/fixed_debug_miner_error.md diff --git a/.changeset/fixed_debug_miner_error.md b/.changeset/fixed_debug_miner_error.md new file mode 100644 index 0000000..a2e5abd --- /dev/null +++ b/.changeset/fixed_debug_miner_error.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Fixed debug miner error. diff --git a/api/server.go b/api/server.go index 72875fe..4164b92 100644 --- a/api/server.go +++ b/api/server.go @@ -1553,12 +1553,12 @@ func (s *server) debugMineHandler(jc jape.Context) { } if b.V2 == nil { - if jc.Check("failed to broadcast header", s.s.BroadcastHeader(b.Header())) != nil { - return + if err := s.s.BroadcastHeader(b.Header()); err != nil { + log.Warn("failed to broadcast header", zap.Error(err)) } } else { - if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { - return + if err := s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())); err != nil { + log.Warn("failed to broadcast block outline", zap.Error(err)) } } From 5fc6e307e05d2a1f5e78f65547d41cb0cb8bf73c Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 19 Jun 2025 13:08:20 -0700 Subject: [PATCH 509/630] fix broadcast errors when debugging --- ...e_broadcast_error_when_debug_is_enabled.md | 5 ++++ api/server.go | 23 +++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 .changeset/ignore_broadcast_error_when_debug_is_enabled.md diff --git a/.changeset/ignore_broadcast_error_when_debug_is_enabled.md b/.changeset/ignore_broadcast_error_when_debug_is_enabled.md new file mode 100644 index 0000000..4da0bdc --- /dev/null +++ b/.changeset/ignore_broadcast_error_when_debug_is_enabled.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Ignore broadcast error when debug is enabled. diff --git a/api/server.go b/api/server.go index 4164b92..a494db7 100644 --- a/api/server.go +++ b/api/server.go @@ -421,8 +421,15 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if err != nil { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return - } else if jc.Check("failed to broadcast transaction set", s.s.BroadcastTransactionSet(tbr.Transactions)) != nil { - return + } + + err = s.s.BroadcastTransactionSet(tbr.Transactions) + if err != nil { + if s.debugEnabled { + s.log.Warn("failed to broadcast transaction set", zap.Error(err), zap.Any("transactions", tbr.Transactions)) + } else { + jc.Error(fmt.Errorf("failed to broadcast transaction set: %w", err), http.StatusInternalServerError) + } } } if len(tbr.V2Transactions) != 0 { @@ -456,8 +463,16 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { if _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions); err != nil { jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return - } else if jc.Check("failed to broadcast transaction set", s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions)) != nil { - return + } + + err = s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions) + if err != nil { + if s.debugEnabled { + s.log.Warn("failed to broadcast v2 transaction set", zap.Error(err), zap.Any("basis", tbr.Basis), zap.Any("transactions", tbr.V2Transactions)) + } else { + jc.Error(fmt.Errorf("failed to broadcast v2 transaction set: %w", err), http.StatusInternalServerError) + return + } } } resp.Basis = tbr.Basis From 77034ca582821f1bda4d5a9ad0a2de8447db803f Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 19 Jun 2025 20:42:11 -0700 Subject: [PATCH 510/630] update core and coreutils --- .changeset/update_core_to_v0140_and_coreutils_to_v0163.md | 5 +++++ go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .changeset/update_core_to_v0140_and_coreutils_to_v0163.md diff --git a/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md b/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md new file mode 100644 index 0000000..ff34733 --- /dev/null +++ b/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core to v0.14.0 and coreutils to v0.16.3 diff --git a/go.mod b/go.mod index abd8e70..da606be 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7 - go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758 + go.sia.tech/core v0.14.0 + go.sia.tech/coreutils v0.16.3 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 01e8790..623287d 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= -go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7 h1:4KCpwSuMMZuiWcSOpQ9GFLpGhIxVdaqOnd4Bz2U590c= -go.sia.tech/core v0.13.3-0.20250616154238-4c58987023c7/go.mod h1:bur1jeLA1JQbwzZkc2ijSTdpJYusG4h0pV9IwLOHT0g= -go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758 h1:uVnxQpv/JtThI01nb3jqDyLHJrCm0QONSyOW3WGYW4Q= -go.sia.tech/coreutils v0.16.3-0.20250618174006-041c22c13758/go.mod h1:tcO75XD7wSV+neJJiS036882NnWSfOXDcU9nlw1r6U8= +go.sia.tech/core v0.14.0 h1:U8riaW0GBjeC1JSGbOJtotJ4XdYVRpXZpaJeuEugYBY= +go.sia.tech/core v0.14.0/go.mod h1:LhT4M4HZjOvabLFcTZUO52XjzJiUJCPcKdVX1N+hQ14= +go.sia.tech/coreutils v0.16.3 h1:7fmxTJa2QeK68ra9BMsLcCwQf9b+f4dgevuVZDySs0o= +go.sia.tech/coreutils v0.16.3/go.mod h1:adcWbmTxWcgxHkHM93BVnTa/NyV1SJ2GyycwM1LaquU= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 131bd45a5c03638d93af312897ecd80bd14500bb Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 19 Jun 2025 20:49:39 -0700 Subject: [PATCH 511/630] fix test ndf --- wallet/addresses_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 4327d75..745c19b 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -176,11 +176,13 @@ func TestBatchSiafundOutputs(t *testing.T) { } defer wm.Close() + cn.WaitForSync(t) + // distribute the siafund output to multiple addresses var addresses []types.Address outputID := genesisBlock.Transactions[0].SiafundOutputID(0) outputValue := genesisBlock.Transactions[0].SiafundOutputs[0].Value - for range 100 { + for i := range 100 { txn := types.V2Transaction{ SiafundInputs: []types.V2SiafundInput{ { @@ -216,10 +218,10 @@ func TestBatchSiafundOutputs(t *testing.T) { outputID = txn.SiafundOutputID(txn.ID(), len(txn.SiafundOutputs)-1) basis, txns, err := db.OverwriteElementProofs([]types.V2Transaction{txn}) if err != nil { - t.Fatal(err) + t.Fatalf("failed to update element proofs %d: %s", i, err) } if _, err := cm.AddV2PoolTransactions(basis, txns); err != nil { - t.Fatal(err) + t.Fatalf("failed to add pool transactions %d: %s", i, err) } cn.MineBlocks(t, types.VoidAddress, 1) } From aaa8c9b4c35401cbe4b710b9ec1d540345125db1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 03:49:53 +0000 Subject: [PATCH 512/630] chore: prepare release 2.10.3 --- .changeset/fixed_debug_miner_error.md | 5 ----- .../ignore_broadcast_error_when_debug_is_enabled.md | 5 ----- .changeset/update_core_to_v0140_and_coreutils_to_v0163.md | 5 ----- CHANGELOG.md | 8 ++++++++ go.mod | 2 +- 5 files changed, 9 insertions(+), 16 deletions(-) delete mode 100644 .changeset/fixed_debug_miner_error.md delete mode 100644 .changeset/ignore_broadcast_error_when_debug_is_enabled.md delete mode 100644 .changeset/update_core_to_v0140_and_coreutils_to_v0163.md diff --git a/.changeset/fixed_debug_miner_error.md b/.changeset/fixed_debug_miner_error.md deleted file mode 100644 index a2e5abd..0000000 --- a/.changeset/fixed_debug_miner_error.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Fixed debug miner error. diff --git a/.changeset/ignore_broadcast_error_when_debug_is_enabled.md b/.changeset/ignore_broadcast_error_when_debug_is_enabled.md deleted file mode 100644 index 4da0bdc..0000000 --- a/.changeset/ignore_broadcast_error_when_debug_is_enabled.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Ignore broadcast error when debug is enabled. diff --git a/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md b/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md deleted file mode 100644 index ff34733..0000000 --- a/.changeset/update_core_to_v0140_and_coreutils_to_v0163.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core to v0.14.0 and coreutils to v0.16.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d446f5..14ef263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.10.3 (2025-06-20) + +### Fixes + +- Fixed debug miner error. +- Ignore broadcast error when debug is enabled. +- Update core to v0.14.0 and coreutils to v0.16.3 + ## 2.10.2 (2025-06-17) ### Fixes diff --git a/go.mod b/go.mod index da606be..eb46215 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.2 +module go.sia.tech/walletd/v2 // v2.10.3 go 1.24.2 From 5dd23bc2d8344140d52d5a99855ef46b42f9c9b6 Mon Sep 17 00:00:00 2001 From: CtrlAltDefeat94 Date: Wed, 25 Jun 2025 10:54:19 +0200 Subject: [PATCH 513/630] Update incorrect default mode Closes #324 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b30288..8e3b8a3 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Flags: -index.batch int max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 1000) -index.mode string - address index mode (personal, full, none) (default "full") + address index mode (personal, full, none) (default "personal") -network string network to connect to; must be one of 'mainnet', 'zen', 'anagami', or the path to a custom network file for a local testnet -upnp From 654169654c660b0918c108d9db67238b99b9c769 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 26 Jun 2025 05:46:43 -0700 Subject: [PATCH 514/630] update core --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eb46215..e38af76 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.14.0 + go.sia.tech/core v0.14.1 go.sia.tech/coreutils v0.16.3 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 diff --git a/go.sum b/go.sum index 623287d..7b8c279 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= -go.sia.tech/core v0.14.0 h1:U8riaW0GBjeC1JSGbOJtotJ4XdYVRpXZpaJeuEugYBY= -go.sia.tech/core v0.14.0/go.mod h1:LhT4M4HZjOvabLFcTZUO52XjzJiUJCPcKdVX1N+hQ14= +go.sia.tech/core v0.14.1 h1:4MvZBjuZiz0IQk9eqU/NL8mTK5BAMca3L3lUzszGHqs= +go.sia.tech/core v0.14.1/go.mod h1:Uhrw7JKxtDCr5ZW2homCKrRPKTgZBy9t35vfBWWFMxc= go.sia.tech/coreutils v0.16.3 h1:7fmxTJa2QeK68ra9BMsLcCwQf9b+f4dgevuVZDySs0o= go.sia.tech/coreutils v0.16.3/go.mod h1:adcWbmTxWcgxHkHM93BVnTa/NyV1SJ2GyycwM1LaquU= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= From 4fb210c5a7b2b27ccac1acd8dc31152df46a7449 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 08:56:27 +0000 Subject: [PATCH 515/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.16.3 to 0.16.4 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.16.3...v0.16.4) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.16.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 13 +++++-------- go.sum | 38 ++++++++++---------------------------- 2 files changed, 15 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index e38af76..e2f97b7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.2 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.14.1 - go.sia.tech/coreutils v0.16.3 + go.sia.tech/coreutils v0.16.4 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.30.0 go.uber.org/zap v1.27.0 @@ -18,17 +18,14 @@ require ( ) require ( - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.52.0 // indirect - github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect - go.etcd.io/bbolt v1.4.1 // indirect + github.com/quic-go/quic-go v0.53.0 // indirect + github.com/quic-go/webtransport-go v0.9.0 // indirect + go.etcd.io/bbolt v1.4.2 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect - go.uber.org/mock v0.5.0 // indirect + go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.39.0 // indirect golang.org/x/mod v0.25.0 // indirect diff --git a/go.sum b/go.sum index 7b8c279..087fcdd 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,9 @@ -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -21,30 +12,24 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= -github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= -github.com/quic-go/quic-go v0.52.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= -github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= -github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI= +github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= +github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= -go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= +go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= +go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= go.sia.tech/core v0.14.1 h1:4MvZBjuZiz0IQk9eqU/NL8mTK5BAMca3L3lUzszGHqs= go.sia.tech/core v0.14.1/go.mod h1:Uhrw7JKxtDCr5ZW2homCKrRPKTgZBy9t35vfBWWFMxc= -go.sia.tech/coreutils v0.16.3 h1:7fmxTJa2QeK68ra9BMsLcCwQf9b+f4dgevuVZDySs0o= -go.sia.tech/coreutils v0.16.3/go.mod h1:adcWbmTxWcgxHkHM93BVnTa/NyV1SJ2GyycwM1LaquU= +go.sia.tech/coreutils v0.16.4 h1:1uKEq6c2/Ad+pBNrhSYEx552MsJSed0PrcKaszII0nc= +go.sia.tech/coreutils v0.16.4/go.mod h1:BmdMVo+MYSOhc2D/C04+JPgcmpwYgBguj0kk219uvBQ= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -55,8 +40,8 @@ go.sia.tech/web/walletd v0.30.0 h1:nTDEmN7dWHT/gddBs6LSYzYGjF0MDofe26yxz56YH7k= go.sia.tech/web/walletd v0.30.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= @@ -79,12 +64,9 @@ golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= From f347d868cee130e96c94d4400c98041b799e32e0 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 1 Jul 2025 11:03:07 +0200 Subject: [PATCH 516/630] changeset --- .changeset/update_coreutils_from_0163_to_0164.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_coreutils_from_0163_to_0164.md diff --git a/.changeset/update_coreutils_from_0163_to_0164.md b/.changeset/update_coreutils_from_0163_to_0164.md new file mode 100644 index 0000000..b9d233c --- /dev/null +++ b/.changeset/update_coreutils_from_0163_to_0164.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils from 0.16.3 to 0.16.4 From 04cae3f45b8bef0b74b8b43624e7390cd4e25b77 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 09:05:30 +0000 Subject: [PATCH 517/630] chore: prepare release 2.10.4 --- .changeset/update_coreutils_from_0163_to_0164.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_coreutils_from_0163_to_0164.md diff --git a/.changeset/update_coreutils_from_0163_to_0164.md b/.changeset/update_coreutils_from_0163_to_0164.md deleted file mode 100644 index b9d233c..0000000 --- a/.changeset/update_coreutils_from_0163_to_0164.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils from 0.16.3 to 0.16.4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ef263..9153ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.10.4 (2025-07-01) + +### Fixes + +- Update coreutils from 0.16.3 to 0.16.4 + ## 2.10.3 (2025-06-20) ### Fixes diff --git a/go.mod b/go.mod index e2f97b7..8441d81 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.3 +module go.sia.tech/walletd/v2 // v2.10.4 go 1.24.2 From e223b8096e3d13fd74fa86e9c955afac53e1e4f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 19:31:37 +0000 Subject: [PATCH 518/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/coreutils` from 0.16.4 to 0.16.5 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.16.4...v0.16.5) Updates `go.sia.tech/web/walletd` from 0.30.0 to 0.32.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.30.0...hostd@0.32.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.16.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.32.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 8441d81..338f943 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module go.sia.tech/walletd/v2 // v2.10.4 -go 1.24.2 +go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.28 go.sia.tech/core v0.14.1 - go.sia.tech/coreutils v0.16.4 + go.sia.tech/coreutils v0.16.5 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.30.0 + go.sia.tech/web/walletd v0.32.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.32.0 diff --git a/go.sum b/go.sum index 087fcdd..a7ddc96 100644 --- a/go.sum +++ b/go.sum @@ -28,16 +28,16 @@ go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= go.sia.tech/core v0.14.1 h1:4MvZBjuZiz0IQk9eqU/NL8mTK5BAMca3L3lUzszGHqs= go.sia.tech/core v0.14.1/go.mod h1:Uhrw7JKxtDCr5ZW2homCKrRPKTgZBy9t35vfBWWFMxc= -go.sia.tech/coreutils v0.16.4 h1:1uKEq6c2/Ad+pBNrhSYEx552MsJSed0PrcKaszII0nc= -go.sia.tech/coreutils v0.16.4/go.mod h1:BmdMVo+MYSOhc2D/C04+JPgcmpwYgBguj0kk219uvBQ= +go.sia.tech/coreutils v0.16.5 h1:IgXsid6NiswiRmCiq9I+NWxIapsWwEKheNHAEUgvq3Y= +go.sia.tech/coreutils v0.16.5/go.mod h1:XHCKZPfZ9622VkpRTrLXxDtG4ytpnDOC6y7ZxfYizT0= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.30.0 h1:nTDEmN7dWHT/gddBs6LSYzYGjF0MDofe26yxz56YH7k= -go.sia.tech/web/walletd v0.30.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.sia.tech/web/walletd v0.32.0 h1:VF9xvSLwo94OIYRfrX0LZHMYRdeeOQAmGTTtL7W1F0k= +go.sia.tech/web/walletd v0.32.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From 69001916184d13b59b27c44beb215f9de5651bd9 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 26 Jul 2025 11:30:33 -0700 Subject: [PATCH 519/630] update coreutils --- .golangci.yml | 234 +++++++++++++++++++++----------------------------- api/server.go | 33 ++----- go.mod | 20 ++--- go.sum | 40 ++++----- 4 files changed, 135 insertions(+), 192 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 050db11..bf878d6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,150 +1,110 @@ -# Based off of the example file at https://github.com/golangci/golangci-lint - -# options for analysis running +version: "2" run: - # default concurrency is a available CPU number concurrency: 4 - - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 600s - - # exit code when at least one issue was found, default is 1 issues-exit-code: 1 - - # include test files or not, default is true tests: true - - # list of build tags, all linters use it. Default is empty list. - build-tags: [] - -# output configuration options -output: - # print lines of code with issue, default is true - print-issued-lines: true - - # print linter name in the end of issue text, default is true - print-linter-name: true - -# all available settings of specific linters -linters-settings: - ## Enabled linters: - govet: - # report about shadowed variables - disable-all: false - - tagliatelle: - case: - rules: - json: goCamel - yaml: goCamel - - - gocritic: - # Which checks should be enabled; can't be combined with 'disabled-checks'; - # See https://go-critic.github.io/overview#checks-overview - # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` - # By default list of stable checks is used. - enabled-tags: - - diagnostic - - style - disabled-checks: - # diagnostic - - appendAssign - - commentedOutCode - - uncheckedInlineErr - # style - - httpNoBody - - exitAfterDefer - - ifElseChain - - importShadow - - initClause - - nestingReduce - - octalLiteral - - paramTypeCombine - - ptrToRefParam - - stringsCompare - - tooManyResultsChecker - - typeDefFirst - - typeUnparen - - unlabelStmt - - unnamedResult - - whyNoLint - revive: - ignore-generated-header: true - rules: - - name: blank-imports - disabled: false - - name: bool-literal-in-expr - disabled: false - - name: confusing-naming - disabled: false - - name: confusing-results - disabled: false - - name: constant-logical-expr - disabled: false - - name: context-as-argument - disabled: false - - name: exported - disabled: false - - name: errorf - disabled: false - - name: if-return - disabled: false - - name: indent-error-flow - disabled: true - - name: increment-decrement - disabled: false - - name: modifies-value-receiver - disabled: true - - name: optimize-operands-order - disabled: false - - name: range-val-in-closure - disabled: false - - name: struct-tag - disabled: false - - name: superfluous-else - disabled: false - - name: time-equal - disabled: false - - name: unexported-naming - disabled: false - - name: unexported-return - disabled: false - - name: unnecessary-stmt - disabled: false - - name: unreachable-code - disabled: false - - name: package-comments - disabled: true - linters: - disable-all: true - fast: false + default: none enable: - - tagliatelle - gocritic - - gofmt - - revive - govet - misspell - - typecheck + - revive + - tagliatelle - whitespace - + settings: + gocritic: + disabled-checks: + - appendAssign + - commentedOutCode + - uncheckedInlineErr + - httpNoBody + - exitAfterDefer + - ifElseChain + - importShadow + - initClause + - nestingReduce + - octalLiteral + - paramTypeCombine + - ptrToRefParam + - stringsCompare + - tooManyResultsChecker + - typeDefFirst + - typeUnparen + - unlabelStmt + - unnamedResult + - whyNoLint + enabled-tags: + - diagnostic + - style + govet: + disable-all: false + revive: + rules: + - name: blank-imports + disabled: false + - name: bool-literal-in-expr + disabled: false + - name: confusing-naming + disabled: false + - name: confusing-results + disabled: false + - name: constant-logical-expr + disabled: false + - name: context-as-argument + disabled: false + - name: exported + disabled: false + - name: errorf + disabled: false + - name: if-return + disabled: false + - name: indent-error-flow + disabled: true + - name: increment-decrement + disabled: false + - name: modifies-value-receiver + disabled: true + - name: optimize-operands-order + disabled: false + - name: range-val-in-closure + disabled: false + - name: struct-tag + disabled: false + - name: superfluous-else + disabled: false + - name: time-equal + disabled: false + - name: unexported-naming + disabled: false + - name: unexported-return + disabled: false + - name: unnecessary-stmt + disabled: false + - name: unreachable-code + disabled: false + - name: package-comments + disabled: true + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ issues: - # Maximum issues count per one linter. Set to 0 to disable. Default is 50. max-issues-per-linter: 0 - - # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 - - # List of regexps of issue texts to exclude, empty list by default. - # But independently from this option we use default exclude patterns, - # it can be disabled by `exclude-use-default: false`. To list all - # excluded by default patterns execute `golangci-lint run --help` - exclude: [] - - # Independently from option `exclude` we use default exclude patterns, - # it can be disabled by this option. To list all - # excluded by default patterns execute `golangci-lint run --help`. - # Default value for this option is true. - exclude-use-default: false \ No newline at end of file +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/api/server.go b/api/server.go index a494db7..a8d16fa 100644 --- a/api/server.go +++ b/api/server.go @@ -83,8 +83,6 @@ type ( Peers() []*syncer.Peer PeerInfo(addr string) (syncer.PeerInfo, error) Connect(ctx context.Context, addr string) (*syncer.Peer, error) - BroadcastHeader(types.BlockHeader) error - BroadcastTransactionSet(txns []types.Transaction) error BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) error BroadcastV2BlockOutline(bo gateway.V2BlockOutline) error } @@ -364,15 +362,11 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { return } else if jc.Check("block is invalid", s.cm.AddBlocks([]types.Block{b})) != nil { return - } - if b.V2 == nil { - if jc.Check("failed to broadcast header", s.s.BroadcastHeader(b.Header())) != nil { - return - } - } else { - if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { - return - } + } else if b.V2 == nil { + jc.Error(errors.New("v1 blocks are unsupported"), http.StatusBadRequest) + return + } else if jc.Check("failed to broadcast block outline", s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions()))) != nil { + return } jc.Encode(nil) } @@ -399,6 +393,7 @@ func (s *server) txpoolFeeHandler(jc jape.Context) { } func (s *server) txpoolBroadcastHandler(jc jape.Context) { + // TODO: remove support for V1 transactions in a follow up var tbr TxpoolBroadcastRequest if jc.Decode(&tbr) != nil { return @@ -422,15 +417,6 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return } - - err = s.s.BroadcastTransactionSet(tbr.Transactions) - if err != nil { - if s.debugEnabled { - s.log.Warn("failed to broadcast transaction set", zap.Error(err), zap.Any("transactions", tbr.Transactions)) - } else { - jc.Error(fmt.Errorf("failed to broadcast transaction set: %w", err), http.StatusInternalServerError) - } - } } if len(tbr.V2Transactions) != 0 { var err error @@ -1567,11 +1553,8 @@ func (s *server) debugMineHandler(jc jape.Context) { log.Warn("failed to add block", zap.Error(err)) } - if b.V2 == nil { - if err := s.s.BroadcastHeader(b.Header()); err != nil { - log.Warn("failed to broadcast header", zap.Error(err)) - } - } else { + // TODO: remove support for V1 blocks in a follow up + if b.V2 != nil { if err := s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())); err != nil { log.Warn("failed to broadcast block outline", zap.Error(err)) } diff --git a/go.mod b/go.mod index 338f943..57ef314 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.14.1 - go.sia.tech/coreutils v0.16.5 + go.sia.tech/core v0.14.3 + go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.32.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.32.0 + golang.org/x/term v0.33.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -27,11 +27,11 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/mod v0.25.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - golang.org/x/tools v0.34.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + golang.org/x/tools v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index a7ddc96..224e861 100644 --- a/go.sum +++ b/go.sum @@ -26,10 +26,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.14.1 h1:4MvZBjuZiz0IQk9eqU/NL8mTK5BAMca3L3lUzszGHqs= -go.sia.tech/core v0.14.1/go.mod h1:Uhrw7JKxtDCr5ZW2homCKrRPKTgZBy9t35vfBWWFMxc= -go.sia.tech/coreutils v0.16.5 h1:IgXsid6NiswiRmCiq9I+NWxIapsWwEKheNHAEUgvq3Y= -go.sia.tech/coreutils v0.16.5/go.mod h1:XHCKZPfZ9622VkpRTrLXxDtG4ytpnDOC6y7ZxfYizT0= +go.sia.tech/core v0.14.3 h1:cqg+Ub5+FCBtwDs2qAsSXRRS7y9EfeQ3YSiDoQqYPlA= +go.sia.tech/core v0.14.3/go.mod h1:LTwIv96zPnsUbHNOUlXmo73B1A3PCmFFjilr14R1/dY= +go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580 h1:dj8doTEf8tx9uT47jWiQqoxBYB0IydppAxCJnrbIIDg= +go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580/go.mod h1:KgkWPWGFsJE/Rq4bosAF60Maxu45JfaM5Kh7AEEGwJ0= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 647bc0c35e83420e4c88ba13bae4f05dd81e8525 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:00:25 +0000 Subject: [PATCH 520/630] build(deps): bump the all-dependencies group with 4 updates Bumps the all-dependencies group with 4 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `github.com/mattn/go-sqlite3` from 1.14.28 to 1.14.29 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.28...v1.14.29) Updates `go.sia.tech/core` from 0.14.3 to 0.16.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.14.3...v0.16.0) Updates `go.sia.tech/coreutils` from 0.16.6-0.20250725192801-b7206fb99580 to 0.17.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/commits/v0.17.0) Updates `go.sia.tech/web/walletd` from 0.32.0 to 0.34.1 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.32.0...walletd@0.34.1) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.29 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/core dependency-version: 0.16.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.17.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.34.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 57ef314..8693ab9 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module go.sia.tech/walletd/v2 // v2.10.4 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.28 - go.sia.tech/core v0.14.3 - go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580 + github.com/mattn/go-sqlite3 v1.14.29 + go.sia.tech/core v0.16.0 + go.sia.tech/coreutils v0.17.0 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.32.0 + go.sia.tech/web/walletd v0.34.1 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.33.0 diff --git a/go.sum b/go.sum index 224e861..52b3a9c 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= -github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ= +github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -26,18 +26,18 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.14.3 h1:cqg+Ub5+FCBtwDs2qAsSXRRS7y9EfeQ3YSiDoQqYPlA= -go.sia.tech/core v0.14.3/go.mod h1:LTwIv96zPnsUbHNOUlXmo73B1A3PCmFFjilr14R1/dY= -go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580 h1:dj8doTEf8tx9uT47jWiQqoxBYB0IydppAxCJnrbIIDg= -go.sia.tech/coreutils v0.16.6-0.20250725192801-b7206fb99580/go.mod h1:KgkWPWGFsJE/Rq4bosAF60Maxu45JfaM5Kh7AEEGwJ0= +go.sia.tech/core v0.16.0 h1:XAvmyB48oqoYU+U8uCxuJBUWUEUpCv6hKMFpY7aOOCc= +go.sia.tech/core v0.16.0/go.mod h1:UfNGef3Jk7tWykIw7bIPIn7cNT7hRAxk+FgwdsmFLlg= +go.sia.tech/coreutils v0.17.0 h1:0ucYewcpBXHuBRqwt1fZNDDC73wt9Lsas+usIhotpGg= +go.sia.tech/coreutils v0.17.0/go.mod h1:4u2CM3gWaCk8833gH46YicYWDLT9Dyw3l9yRcrkCvHQ= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.32.0 h1:VF9xvSLwo94OIYRfrX0LZHMYRdeeOQAmGTTtL7W1F0k= -go.sia.tech/web/walletd v0.32.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.34.1 h1:vZmgzSDg4HM7j3a+gvoovWeKLe/o4jz0Vskl3OW0KFs= +go.sia.tech/web/walletd v0.34.1/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From c9da2f76f5d08ce9a2ed7cabfe4df281bba27d5e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 30 Jul 2025 12:01:13 +0200 Subject: [PATCH 521/630] fix build --- cmd/walletd/node.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 7d017eb..5e45e90 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -209,12 +209,6 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { case "zen": network, genesisBlock = chain.TestnetZen() bootstrapPeers = syncer.ZenBootstrapPeers - case "anagami": - network, genesisBlock = chain.TestnetAnagami() - bootstrapPeers = syncer.AnagamiBootstrapPeers - case "erravimus": - network, genesisBlock = chain.TestnetErravimus() - bootstrapPeers = syncer.ErravimusBootstrapPeers default: var err error network, genesisBlock, err = loadCustomNetwork(cfg.Consensus.Network) From f230faedfef2862029c14878bbc8b1d955efba4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:15:32 +0000 Subject: [PATCH 522/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.29 to 1.14.30 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.29...v1.14.30) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.30 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8693ab9..1f21551 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.10.4 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.29 + github.com/mattn/go-sqlite3 v1.14.30 go.sia.tech/core v0.16.0 go.sia.tech/coreutils v0.17.0 go.sia.tech/jape v0.14.0 diff --git a/go.sum b/go.sum index 52b3a9c..93837eb 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ= -github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.30 h1:bVreufq3EAIG1Quvws73du3/QgdeZ3myglJlrzSYYCY= +github.com/mattn/go-sqlite3 v1.14.30/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= From 315bc313873a77970301bf0b5da164bd9b84e66e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 23:28:13 +0000 Subject: [PATCH 523/630] build(deps): bump the all-dependencies group with 4 updates Bumps the all-dependencies group with 4 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.30 to 1.14.31 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.30...v1.14.31) Updates `go.sia.tech/core` from 0.16.0 to 0.17.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.16.0...v0.17.0) Updates `go.sia.tech/web/walletd` from 0.34.1 to 0.34.2 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.34.1...walletd@0.34.2) Updates `golang.org/x/term` from 0.33.0 to 0.34.0 - [Commits](https://github.com/golang/term/compare/v0.33.0...v0.34.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.31 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/core dependency-version: 0.17.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.34.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1f21551..2839e8d 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module go.sia.tech/walletd/v2 // v2.10.4 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.30 - go.sia.tech/core v0.16.0 + github.com/mattn/go-sqlite3 v1.14.31 + go.sia.tech/core v0.17.0 go.sia.tech/coreutils v0.17.0 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.34.1 + go.sia.tech/web/walletd v0.34.2 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.33.0 + golang.org/x/term v0.34.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -31,7 +31,7 @@ require ( golang.org/x/mod v0.26.0 // indirect golang.org/x/net v0.42.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/tools v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index 93837eb..f31cb50 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.30 h1:bVreufq3EAIG1Quvws73du3/QgdeZ3myglJlrzSYYCY= -github.com/mattn/go-sqlite3 v1.14.30/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.31 h1:ldt6ghyPJsokUIlksH63gWZkG6qVGeEAu4zLeS4aVZM= +github.com/mattn/go-sqlite3 v1.14.31/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -26,8 +26,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.16.0 h1:XAvmyB48oqoYU+U8uCxuJBUWUEUpCv6hKMFpY7aOOCc= -go.sia.tech/core v0.16.0/go.mod h1:UfNGef3Jk7tWykIw7bIPIn7cNT7hRAxk+FgwdsmFLlg= +go.sia.tech/core v0.17.0 h1:2uMm1dgw+vvb5LaaA7Dix+96JqEZDgYen8KFnu1oqYE= +go.sia.tech/core v0.17.0/go.mod h1:X6OoWgOU5I8m6nmIpY4PhxK2eExwv8VzeR538DjNyLw= go.sia.tech/coreutils v0.17.0 h1:0ucYewcpBXHuBRqwt1fZNDDC73wt9Lsas+usIhotpGg= go.sia.tech/coreutils v0.17.0/go.mod h1:4u2CM3gWaCk8833gH46YicYWDLT9Dyw3l9yRcrkCvHQ= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= @@ -36,8 +36,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.34.1 h1:vZmgzSDg4HM7j3a+gvoovWeKLe/o4jz0Vskl3OW0KFs= -go.sia.tech/web/walletd v0.34.1/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.34.2 h1:GtB4yMIod9/eyt0oH8uDg3hzeuEIWMMZLWecR8pQ4fU= +go.sia.tech/web/walletd v0.34.2/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -56,10 +56,10 @@ golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= From 72a59a695f30701e361477283dd9ea9c4d5888ad Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 18 Aug 2025 16:59:33 -0700 Subject: [PATCH 524/630] update coreutils --- api/mine.go | 2 +- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- persist/sqlite/consensus_test.go | 2 +- wallet/wallet_test.go | 4 ++-- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/mine.go b/api/mine.go index 6e43658..763f901 100644 --- a/api/mine.go +++ b/api/mine.go @@ -50,7 +50,7 @@ func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types. b.Nonce = 0 factor := cs.NonceFactor() - for b.ID().CmpWork(cs.ChildTarget) < 0 { + for b.ID().CmpWork(cs.PoWTarget()) < 0 { select { case <-ctx.Done(): return types.Block{}, ctx.Err() diff --git a/go.mod b/go.mod index 2839e8d..2e05873 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.31 - go.sia.tech/core v0.17.0 - go.sia.tech/coreutils v0.17.0 + go.sia.tech/core v0.17.1 + go.sia.tech/coreutils v0.18.0 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.34.2 go.uber.org/zap v1.27.0 @@ -20,18 +20,18 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.53.0 // indirect + github.com/quic-go/quic-go v0.54.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.2 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.42.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.27.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index f31cb50..a6b067e 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI= -github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= +github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -26,10 +26,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.17.0 h1:2uMm1dgw+vvb5LaaA7Dix+96JqEZDgYen8KFnu1oqYE= -go.sia.tech/core v0.17.0/go.mod h1:X6OoWgOU5I8m6nmIpY4PhxK2eExwv8VzeR538DjNyLw= -go.sia.tech/coreutils v0.17.0 h1:0ucYewcpBXHuBRqwt1fZNDDC73wt9Lsas+usIhotpGg= -go.sia.tech/coreutils v0.17.0/go.mod h1:4u2CM3gWaCk8833gH46YicYWDLT9Dyw3l9yRcrkCvHQ= +go.sia.tech/core v0.17.1 h1:7Dkw9H92xM20A48LpVWNBfXRZwmQYteN0brRD4TvEm4= +go.sia.tech/core v0.17.1/go.mod h1:yS+Uwpjs9C6mWSltwv410xOYwVgPtZPQcWJclVyjkMg= +go.sia.tech/coreutils v0.18.0 h1:tl5DJbfbj4YTduUyRTZ8l1VMTqLeT4qb/PmDpPzwAyM= +go.sia.tech/coreutils v0.18.0/go.mod h1:hpk9tlWoi8JprlTUe7NPuCSOG4E+ouHSbfmbtXXFNJY= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 6e70456..d1b1325 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -20,7 +20,7 @@ func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types. Transactions: txns, MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, } - for b.ID().CmpWork(state.ChildTarget) < 0 { + for b.ID().CmpWork(state.PoWTarget()) < 0 { b.Nonce += state.NonceFactor() } return b diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 0aefac2..a43cfe8 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -85,7 +85,7 @@ func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types. Transactions: txns, MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, } - for b.ID().CmpWork(state.ChildTarget) < 0 { + for b.ID().CmpWork(state.PoWTarget()) < 0 { b.Nonce += state.NonceFactor() } return b @@ -103,7 +103,7 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty }, } b.V2.Commitment = state.Commitment(b.MinerPayouts[0].Address, b.Transactions, b.V2Transactions()) - for b.ID().CmpWork(state.ChildTarget) < 0 { + for b.ID().CmpWork(state.PoWTarget()) < 0 { b.Nonce += state.NonceFactor() } return b From e2bc2048bdcae6feaac2efb1654a64ecf5f63f1d Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 18 Aug 2025 16:59:57 -0700 Subject: [PATCH 525/630] changeset --- .changeset/update_core_to_v0171_and_coreutils_to_v0180.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_core_to_v0171_and_coreutils_to_v0180.md diff --git a/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md b/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md new file mode 100644 index 0000000..5415e96 --- /dev/null +++ b/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core to v0.17.1 and coreutils to v0.18.0 From f61b754e1eda748c9a3800a735a4449286525323 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 00:18:49 +0000 Subject: [PATCH 526/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `github.com/mattn/go-sqlite3` from 1.14.31 to 1.14.32 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.31...v1.14.32) Updates `go.sia.tech/web/walletd` from 0.34.2 to 0.34.3 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.34.2...walletd@0.34.3) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.32 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.34.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2e05873..31bb6a8 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module go.sia.tech/walletd/v2 // v2.10.4 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.31 + github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.17.1 go.sia.tech/coreutils v0.18.0 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.34.2 + go.sia.tech/web/walletd v0.34.3 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.34.0 diff --git a/go.sum b/go.sum index a6b067e..20a752b 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.31 h1:ldt6ghyPJsokUIlksH63gWZkG6qVGeEAu4zLeS4aVZM= -github.com/mattn/go-sqlite3 v1.14.31/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -36,8 +36,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.34.2 h1:GtB4yMIod9/eyt0oH8uDg3hzeuEIWMMZLWecR8pQ4fU= -go.sia.tech/web/walletd v0.34.2/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.34.3 h1:t1XxjNFyxFkFgX3lYTq8rTxslL5knSVySMsfuz4JOPY= +go.sia.tech/web/walletd v0.34.3/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From 551c2b3be838b45d961f79dabbcecd50457b2206 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 03:28:16 +0000 Subject: [PATCH 527/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.17.1 to 0.17.3 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.17.1...v0.17.3) Updates `go.sia.tech/coreutils` from 0.18.0 to 0.18.1 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.0...v0.18.1) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.17.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.18.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 31bb6a8..e67d29d 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 - go.sia.tech/core v0.17.1 - go.sia.tech/coreutils v0.18.0 + go.sia.tech/core v0.17.3 + go.sia.tech/coreutils v0.18.1 go.sia.tech/jape v0.14.0 go.sia.tech/web/walletd v0.34.3 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 20a752b..fd5b8d2 100644 --- a/go.sum +++ b/go.sum @@ -26,10 +26,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.17.1 h1:7Dkw9H92xM20A48LpVWNBfXRZwmQYteN0brRD4TvEm4= -go.sia.tech/core v0.17.1/go.mod h1:yS+Uwpjs9C6mWSltwv410xOYwVgPtZPQcWJclVyjkMg= -go.sia.tech/coreutils v0.18.0 h1:tl5DJbfbj4YTduUyRTZ8l1VMTqLeT4qb/PmDpPzwAyM= -go.sia.tech/coreutils v0.18.0/go.mod h1:hpk9tlWoi8JprlTUe7NPuCSOG4E+ouHSbfmbtXXFNJY= +go.sia.tech/core v0.17.3 h1:Q+5lR7cPa4nR2MSyLhFxn9FucLzf3qJugDQxRAEAj/M= +go.sia.tech/core v0.17.3/go.mod h1:mLJJV1ov732bl3lRvLJrpms3Vkiq4qyBXPZAljSNC4g= +go.sia.tech/coreutils v0.18.1 h1:60n294GVuoS+KM1fdxt46Yc3kUqtxCF1m/sqIPdi8SU= +go.sia.tech/coreutils v0.18.1/go.mod h1:3r8H7wIVMfYfbkT6pIjFJPvsYsrY1rCFLKj6hag6Dl0= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 90842c1ea9bb2022c2272161625efe5dc157a34f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:15:13 +0000 Subject: [PATCH 528/630] build(deps): bump the all-dependencies group with 4 updates Bumps the all-dependencies group with 4 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils), [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/core` from 0.17.3 to 0.17.5 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.17.3...v0.17.5) Updates `go.sia.tech/coreutils` from 0.18.1 to 0.18.4 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.1...v0.18.4) Updates `go.sia.tech/web/walletd` from 0.34.3 to 0.34.4 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.34.3...walletd@0.34.4) Updates `golang.org/x/term` from 0.34.0 to 0.35.0 - [Commits](https://github.com/golang/term/compare/v0.34.0...v0.35.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.17.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.18.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.34.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.35.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index e67d29d..7527aa4 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 - go.sia.tech/core v0.17.3 - go.sia.tech/coreutils v0.18.1 + go.sia.tech/core v0.17.5 + go.sia.tech/coreutils v0.18.4 go.sia.tech/jape v0.14.0 - go.sia.tech/web/walletd v0.34.3 + go.sia.tech/web/walletd v0.34.4 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.34.0 + golang.org/x/term v0.35.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -22,7 +22,7 @@ require ( github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect - go.etcd.io/bbolt v1.4.2 // indirect + go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.2 // indirect @@ -31,7 +31,7 @@ require ( golang.org/x/mod v0.27.0 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.36.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/tools v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index fd5b8d2..e97052f 100644 --- a/go.sum +++ b/go.sum @@ -24,20 +24,20 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= -go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.sia.tech/core v0.17.3 h1:Q+5lR7cPa4nR2MSyLhFxn9FucLzf3qJugDQxRAEAj/M= -go.sia.tech/core v0.17.3/go.mod h1:mLJJV1ov732bl3lRvLJrpms3Vkiq4qyBXPZAljSNC4g= -go.sia.tech/coreutils v0.18.1 h1:60n294GVuoS+KM1fdxt46Yc3kUqtxCF1m/sqIPdi8SU= -go.sia.tech/coreutils v0.18.1/go.mod h1:3r8H7wIVMfYfbkT6pIjFJPvsYsrY1rCFLKj6hag6Dl0= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.sia.tech/core v0.17.5 h1:ZZqdJEar9n41dHCKrru/IByJDQKx3W63ZSBa8CbXGQQ= +go.sia.tech/core v0.17.5/go.mod h1:K63SSC1Wz0mPDj7zqEuzwH2/BiYlyGgnoAG8bD784iI= +go.sia.tech/coreutils v0.18.4 h1:H6wmGz2IXt3xCDO6UWlFTieEBTg74Mn+nxQzni3PRUQ= +go.sia.tech/coreutils v0.18.4/go.mod h1:xspNaTkWH1ytMHuw875FP03KeIPhDSmqDtlbfQgTefc= go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.34.3 h1:t1XxjNFyxFkFgX3lYTq8rTxslL5knSVySMsfuz4JOPY= -go.sia.tech/web/walletd v0.34.3/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.34.4 h1:rKmToymUu9QYQhh2NmVN/X8LjTFcDbnXyJqv4USiBqw= +go.sia.tech/web/walletd v0.34.4/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -56,10 +56,10 @@ golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= From 9a0c049726814fc5c7877958e97bc5e8885ca5f7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 20:32:12 +0000 Subject: [PATCH 529/630] chore: prepare release 2.10.5 --- .changeset/update_core_to_v0171_and_coreutils_to_v0180.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_core_to_v0171_and_coreutils_to_v0180.md diff --git a/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md b/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md deleted file mode 100644 index 5415e96..0000000 --- a/.changeset/update_core_to_v0171_and_coreutils_to_v0180.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core to v0.17.1 and coreutils to v0.18.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9153ae5..37e809b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.10.5 (2025-09-10) + +### Fixes + +- Update core to v0.17.1 and coreutils to v0.18.0 + ## 2.10.4 (2025-07-01) ### Fixes diff --git a/go.mod b/go.mod index 7527aa4..8f71b5e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.4 +module go.sia.tech/walletd/v2 // v2.10.5 go 1.24.3 From 7d7e3b0e27f1fe992a2588a1136cf06fc2dd550f Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 21 Sep 2025 11:56:39 -0700 Subject: [PATCH 530/630] prevent sending funds to the void --- .changeset/prevent_sending_to_void_address.md | 5 + api/api_test.go | 95 ++++++++++++++++++- api/client.go | 13 ++- api/opts.go | 13 +++ api/server.go | 38 ++++++++ 5 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 .changeset/prevent_sending_to_void_address.md create mode 100644 api/opts.go diff --git a/.changeset/prevent_sending_to_void_address.md b/.changeset/prevent_sending_to_void_address.md new file mode 100644 index 0000000..031783b --- /dev/null +++ b/.changeset/prevent_sending_to_void_address.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Prevent sending to void address diff --git a/api/api_test.go b/api/api_test.go index e9a3960..44d10b0 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1543,7 +1543,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { }, }, SiacoinOutputs: []types.SiacoinOutput{ - {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + {Address: frand.Entropy256(), Value: sce.SiacoinOutput.Value}, }, } childSigHash := cs.InputSigHash(childTxn) @@ -1636,7 +1636,7 @@ func TestAddressTPool(t *testing.T) { }, SiacoinOutputs: []types.SiacoinOutput{ { - Address: types.VoidAddress, + Address: frand.Entropy256(), Value: types.Siacoins(25), }, { @@ -1695,7 +1695,7 @@ func TestEphemeralTransactions(t *testing.T) { }, SiacoinOutputs: []types.SiacoinOutput{ { - Address: types.VoidAddress, + Address: frand.Entropy256(), Value: types.Siacoins(50), }, { @@ -1738,7 +1738,7 @@ func TestEphemeralTransactions(t *testing.T) { }, SiacoinOutputs: []types.SiacoinOutput{ { - Address: types.VoidAddress, + Address: frand.Entropy256(), Value: sces[0].SiacoinOutput.Value, }, }, @@ -1831,7 +1831,7 @@ func TestBroadcastRace(t *testing.T) { }, SiacoinOutputs: []types.SiacoinOutput{ { - Address: types.VoidAddress, + Address: frand.Entropy256(), Value: burn, }, { @@ -2194,3 +2194,88 @@ func TestWalletConfirmations(t *testing.T) { cn.MineBlocks(t, types.VoidAddress, 10) assertConfirmations(t, 11) } + +func TestTxPoolAllowVoid(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + cm := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cm, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + sces, basis, err := wc.SiacoinOutputs(0, 100) + if err != nil { + t.Fatal(err) + } else if len(sces) != 1 { + t.Fatalf("expected 1 siacoin output, got %v", len(sces)) + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Address: types.VoidAddress, + Value: types.Siacoins(50), + }, + { + Address: senderAddr, + Value: sces[0].SiacoinElement.SiacoinOutput.Value.Sub(types.Siacoins(50)), + }, + }, + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + sigHash := cs.InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{senderPrivateKey.SignHash(sigHash)} + + // attempt to broadcast without allowing void + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err == nil { + t.Fatal("expected error") + } else if !strings.Contains(err.Error(), "cannot send to void address") { + t.Fatalf("expected error to contain %q, got %v", "cannot send to void address", err) + } + + // broadcast with allowing void + if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}, api.WithAllowVoid()); err != nil { + t.Fatal(err) + } +} diff --git a/api/client.go b/api/client.go index 333e491..eff6c70 100644 --- a/api/client.go +++ b/api/client.go @@ -3,6 +3,7 @@ package api import ( "context" "fmt" + "net/url" "sync" "time" @@ -47,8 +48,16 @@ func (c *Client) State() (resp StateResponse, err error) { } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (resp TxpoolBroadcastResponse, err error) { - err = c.c.POST(context.Background(), "/txpool/broadcast", TxpoolBroadcastRequest{ +func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction, opts ...TxPoolOpt) (resp TxpoolBroadcastResponse, err error) { + v := url.Values{} + for _, opt := range opts { + opt(&v) + } + broadcastUrl := "/txpool/broadcast" + if len(v) > 0 { + broadcastUrl += "?" + v.Encode() + } + err = c.c.POST(context.Background(), broadcastUrl, TxpoolBroadcastRequest{ Basis: basis, Transactions: txns, V2Transactions: v2txns, diff --git a/api/opts.go b/api/opts.go new file mode 100644 index 0000000..59ca5a6 --- /dev/null +++ b/api/opts.go @@ -0,0 +1,13 @@ +package api + +import "net/url" + +// A TxPoolOpt is an option for configuring transaction pool behavior. +type TxPoolOpt func(*url.Values) + +// WithAllowVoid allows transactions that send outputs to the void address +func WithAllowVoid() TxPoolOpt { + return func(v *url.Values) { + v.Set("allowVoid", "true") + } +} diff --git a/api/server.go b/api/server.go index a8d16fa..0339360 100644 --- a/api/server.go +++ b/api/server.go @@ -399,6 +399,44 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { return } + var allowVoid bool + if jc.DecodeForm("allowVoid", &allowVoid) != nil { + return + } + + if !allowVoid { + for _, txn := range tbr.Transactions { + for _, sco := range txn.SiacoinOutputs { + if sco.Address == types.VoidAddress { + jc.Error(errors.New("cannot send to void address"), http.StatusBadRequest) + return + } + } + + for _, sfo := range txn.SiafundOutputs { + if sfo.Address == types.VoidAddress { + jc.Error(errors.New("cannot send to void address"), http.StatusBadRequest) + return + } + } + } + + for _, txn := range tbr.V2Transactions { + for _, sco := range txn.SiacoinOutputs { + if sco.Address == types.VoidAddress { + jc.Error(errors.New("cannot send to void address"), http.StatusBadRequest) + return + } + } + for _, sfo := range txn.SiafundOutputs { + if sfo.Address == types.VoidAddress { + jc.Error(errors.New("cannot send to void address"), http.StatusBadRequest) + return + } + } + } + } + // the transactions are sent back to the client because the // transaction set may have been modified and the transactions // include additional convenience fields when being marshalled From 56bcccaafb1ed1546250f585e1279a54432e97dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 16:21:45 +0000 Subject: [PATCH 531/630] build(deps): bump go.sia.tech/jape in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/jape](https://github.com/SiaFoundation/jape). Updates `go.sia.tech/jape` from 0.14.0 to 0.14.1 - [Release notes](https://github.com/SiaFoundation/jape/releases) - [Changelog](https://github.com/SiaFoundation/jape/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/jape/compare/v0.14.0...v0.14.1) --- updated-dependencies: - dependency-name: go.sia.tech/jape dependency-version: 0.14.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 8f71b5e..cb39099 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.17.5 go.sia.tech/coreutils v0.18.4 - go.sia.tech/jape v0.14.0 + go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.4 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 @@ -27,11 +27,11 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.16.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/mod v0.28.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/tools v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index e97052f..1fea717 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ go.sia.tech/core v0.17.5 h1:ZZqdJEar9n41dHCKrru/IByJDQKx3W63ZSBa8CbXGQQ= go.sia.tech/core v0.17.5/go.mod h1:K63SSC1Wz0mPDj7zqEuzwH2/BiYlyGgnoAG8bD784iI= go.sia.tech/coreutils v0.18.4 h1:H6wmGz2IXt3xCDO6UWlFTieEBTg74Mn+nxQzni3PRUQ= go.sia.tech/coreutils v0.18.4/go.mod h1:xspNaTkWH1ytMHuw875FP03KeIPhDSmqDtlbfQgTefc= -go.sia.tech/jape v0.14.0 h1:hyocTKqvcji+rC1vDE1djINlpErQQVDS6zoLMmxW3Xs= -go.sia.tech/jape v0.14.0/go.mod h1:tONxoKrNr0iQWzBCygwlTkGoGjuEhyVpLGInvGd2mGY= +go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= +go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From dce6309e50b978df29c718ebef76a1f6f27c3a07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 19:28:06 +0000 Subject: [PATCH 532/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/coreutils` from 0.18.4 to 0.18.5 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.4...v0.18.5) Updates `go.sia.tech/web/walletd` from 0.34.4 to 0.34.5 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.34.4...walletd@0.34.5) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.18.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.34.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cb39099..7a88ba1 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.17.5 - go.sia.tech/coreutils v0.18.4 + go.sia.tech/coreutils v0.18.5 go.sia.tech/jape v0.14.1 - go.sia.tech/web/walletd v0.34.4 + go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.35.0 diff --git a/go.sum b/go.sum index 1fea717..8b50566 100644 --- a/go.sum +++ b/go.sum @@ -28,16 +28,16 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.17.5 h1:ZZqdJEar9n41dHCKrru/IByJDQKx3W63ZSBa8CbXGQQ= go.sia.tech/core v0.17.5/go.mod h1:K63SSC1Wz0mPDj7zqEuzwH2/BiYlyGgnoAG8bD784iI= -go.sia.tech/coreutils v0.18.4 h1:H6wmGz2IXt3xCDO6UWlFTieEBTg74Mn+nxQzni3PRUQ= -go.sia.tech/coreutils v0.18.4/go.mod h1:xspNaTkWH1ytMHuw875FP03KeIPhDSmqDtlbfQgTefc= +go.sia.tech/coreutils v0.18.5 h1:lFKgeC2jAV5mkAcDAs13jZmaPiQ/kVxyDEx6DxzqXgU= +go.sia.tech/coreutils v0.18.5/go.mod h1:RbHOI5chN8xU28mBWJbWnsp/jAbDGJjqqgyeENpTvAc= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.34.4 h1:rKmToymUu9QYQhh2NmVN/X8LjTFcDbnXyJqv4USiBqw= -go.sia.tech/web/walletd v0.34.4/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.34.5 h1:IrS5ktvLvUBkNYcC9w56bVnqV6X+PxvxjY9vlsfAH/M= +go.sia.tech/web/walletd v0.34.5/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From 8aaa70079a2abb3ac2e4065224ef2d2c0c7346a0 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 1 Oct 2025 10:04:00 -0700 Subject: [PATCH 533/630] add changesets --- .changeset/prevent_sending_to_void_address.md | 6 ++++-- .changeset/support_v2_final_cut_hardfork.md | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .changeset/support_v2_final_cut_hardfork.md diff --git a/.changeset/prevent_sending_to_void_address.md b/.changeset/prevent_sending_to_void_address.md index 031783b..68bd1cb 100644 --- a/.changeset/prevent_sending_to_void_address.md +++ b/.changeset/prevent_sending_to_void_address.md @@ -1,5 +1,7 @@ --- -default: patch +default: minor --- -# Prevent sending to void address +# Adds an `allowVoid` query parameter to [POST] /txpool/broadcast to guard against accidental burns. + +By default, transactions sent to the void (zero) address are rejected. Integrators must explicitly set allowVoid=true to broadcast to the void. This prevents cases where address parsing errors (e.g. ignoring the error from UnmarshalText and falling back to the zero address) would unintentionally destroy funds. diff --git a/.changeset/support_v2_final_cut_hardfork.md b/.changeset/support_v2_final_cut_hardfork.md new file mode 100644 index 0000000..8f5e8cd --- /dev/null +++ b/.changeset/support_v2_final_cut_hardfork.md @@ -0,0 +1,7 @@ +--- +default: minor +--- + +# Added support for V2 Final Cut Hardfork + +Includes minor improvements to the consensus state and accumulator in preparation for instant syncing and light node support. \ No newline at end of file From 78fff5dc9065ca0800cc017a65d995041d77e11e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 17:04:13 +0000 Subject: [PATCH 534/630] chore: prepare release 2.11.0 --- .changeset/prevent_sending_to_void_address.md | 7 ------- .changeset/support_v2_final_cut_hardfork.md | 7 ------- CHANGELOG.md | 12 ++++++++++++ go.mod | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) delete mode 100644 .changeset/prevent_sending_to_void_address.md delete mode 100644 .changeset/support_v2_final_cut_hardfork.md diff --git a/.changeset/prevent_sending_to_void_address.md b/.changeset/prevent_sending_to_void_address.md deleted file mode 100644 index 68bd1cb..0000000 --- a/.changeset/prevent_sending_to_void_address.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Adds an `allowVoid` query parameter to [POST] /txpool/broadcast to guard against accidental burns. - -By default, transactions sent to the void (zero) address are rejected. Integrators must explicitly set allowVoid=true to broadcast to the void. This prevents cases where address parsing errors (e.g. ignoring the error from UnmarshalText and falling back to the zero address) would unintentionally destroy funds. diff --git a/.changeset/support_v2_final_cut_hardfork.md b/.changeset/support_v2_final_cut_hardfork.md deleted file mode 100644 index 8f5e8cd..0000000 --- a/.changeset/support_v2_final_cut_hardfork.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -default: minor ---- - -# Added support for V2 Final Cut Hardfork - -Includes minor improvements to the consensus state and accumulator in preparation for instant syncing and light node support. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e809b..418d7ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## 2.11.0 (2025-10-01) + +### Features + +#### Adds an `allowVoid` query parameter to [POST] /txpool/broadcast to guard against accidental burns. + +By default, transactions sent to the void (zero) address are rejected. Integrators must explicitly set allowVoid=true to broadcast to the void. This prevents cases where address parsing errors (e.g. ignoring the error from UnmarshalText and falling back to the zero address) would unintentionally destroy funds. + +#### Added support for V2 Final Cut Hardfork + +Includes minor improvements to the consensus state and accumulator in preparation for instant syncing and light node support. + ## 2.10.5 (2025-09-10) ### Fixes diff --git a/go.mod b/go.mod index 7a88ba1..1c11ff6 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.10.5 +module go.sia.tech/walletd/v2 // v2.11.0 go 1.24.3 From 55793df73c1b77b73c0df6d1bb7144a4a7ea3752 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 17:06:20 +0000 Subject: [PATCH 535/630] build(deps): bump github.com/quic-go/quic-go Bumps the go_modules group with 1 update in the / directory: [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go). Updates `github.com/quic-go/quic-go` from 0.54.0 to 0.54.1 - [Release notes](https://github.com/quic-go/quic-go/releases) - [Commits](https://github.com/quic-go/quic-go/compare/v0.54.0...v0.54.1) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-version: 0.54.1 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1c11ff6..51ddd77 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/quic-go v0.54.1 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect diff --git a/go.sum b/go.sum index 8b50566..39f3b45 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= -github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= From f8226eec5b290dff03c96b2104755f3034e191db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 08:31:05 +0000 Subject: [PATCH 536/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/core` from 0.17.5 to 0.18.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.17.5...v0.18.0) Updates `golang.org/x/term` from 0.35.0 to 0.36.0 - [Commits](https://github.com/golang/term/compare/v0.35.0...v0.36.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.18.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.36.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 51ddd77..080f0a6 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 - go.sia.tech/core v0.17.5 + go.sia.tech/core v0.18.0 go.sia.tech/coreutils v0.18.5 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.35.0 + golang.org/x/term v0.36.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -31,7 +31,7 @@ require ( golang.org/x/mod v0.28.0 // indirect golang.org/x/net v0.44.0 // indirect golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect + golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.29.0 // indirect golang.org/x/tools v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 39f3b45..c2b35b5 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.17.5 h1:ZZqdJEar9n41dHCKrru/IByJDQKx3W63ZSBa8CbXGQQ= -go.sia.tech/core v0.17.5/go.mod h1:K63SSC1Wz0mPDj7zqEuzwH2/BiYlyGgnoAG8bD784iI= +go.sia.tech/core v0.18.0 h1:b4xOGdbMmbrqji5y+mHiP0TCzJADQY1okPPapiKuSag= +go.sia.tech/core v0.18.0/go.mod h1:evyyK5QluEbh0QmVAgtttrbUafj3365fZ4MLePvGTM4= go.sia.tech/coreutils v0.18.5 h1:lFKgeC2jAV5mkAcDAs13jZmaPiQ/kVxyDEx6DxzqXgU= go.sia.tech/coreutils v0.18.5/go.mod h1:RbHOI5chN8xU28mBWJbWnsp/jAbDGJjqqgyeENpTvAc= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= @@ -56,10 +56,10 @@ golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= From e0f88557df8084165d47cb0a406174a9dea72c6d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:42:42 +0200 Subject: [PATCH 537/630] changeset --- .changeset/update_core_dependency_from_1175_to_1800.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_core_dependency_from_1175_to_1800.md diff --git a/.changeset/update_core_dependency_from_1175_to_1800.md b/.changeset/update_core_dependency_from_1175_to_1800.md new file mode 100644 index 0000000..5208eb5 --- /dev/null +++ b/.changeset/update_core_dependency_from_1175_to_1800.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core dependency from 1.17.5 to 1.80.0. From e88115a20885afce592d032fec8990b8abfe5f51 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:50:37 +0200 Subject: [PATCH 538/630] fix changeset --- .changeset/update_core_dependency_from_0175_to_0800.md | 5 +++++ .changeset/update_core_dependency_from_1175_to_1800.md | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .changeset/update_core_dependency_from_0175_to_0800.md delete mode 100644 .changeset/update_core_dependency_from_1175_to_1800.md diff --git a/.changeset/update_core_dependency_from_0175_to_0800.md b/.changeset/update_core_dependency_from_0175_to_0800.md new file mode 100644 index 0000000..4cc7735 --- /dev/null +++ b/.changeset/update_core_dependency_from_0175_to_0800.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core dependency from 0.17.5 to 0.80.0. diff --git a/.changeset/update_core_dependency_from_1175_to_1800.md b/.changeset/update_core_dependency_from_1175_to_1800.md deleted file mode 100644 index 5208eb5..0000000 --- a/.changeset/update_core_dependency_from_1175_to_1800.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core dependency from 1.17.5 to 1.80.0. From dea13511671e970eba1b7ba373cde1901382f7c6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Wed, 15 Oct 2025 11:46:22 +0200 Subject: [PATCH 539/630] update changeset --- .changeset/update_core_dependency_from_0175_to_0180.md | 5 +++++ .changeset/update_core_dependency_from_0175_to_0800.md | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .changeset/update_core_dependency_from_0175_to_0180.md delete mode 100644 .changeset/update_core_dependency_from_0175_to_0800.md diff --git a/.changeset/update_core_dependency_from_0175_to_0180.md b/.changeset/update_core_dependency_from_0175_to_0180.md new file mode 100644 index 0000000..65dcb23 --- /dev/null +++ b/.changeset/update_core_dependency_from_0175_to_0180.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core dependency from 0.17.5 to 0.18.0. diff --git a/.changeset/update_core_dependency_from_0175_to_0800.md b/.changeset/update_core_dependency_from_0175_to_0800.md deleted file mode 100644 index 4cc7735..0000000 --- a/.changeset/update_core_dependency_from_0175_to_0800.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core dependency from 0.17.5 to 0.80.0. From b99ea2bd3f151e60509f62bbcd9ba506cfc3b85e Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 15 Oct 2025 08:17:48 -0700 Subject: [PATCH 540/630] update readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e3b8a3..81f05fa 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,8 @@ You can create a custom local testnet by creating a network.json file locally an "hardforkASIC": { "height": 20, "oakTime": 10000000000000, - "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000" + "oakTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "nonceFactor": 1009 }, "hardforkFoundation": { "height": 30, @@ -221,7 +222,8 @@ You can create a custom local testnet by creating a network.json file locally an }, "hardforkV2": { "allowHeight": 112000, - "requireHeight": 114000 + "requireHeight": 114000, + "finalCutHeight": 116000, } }, "genesis": { From df0396949aff76e3835b1cf79ed9eaa362c0e468 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 8 Oct 2025 15:08:24 +0200 Subject: [PATCH 541/630] docs: add openapi spec --- openapi.yml | 1912 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1912 insertions(+) create mode 100644 openapi.yml diff --git a/openapi.yml b/openapi.yml new file mode 100644 index 0000000..d491921 --- /dev/null +++ b/openapi.yml @@ -0,0 +1,1912 @@ +openapi: "3.0.0" +info: + title: Walletd API + description: > + Walletd exposes a REST API for managing wallets, querying blockchain data, and + broadcasting transactions on the Sia network. These endpoints are primarily + intended for integrators that need low-level access to wallet state, UTXOs, + and transaction construction utilities. + version: 2.11.0 + +servers: + - url: http://localhost:9980/api + +tags: + - name: misc + description: Build information and basic health probes. + - name: consensus + description: Access on-chain state, blocks, and historical updates. + - name: syncer + description: Manage P2P peers and broadcast blocks. + - name: txpool + description: Inspect and broadcast transactions. + - name: wallets + description: Create wallets, manage addresses, and build transactions. + - name: addresses + description: Query balances, events, and UTXOs for individual addresses. + - name: batch + description: Batch operations that act on multiple addresses at once. + - name: outputs + description: Inspect individual siacoin and siafund outputs. + - name: events + description: Fetch individual wallet events by ID. + - name: rescan + description: Control background chain rescans. + - name: debug + description: Debug and profiling endpoints. Only available when walletd runs with debug options. + +paths: + /state: + get: + tags: + - misc + summary: Get daemon state + description: Returns build metadata and runtime information about the walletd instance. + operationId: getState + responses: + "200": + description: Current daemon state. + content: + application/json: + schema: + $ref: "#/components/schemas/StateResponse" + + /health: + get: + tags: + - misc + summary: Health probe + description: Returns 200 when walletd is healthy. + operationId: getHealth + responses: + "200": + description: Walletd is healthy. + content: + application/json: + schema: + type: object + nullable: true + description: Always `null` on success. + + /consensus/network: + get: + tags: + - consensus + summary: Network parameters + description: Returns consensus parameters for the active network. + operationId: getConsensusNetwork + responses: + "200": + description: Consensus network parameters. + content: + application/json: + schema: + $ref: "#/components/schemas/ConsensusNetwork" + + /consensus/tip: + get: + tags: + - consensus + summary: Current chain tip index + operationId: getConsensusTip + responses: + "200": + description: Current best chain index. + content: + application/json: + schema: + $ref: "#/components/schemas/ChainIndex" + + /consensus/tipstate: + get: + tags: + - consensus + summary: Current consensus state + operationId: getConsensusTipState + responses: + "200": + description: Consensus state at the current tip. + content: + application/json: + schema: + $ref: "#/components/schemas/ConsensusState" + + /consensus/checkpoint/{id}: + get: + tags: + - consensus + summary: Retrieve checkpoint by block ID or height + description: > + Returns the consensus state and block associated with the supplied block ID + or height. + operationId: getConsensusCheckpoint + parameters: + - name: id + in: path + description: Block ID (hex) or height. + required: true + schema: + oneOf: + - type: string + pattern: "^[0-9a-fA-F]{64}$" + - type: integer + format: uint64 + responses: + "200": + description: Consensus checkpoint for the supplied block. + content: + application/json: + schema: + $ref: "#/components/schemas/ConsensusCheckpointResponse" + "404": + description: Block not found. + + /consensus/blocks/{id}: + get: + tags: + - consensus + summary: Fetch block by ID or height + operationId: getConsensusBlock + parameters: + - name: id + in: path + description: Block ID (hex) or height. + required: true + schema: + oneOf: + - type: string + pattern: "^[0-9a-fA-F]{64}$" + - type: integer + format: uint64 + responses: + "200": + description: Block data. + content: + application/json: + schema: + $ref: "#/components/schemas/Block" + "404": + description: Block not found. + + /consensus/updates/{index}: + get: + tags: + - consensus + summary: Stream consensus updates since an index + description: > + Returns the set of reverted and applied updates needed to move from the supplied + chain index to the current tip. + operationId: getConsensusUpdates + parameters: + - name: index + in: path + required: true + description: Starting chain index. + schema: + $ref: "#/components/schemas/ChainIndex" + - name: limit + in: query + description: Maximum number of updates to return (default 10, max 100). + required: false + schema: + type: integer + minimum: 1 + maximum: 100 + responses: + "200": + description: Consensus updates needed to synchronize the supplied index. + content: + application/json: + schema: + $ref: "#/components/schemas/ConsensusUpdatesResponse" + + /consensus/index/{height}: + get: + tags: + - consensus + summary: Resolve a chain index by height + operationId: getConsensusIndexByHeight + parameters: + - name: height + in: path + required: true + description: Block height to resolve. + schema: + type: integer + format: uint64 + responses: + "200": + description: Chain index at the requested height. + content: + application/json: + schema: + $ref: "#/components/schemas/ChainIndex" + "404": + description: Height not found. + + /syncer/connect: + post: + tags: + - syncer + summary: Connect to a peer + description: Attempts to connect to the supplied peer address. + operationId: postSyncerConnect + requestBody: + required: true + content: + application/json: + schema: + type: string + description: Peer address in `host:port` form. + responses: + "200": + description: Connection initiated. + "500": + description: Failed to connect to peer. + + /syncer/peers: + get: + tags: + - syncer + summary: List connected peers + operationId: getSyncerPeers + responses: + "200": + description: Connected peers and their metadata. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/GatewayPeer" + + /syncer/broadcast/block: + post: + tags: + - syncer + summary: Broadcast a block to peers + description: > + Adds the supplied block to the local chain (if valid) and broadcasts it to peers. + When broadcasting a v2 block, walletd sends the block outline. + operationId: postSyncerBroadcastBlock + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Block" + responses: + "200": + description: Block accepted and broadcast. + "400": + description: Submitted block was invalid. + + /txpool/transactions: + get: + tags: + - txpool + summary: List unconfirmed transactions + operationId: getTxpoolTransactions + responses: + "200": + description: Current contents of the transaction pools. + content: + application/json: + schema: + $ref: "#/components/schemas/TxpoolTransactionsResponse" + + /txpool/transactions/v2/basis: + post: + tags: + - txpool + summary: Rebase v2 transaction set + description: > + Adjusts a v2 transaction set from one basis chain index to another. + Use this to keep proofs and references valid when the chain advances. + operationId: postTxpoolV2TransactionsBasis + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TxpoolUpdateV2TransactionsRequest" + responses: + "200": + description: Updated transaction set anchored at the target basis. + content: + application/json: + schema: + $ref: "#/components/schemas/TxpoolUpdateV2TransactionsResponse" + "400": + description: Invalid transaction set. + + /txpool/fee: + get: + tags: + - txpool + summary: Recommended miner fee + operationId: getTxpoolFee + responses: + "200": + description: Fee rate recommended for prompt confirmation. + content: + application/json: + schema: + $ref: "#/components/schemas/Currency" + + /txpool/parents: + post: + tags: + - txpool + summary: Lookup unconfirmed parents + description: Returns unconfirmed parent transactions required to validate the supplied transaction. + operationId: postTxpoolParents + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Transaction" + responses: + "200": + description: Parent transactions that must be included ahead of the supplied transaction. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Transaction" + + /txpool/broadcast: + post: + tags: + - txpool + summary: Broadcast transaction set + description: > + Broadcasts v1 or v2 transaction sets. Walletd may augment the submitted transactions + with missing parents or overwritten proofs before sending them to peers. + operationId: postTxpoolBroadcast + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TxpoolBroadcastRequest" + responses: + "200": + description: Transactions accepted by the pool and broadcast to peers. + content: + application/json: + schema: + $ref: "#/components/schemas/TxpoolBroadcastResponse" + "400": + description: Invalid transaction set. + + /txpool/events: + get: + tags: + - txpool + summary: List unconfirmed wallet events + description: Returns unconfirmed wallet events backed by transactions in the pool. + operationId: getTxpoolEvents + responses: + "200": + description: Pending wallet events derived from the transaction pool. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + + /addresses/{addr}/balance: + get: + tags: + - addresses + summary: Address balance + operationId: getAddressBalance + parameters: + - $ref: "#/components/parameters/AddressParam" + responses: + "200": + description: Aggregated siacoin and siafund balance for the address. + content: + application/json: + schema: + $ref: "#/components/schemas/Balance" + + /addresses/{addr}/events: + get: + tags: + - addresses + summary: Address events + description: Returns confirmed events affecting the supplied address. + operationId: getAddressEvents + parameters: + - $ref: "#/components/parameters/AddressParam" + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamLarge" + responses: + "200": + description: Events involving the address. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + + /addresses/{addr}/events/unconfirmed: + get: + tags: + - addresses + summary: Address unconfirmed events + description: Returns unconfirmed events generated by unconfirmed transactions that reference the address. + operationId: getAddressEventsUnconfirmed + parameters: + - $ref: "#/components/parameters/AddressParam" + responses: + "200": + description: Unconfirmed events touching the address. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + + /addresses/{addr}/outputs/siacoin: + get: + tags: + - addresses + summary: Address siacoin outputs + operationId: getAddressSiacoinOutputs + parameters: + - $ref: "#/components/parameters/AddressParam" + - name: tpool + in: query + description: Include unconfirmed txpool outputs. + schema: + type: boolean + default: false + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamLarge" + responses: + "200": + description: Unspent siacoin outputs owned by the address. + content: + application/json: + schema: + $ref: "#/components/schemas/AddressSiacoinElementsResponse" + + /addresses/{addr}/outputs/siafund: + get: + tags: + - addresses + summary: Address siafund outputs + operationId: getAddressSiafundOutputs + parameters: + - $ref: "#/components/parameters/AddressParam" + - name: tpool + in: query + description: Include unconfirmed txpool outputs. + schema: + type: boolean + default: false + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamLarge" + responses: + "200": + description: Unspent siafund outputs owned by the address. + content: + application/json: + schema: + $ref: "#/components/schemas/AddressSiafundElementsResponse" + + /batch/addresses/balance: + post: + tags: + - batch + summary: Batch balance lookup + description: Returns the aggregate balance for all supplied addresses. + operationId: postBatchAddressesBalance + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BatchAddressesRequest" + responses: + "200": + description: Combined balance across addresses. + content: + application/json: + schema: + $ref: "#/components/schemas/Balance" + + /batch/addresses/events: + post: + tags: + - batch + summary: Batch events lookup + operationId: postBatchAddressesEvents + parameters: + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamSmall" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BatchAddressesRequest" + responses: + "200": + description: Events touching any of the supplied addresses. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + + /batch/addresses/outputs/siacoin: + post: + tags: + - batch + summary: Batch siacoin outputs lookup + operationId: postBatchAddressesSiacoinOutputs + parameters: + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamSmall" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BatchAddressesRequest" + responses: + "200": + description: Unspent siacoin outputs for the supplied addresses. + content: + application/json: + schema: + $ref: "#/components/schemas/AddressSiacoinElementsResponse" + + /batch/addresses/outputs/siafund: + post: + tags: + - batch + summary: Batch siafund outputs lookup + operationId: postBatchAddressesSiafundOutputs + parameters: + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamSmall" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BatchAddressesRequest" + responses: + "200": + description: Unspent siafund outputs for the supplied addresses. + content: + application/json: + schema: + $ref: "#/components/schemas/AddressSiafundElementsResponse" + + /outputs/siacoin/{id}: + get: + tags: + - outputs + summary: Fetch siacoin output + operationId: getSiacoinOutput + parameters: + - $ref: "#/components/parameters/SiacoinOutputIDParam" + responses: + "200": + description: Siacoin element details. + content: + application/json: + schema: + $ref: "#/components/schemas/SiacoinElement" + + /outputs/siacoin/{id}/spent: + get: + tags: + - outputs + summary: Check siacoin output spent status + operationId: getSiacoinOutputSpent + parameters: + - $ref: "#/components/parameters/SiacoinOutputIDParam" + responses: + "200": + description: Spent status and optional spend event. + content: + application/json: + schema: + $ref: "#/components/schemas/ElementSpentResponse" + + /outputs/siafund/{id}: + get: + tags: + - outputs + summary: Fetch siafund output + operationId: getSiafundOutput + parameters: + - $ref: "#/components/parameters/SiafundOutputIDParam" + responses: + "200": + description: Siafund element details. + content: + application/json: + schema: + $ref: "#/components/schemas/SiafundElement" + + /outputs/siafund/{id}/spent: + get: + tags: + - outputs + summary: Check siafund output spent status + operationId: getSiafundOutputSpent + parameters: + - $ref: "#/components/parameters/SiafundOutputIDParam" + responses: + "200": + description: Spent status and optional spend event. + content: + application/json: + schema: + $ref: "#/components/schemas/ElementSpentResponse" + + /check/addresses: + post: + tags: + - addresses + summary: Check address membership + description: > + Returns whether any supplied addresses are currently tracked by walletd. + In personal indexing mode, only registered wallet addresses are recognized. + operationId: postCheckAddresses + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CheckAddressesRequest" + responses: + "200": + description: Address membership check result. + content: + application/json: + schema: + $ref: "#/components/schemas/CheckAddressesResponse" + + /events/{id}: + get: + tags: + - outputs + summary: Fetch event by ID + operationId: getEvent + parameters: + - name: id + in: path + required: true + description: Event ID (hash). + schema: + $ref: "#/components/schemas/Hash256" + responses: + "200": + description: Event information. + content: + application/json: + schema: + $ref: "#/components/schemas/Event" + "404": + description: Event not found. + + /rescan: + get: + tags: + - rescan + summary: Get rescan status + operationId: getRescan + responses: + "200": + description: Current rescan progress. + content: + application/json: + schema: + $ref: "#/components/schemas/RescanResponse" + post: + tags: + - rescan + summary: Start a rescan + description: > + Initiates a background rescan from the supplied height. Omitting the height + or providing zero triggers a full rescan from genesis. + operationId: postRescan + requestBody: + required: true + content: + application/json: + schema: + type: integer + format: uint64 + description: > + Height to begin rescanning from. Set to 0 for a full rescan. + responses: + "200": + description: Rescan started. + "409": + description: A rescan is already running. + + /wallets: + get: + tags: + - wallets + summary: List wallets + operationId: getWallets + responses: + "200": + description: Wallets registered with walletd. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Wallet" + post: + tags: + - wallets + summary: Create wallet + operationId: postWallet + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletUpdateRequest" + responses: + "200": + description: Wallet created. + content: + application/json: + schema: + $ref: "#/components/schemas/Wallet" + + /wallets/{id}: + post: + tags: + - wallets + summary: Update wallet metadata + operationId: postWalletByID + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletUpdateRequest" + responses: + "200": + description: Wallet updated. + content: + application/json: + schema: + $ref: "#/components/schemas/Wallet" + "404": + description: Wallet not found. + delete: + tags: + - wallets + summary: Delete wallet + operationId: deleteWalletByID + parameters: + - $ref: "#/components/parameters/WalletIDParam" + responses: + "200": + description: Wallet removed. + "404": + description: Wallet not found. + + /wallets/{id}/addresses: + put: + tags: + - wallets + summary: Add address to wallet + operationId: putWalletAddress + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletAddress" + responses: + "200": + description: Address added to wallet. + get: + tags: + - wallets + summary: List wallet addresses + operationId: getWalletAddresses + parameters: + - $ref: "#/components/parameters/WalletIDParam" + responses: + "200": + description: Addresses assigned to the wallet. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/WalletAddress" + + /wallets/{id}/batch/addresses: + put: + tags: + - wallets + summary: Add multiple addresses + operationId: putWalletBatchAddresses + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + type: array + maxItems: 10000 + items: + $ref: "#/components/schemas/WalletAddress" + responses: + "200": + description: Addresses added to wallet. + + /wallets/{id}/addresses/{addr}: + delete: + tags: + - wallets + summary: Remove wallet address + operationId: deleteWalletAddress + parameters: + - $ref: "#/components/parameters/WalletIDParam" + - $ref: "#/components/parameters/AddressParam" + responses: + "200": + description: Address removed. + "404": + description: Wallet or address not found. + + /wallets/{id}/balance: + get: + tags: + - wallets + summary: Wallet balance + operationId: getWalletBalance + parameters: + - $ref: "#/components/parameters/WalletIDParam" + responses: + "200": + description: Wallet balance with immature breakdown. + content: + application/json: + schema: + $ref: "#/components/schemas/Balance" + "404": + description: Wallet not found. + + /wallets/{id}/events: + get: + tags: + - events + summary: Wallet events + operationId: getWalletEvents + parameters: + - $ref: "#/components/parameters/WalletIDParam" + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamDefault" + responses: + "200": + description: Events relevant to the wallet. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + "404": + description: Wallet not found. + + /wallets/{id}/events/unconfirmed: + get: + tags: + - wallets + summary: Wallet unconfirmed events + operationId: getWalletEventsUnconfirmed + parameters: + - $ref: "#/components/parameters/WalletIDParam" + responses: + "200": + description: Unconfirmed events referencing the wallet. + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Event" + "404": + description: Wallet not found. + + /wallets/{id}/outputs/siacoin: + get: + tags: + - wallets + summary: Wallet siacoin outputs + operationId: getWalletSiacoinOutputs + parameters: + - $ref: "#/components/parameters/WalletIDParam" + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamLarge" + responses: + "200": + description: Unspent siacoin elements tracked by the wallet. + content: + application/json: + schema: + $ref: "#/components/schemas/UnspentSiacoinElementsResponse" + + /wallets/{id}/outputs/siafund: + get: + tags: + - wallets + summary: Wallet siafund outputs + operationId: getWalletSiafundOutputs + parameters: + - $ref: "#/components/parameters/WalletIDParam" + - $ref: "#/components/parameters/OffsetParam" + - $ref: "#/components/parameters/LimitParamLarge" + responses: + "200": + description: Unspent siafund elements tracked by the wallet. + content: + application/json: + schema: + $ref: "#/components/schemas/UnspentSiafundElementsResponse" + + /wallets/{id}/reserve: + post: + tags: + - wallets + summary: Reserve UTXOs + operationId: postWalletReserve + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletReserveRequest" + responses: + "200": + description: Outputs reserved. + + /wallets/{id}/release: + post: + tags: + - wallets + summary: Release reserved UTXOs + operationId: postWalletRelease + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletReleaseRequest" + responses: + "200": + description: Outputs released. + + /wallets/{id}/fund: + post: + tags: + - wallets + summary: Fund v1 transaction + operationId: postWalletFund + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletFundRequest" + responses: + "200": + description: Transaction funded and inputs listed for signing. + content: + application/json: + schema: + $ref: "#/components/schemas/WalletFundResponse" + + /wallets/{id}/fundsf: + post: + tags: + - wallets + summary: Fund v1 transaction with siafunds + operationId: postWalletFundSiafunds + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletFundSFRequest" + responses: + "200": + description: Transaction funded with siafund inputs. + content: + application/json: + schema: + $ref: "#/components/schemas/WalletFundResponse" + + /wallets/{id}/construct/transaction: + post: + tags: + - wallets + summary: Construct v1 transaction + description: > + Selects wallet-managed UTXOs, adds change outputs as needed, and returns a + fully-formed v1 transaction with signature placeholders. + operationId: postWalletConstructTransaction + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletConstructRequest" + responses: + "200": + description: Constructed v1 transaction and metadata. + content: + application/json: + schema: + $ref: "#/components/schemas/WalletConstructResponse" + + /wallets/{id}/construct/v2/transaction: + post: + tags: + - wallets + summary: Construct v2 transaction + description: > + Builds a v2 transaction using wallet-managed UTXOs and optional outputs. + Walletd fills in satisfied spend policies and updates proofs to the latest basis. + operationId: postWalletConstructV2Transaction + parameters: + - $ref: "#/components/parameters/WalletIDParam" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/WalletConstructRequest" + responses: + "200": + description: Constructed v2 transaction and metadata. + content: + application/json: + schema: + $ref: "#/components/schemas/WalletConstructV2Response" + + /debug/mine: + post: + tags: + - debug + summary: Mine blocks locally + description: > + Mines the requested number of blocks to the supplied address using the integrated CPU miner. + Only available when debug endpoints are enabled. + operationId: postDebugMine + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/DebugMineRequest" + responses: + "200": + description: Requested blocks mined or mining stopped early. + + /debug/pprof/{handler}: + get: + tags: + - debug + summary: Get pprof profiling data + description: > + Returns profiling data for the specified handler. This is useful for + debugging and performance analysis. + operationId: getPprofData + parameters: + - name: handler + in: path + required: true + schema: + type: string + enum: + [ + allocs, + block, + cmdline, + goroutine, + heap, + mutex, + profile, + threadcreate, + trace, + ] + responses: + "200": + description: Profiling data in binary format + content: + application/octet-stream: + schema: + type: string + format: binary + +components: + schemas: + Hash256: + type: string + pattern: ^[0-9a-fA-F]{64}$ + description: A 256-bit blake2b hash + + BlockHeight: + type: integer + format: uint64 + description: The height of a block + example: 92813 + + BlockID: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: A unique identifier for a block + + Currency: + type: string + pattern: "^\\d+$" + maxLength: 39 # fits 2^128 - 1 + description: An unsigned amount of Hastings, the smallest unit of currency in Sia. 1 Siacoin (SC) equals 10^24 Hastings (H). + + Address: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: The hash of a set of UnlockConditions + + TransactionID: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: Unique identifier for a transaction. + + Event: + type: object + description: A transaction or other event that affects the wallet including miner payouts, siafund claims, and file contract payouts. + properties: + id: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: The event's ID + index: + allOf: + - $ref: "#/components/schemas/ChainIndex" + - description: Information about the block that triggered the creation of this event + confirmations: + type: integer + format: uint64 + description: The number of blocks on top of the block that triggered the creation of this event + type: + type: string + enum: + - miner + - foundation + - siafundClaim + - v1Transaction + - v1ContractResolution + - v2Transaction + - v2ContractResolution + description: The type of the event + data: + type: object + maturityHeight: + allOf: + - $ref: "#/components/schemas/BlockHeight" + - description: The block height at which the payout matures. + timestamp: + type: string + format: date-time + description: The time the event was created + relevant: + type: array + items: + $ref: "#/components/schemas/Address" + + StateResponse: + type: object + properties: + version: + type: string + description: Walletd semantic version. + commit: + type: string + description: Git commit hash walletd was built from. + os: + type: string + description: Operating system of the running binary. + buildTime: + type: string + format: date-time + description: Build timestamp embedded in the binary. + startTime: + type: string + format: date-time + description: Time when walletd started. + indexMode: + $ref: "#/components/schemas/IndexMode" + required: [version, commit, os, buildTime, startTime, indexMode] + + IndexMode: + type: string + description: Wallet index mode that determines how chain data is tracked. + enum: [personal, full, none] + + ConsensusNetwork: + type: object + description: JSON encoding of `go.sia.tech/core/consensus.Network`. + additionalProperties: true + + ChainIndex: + type: object + properties: + height: + allOf: + - $ref: "#/components/schemas/BlockHeight" + - description: The height of the block in the blockchain + id: + allOf: + - $ref: "#/components/schemas/BlockID" + - description: The ID of the block + + ConsensusState: + type: object + description: JSON encoding of `go.sia.tech/core/consensus.State`. + additionalProperties: true + + Block: + type: object + description: JSON encoding of `go.sia.tech/core/types.Block`. + additionalProperties: true + + ConsensusCheckpointResponse: + type: object + properties: + state: + $ref: "#/components/schemas/ConsensusState" + block: + $ref: "#/components/schemas/Block" + required: [state, block] + + GatewayPeer: + type: object + properties: + address: + type: string + description: Peer network address. + inbound: + type: boolean + description: Whether the connection was inbound. + version: + type: string + description: Peer-reported version. + firstSeen: + type: string + format: date-time + nullable: true + description: Time the peer was first observed, if known. + connectedSince: + type: string + format: date-time + nullable: true + description: Time the peer connection was established. + syncedBlocks: + type: integer + format: uint64 + nullable: true + description: Number of blocks synced during the last session. + syncDuration: + type: string + nullable: true + description: Duration of the last sync, encoded as a Go duration string. + required: [address, inbound, version] + + TxpoolBroadcastRequest: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + transactions: + type: array + items: + $ref: "#/components/schemas/Transaction" + description: v1 transactions to broadcast. + v2transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + description: v2 transactions to broadcast. + description: > + At least one of `transactions` or `v2transactions` must be supplied. When broadcasting + v2 transactions, `basis` should match the chain index the proofs are anchored to. + + TxpoolBroadcastResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + transactions: + type: array + items: + $ref: "#/components/schemas/Transaction" + v2transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + required: [basis] + + TxpoolTransactionsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + transactions: + type: array + items: + $ref: "#/components/schemas/Transaction" + v2transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + required: [basis, transactions, v2transactions] + + TxpoolUpdateV2TransactionsRequest: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + target: + $ref: "#/components/schemas/ChainIndex" + transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + required: [basis, target, transactions] + + TxpoolUpdateV2TransactionsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + required: [basis, transactions] + + Transaction: + type: object + description: JSON encoding of `go.sia.tech/core/types.Transaction`, including derived IDs. + additionalProperties: true + + V2Transaction: + type: object + description: JSON encoding of `go.sia.tech/core/types.V2Transaction`, including derived IDs and proofs. + additionalProperties: true + + Balance: + type: object + properties: + siacoins: + $ref: "#/components/schemas/Currency" + immatureSiacoins: + $ref: "#/components/schemas/Currency" + siafunds: + type: integer + format: uint64 + required: [siacoins, immatureSiacoins, siafunds] + + Wallet: + type: object + properties: + id: + type: integer + format: int64 + name: + type: string + description: + type: string + dateCreated: + type: string + format: date-time + lastUpdated: + type: string + format: date-time + metadata: + type: object + nullable: true + additionalProperties: true + required: [id, name, description, dateCreated, lastUpdated] + + WalletUpdateRequest: + type: object + properties: + name: + type: string + description: + type: string + metadata: + type: object + nullable: true + additionalProperties: true + + WalletAddress: + type: object + properties: + address: + $ref: "#/components/schemas/Address" + description: + type: string + spendPolicy: + $ref: "#/components/schemas/SpendPolicy" + metadata: + type: object + nullable: true + additionalProperties: true + required: [address, description] + + SpendPolicy: + type: object + description: Polymorphic spend policy serialized by `go.sia.tech/core/types.SpendPolicy`. + properties: + type: + type: string + description: Policy discriminator. + enum: [above, after, pk, h, thresh, opaque, uc] + policy: + description: Policy-specific payload; structure depends on the `type`. + nullable: true + required: [type, policy] + additionalProperties: true + + WalletReserveRequest: + type: object + properties: + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/Hash256" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/Hash256" + + WalletReleaseRequest: + type: object + properties: + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/Hash256" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/Hash256" + + WalletFundRequest: + type: object + properties: + transaction: + $ref: "#/components/schemas/Transaction" + amount: + $ref: "#/components/schemas/Currency" + changeAddress: + $ref: "#/components/schemas/Address" + required: [transaction, amount] + + WalletFundSFRequest: + type: object + properties: + transaction: + $ref: "#/components/schemas/Transaction" + amount: + type: integer + format: uint64 + changeAddress: + $ref: "#/components/schemas/Address" + claimAddress: + $ref: "#/components/schemas/Address" + required: [transaction, amount] + + WalletFundResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + transaction: + $ref: "#/components/schemas/Transaction" + toSign: + type: array + items: + $ref: "#/components/schemas/Hash256" + dependsOn: + type: array + items: + $ref: "#/components/schemas/Transaction" + required: [basis, transaction, toSign, dependsOn] + + WalletConstructRequest: + type: object + properties: + siacoins: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + siafunds: + type: array + items: + $ref: "#/components/schemas/SiafundOutput" + changeAddress: + $ref: "#/components/schemas/Address" + required: [changeAddress] + + WalletConstructResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + id: + $ref: "#/components/schemas/TransactionID" + transaction: + $ref: "#/components/schemas/Transaction" + estimatedFee: + $ref: "#/components/schemas/Currency" + required: [basis, id, transaction, estimatedFee] + + WalletConstructV2Response: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + id: + $ref: "#/components/schemas/TransactionID" + transaction: + $ref: "#/components/schemas/V2Transaction" + estimatedFee: + $ref: "#/components/schemas/Currency" + required: [basis, id, transaction, estimatedFee] + + UnspentSiacoinElementsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + outputs: + type: array + items: + $ref: "#/components/schemas/UnspentSiacoinElement" + required: [basis, outputs] + + UnspentSiafundElementsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + outputs: + type: array + items: + $ref: "#/components/schemas/UnspentSiafundElement" + required: [basis, outputs] + + AddressSiacoinElementsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + outputs: + type: array + items: + $ref: "#/components/schemas/UnspentSiacoinElement" + required: [basis, outputs] + + AddressSiafundElementsResponse: + type: object + properties: + basis: + $ref: "#/components/schemas/ChainIndex" + outputs: + type: array + items: + $ref: "#/components/schemas/UnspentSiafundElement" + required: [basis, outputs] + + UnspentSiacoinElement: + type: object + properties: + confirmations: + type: integer + format: uint64 + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + siacoinOutput: + $ref: "#/components/schemas/SiacoinOutput" + maturityHeight: + type: integer + format: uint64 + required: [confirmations, id, stateElement, siacoinOutput, maturityHeight] + description: Combines a siacoin element with confirmation count. + + UnspentSiafundElement: + type: object + properties: + confirmations: + type: integer + format: uint64 + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + siafundOutput: + $ref: "#/components/schemas/SiafundOutput" + claimStart: + $ref: "#/components/schemas/Currency" + required: [confirmations, id, stateElement, siafundOutput, claimStart] + description: Combines a siafund element with confirmation count. + + SiacoinElement: + type: object + properties: + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + siacoinOutput: + $ref: "#/components/schemas/SiacoinOutput" + maturityHeight: + type: integer + format: uint64 + required: [id, stateElement, siacoinOutput, maturityHeight] + + SiafundElement: + type: object + properties: + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + siafundOutput: + $ref: "#/components/schemas/SiafundOutput" + claimStart: + $ref: "#/components/schemas/Currency" + required: [id, stateElement, siafundOutput, claimStart] + + StateElement: + type: object + properties: + leafIndex: + type: integer + format: uint64 + merkleProof: + type: array + items: + $ref: "#/components/schemas/Hash256" + required: [leafIndex] + + SiacoinOutput: + type: object + properties: + value: + $ref: "#/components/schemas/Currency" + address: + $ref: "#/components/schemas/Address" + required: [value, address] + + SiafundOutput: + type: object + properties: + value: + type: integer + format: uint64 + address: + $ref: "#/components/schemas/Address" + required: [value, address] + + ElementSpentResponse: + type: object + properties: + spent: + type: boolean + event: + $ref: "#/components/schemas/Event" + required: [spent] + + CheckAddressesRequest: + type: object + properties: + addresses: + type: array + items: + $ref: "#/components/schemas/Address" + minItems: 1 + maxItems: 1000 + required: [addresses] + + CheckAddressesResponse: + type: object + properties: + known: + type: boolean + required: [known] + + BatchAddressesRequest: + type: object + properties: + addresses: + type: array + items: + $ref: "#/components/schemas/Address" + minItems: 1 + maxItems: 1000 + required: [addresses] + + RescanResponse: + type: object + properties: + startIndex: + $ref: "#/components/schemas/ChainIndex" + index: + $ref: "#/components/schemas/ChainIndex" + startTime: + type: string + format: date-time + error: + type: string + nullable: true + required: [startIndex, index, startTime] + + ConsensusUpdatesResponse: + type: object + properties: + applied: + type: array + items: + $ref: "#/components/schemas/ApplyUpdate" + reverted: + type: array + items: + $ref: "#/components/schemas/RevertUpdate" + required: [applied, reverted] + + ApplyUpdate: + type: object + properties: + update: + type: object + description: JSON encoding of `go.sia.tech/core/consensus.ApplyUpdate`. + additionalProperties: true + state: + $ref: "#/components/schemas/ConsensusState" + block: + $ref: "#/components/schemas/Block" + required: [update, state, block] + + RevertUpdate: + type: object + properties: + update: + type: object + description: JSON encoding of `go.sia.tech/core/consensus.RevertUpdate`. + additionalProperties: true + state: + $ref: "#/components/schemas/ConsensusState" + block: + $ref: "#/components/schemas/Block" + required: [update, state, block] + + DebugMineRequest: + type: object + properties: + blocks: + type: integer + minimum: 1 + description: Number of blocks to mine. + address: + $ref: "#/components/schemas/Address" + required: [blocks, address] + + parameters: + AddressParam: + name: addr + in: path + required: true + description: Sia address (unlock hash). + schema: + $ref: "#/components/schemas/Address" + WalletIDParam: + name: id + in: path + required: true + description: Wallet identifier. + schema: + type: integer + format: int64 + OffsetParam: + name: offset + in: query + required: false + description: Number of items to skip. + schema: + type: integer + minimum: 0 + default: 0 + LimitParamDefault: + name: limit + in: query + required: false + description: Maximum number of items to return (default 500). + schema: + type: integer + minimum: 1 + maximum: 500 + default: 500 + LimitParamLarge: + name: limit + in: query + required: false + description: Maximum number of items to return (default 1000). + schema: + type: integer + minimum: 1 + maximum: 1000 + default: 1000 + LimitParamSmall: + name: limit + in: query + required: false + description: Maximum number of items to return (default 100). + schema: + type: integer + minimum: 1 + maximum: 100 + default: 100 + SiacoinOutputIDParam: + name: id + in: path + required: true + description: Siacoin output ID (hex). + schema: + $ref: "#/components/schemas/Hash256" + SiafundOutputIDParam: + name: id + in: path + required: true + description: Siafund output ID (hex). + schema: + $ref: "#/components/schemas/Hash256" From c3d9b115e6e63ff594023036ff4b4f2bdf041b7b Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 8 Oct 2025 15:13:34 +0200 Subject: [PATCH 542/630] github: add workflows --- .github/workflows/openapi-publish.yml | 21 +++++++++++++++++++++ .github/workflows/openapi-sync.yml | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/openapi-publish.yml create mode 100644 .github/workflows/openapi-sync.yml diff --git a/.github/workflows/openapi-publish.yml b/.github/workflows/openapi-publish.yml new file mode 100644 index 0000000..fadcc4c --- /dev/null +++ b/.github/workflows/openapi-publish.yml @@ -0,0 +1,21 @@ +name: Publish OpenAPI to Scalar Registry + +permissions: + contents: read + +on: + push: + branches: [ master ] + paths: + - "openapi.yml" + workflow_dispatch: + +jobs: + publish: + uses: SiaFoundation/workflows/.github/workflows/publish-openapi.yml@master + with: + slug: walletd + spec_path: openapi.yml + docs_slug: sia + secrets: + SCALAR_API_KEY: ${{ secrets.SCALAR_API_KEY }} diff --git a/.github/workflows/openapi-sync.yml b/.github/workflows/openapi-sync.yml new file mode 100644 index 0000000..7214783 --- /dev/null +++ b/.github/workflows/openapi-sync.yml @@ -0,0 +1,16 @@ +name: Sync OpenAPI Versions + +permissions: + contents: read + pull-requests: write + +on: + release: + types: [published, edited] + workflow_dispatch: + +jobs: + sync: + uses: SiaFoundation/workflows/.github/workflows/sync-openapi-version.yml@master + with: + spec_path: openapi.yml From 78f5124301a3dd03864357e8d48b2170f1efd189 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 8 Oct 2025 15:45:26 +0200 Subject: [PATCH 543/630] docs: filled generic JSON encoding placeholders --- openapi.yml | 974 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 958 insertions(+), 16 deletions(-) diff --git a/openapi.yml b/openapi.yml index d491921..4978ee9 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1191,11 +1191,36 @@ components: - $ref: "#/components/schemas/Hash256" - description: The hash of a set of UnlockConditions + PublicKey: + type: string + pattern: "^ed25519:[0-9a-fA-F]{64}$" + description: Ed25519 public key encoded with algorithm prefix. + TransactionID: allOf: - $ref: "#/components/schemas/Hash256" - description: Unique identifier for a transaction. + Signature: + type: string + pattern: "^[0-9a-fA-F]{128}$" + description: Hex-encoded Ed25519 signature. + + SiacoinOutputID: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: Identifier for a siacoin output. + + SiafundOutputID: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: Identifier for a siafund output. + + FileContractID: + allOf: + - $ref: "#/components/schemas/Hash256" + - description: Identifier for a file contract. + Event: type: object description: A transaction or other event that affects the wallet including miner payouts, siafund claims, and file contract payouts. @@ -1269,8 +1294,117 @@ components: ConsensusNetwork: type: object - description: JSON encoding of `go.sia.tech/core/consensus.Network`. - additionalProperties: true + description: Parameters that define network-wide consensus constants. + properties: + name: + type: string + description: Human friendly name of the network (e.g. mainnet, zen, anagami). + initialCoinbase: + $ref: "#/components/schemas/Currency" + minimumCoinbase: + $ref: "#/components/schemas/Currency" + initialTarget: + $ref: "#/components/schemas/BlockID" + blockInterval: + type: string + description: Target block interval expressed as a Go duration string (e.g. "600s"). + maturityDelay: + type: integer + format: uint64 + description: Number of confirmations before miner payouts mature. + hardforkDevAddr: + type: object + properties: + height: + type: integer + format: uint64 + oldAddress: + $ref: "#/components/schemas/Address" + newAddress: + $ref: "#/components/schemas/Address" + required: [height, oldAddress, newAddress] + hardforkTax: + type: object + properties: + height: + type: integer + format: uint64 + required: [height] + hardforkStorageProof: + type: object + properties: + height: + type: integer + format: uint64 + required: [height] + hardforkOak: + type: object + properties: + height: + type: integer + format: uint64 + fixHeight: + type: integer + format: uint64 + genesisTimestamp: + type: string + format: date-time + required: [height, fixHeight, genesisTimestamp] + hardforkASIC: + type: object + properties: + height: + type: integer + format: uint64 + oakTime: + type: string + description: Duration encoded as a Go duration string. + oakTarget: + $ref: "#/components/schemas/BlockID" + nonceFactor: + type: integer + format: uint64 + required: [height, oakTime, oakTarget, nonceFactor] + hardforkFoundation: + type: object + properties: + height: + type: integer + format: uint64 + primaryAddress: + $ref: "#/components/schemas/Address" + failsafeAddress: + $ref: "#/components/schemas/Address" + required: [height, primaryAddress, failsafeAddress] + hardforkV2: + type: object + properties: + allowHeight: + type: integer + format: uint64 + requireHeight: + type: integer + format: uint64 + finalCutHeight: + type: integer + format: uint64 + required: [allowHeight, requireHeight, finalCutHeight] + required: + [ + name, + initialCoinbase, + minimumCoinbase, + initialTarget, + blockInterval, + maturityDelay, + hardforkDevAddr, + hardforkTax, + hardforkStorageProof, + hardforkOak, + hardforkASIC, + hardforkFoundation, + hardforkV2, + ] ChainIndex: type: object @@ -1286,13 +1420,83 @@ components: ConsensusState: type: object - description: JSON encoding of `go.sia.tech/core/consensus.State`. - additionalProperties: true + description: Snapshot of consensus-related chain state at a specific tip. + properties: + index: + $ref: "#/components/schemas/ChainIndex" + prevTimestamps: + type: array + description: Last 11 block timestamps, newest first. + items: + type: string + format: date-time + depth: + $ref: "#/components/schemas/BlockID" + childTarget: + $ref: "#/components/schemas/BlockID" + siafundTaxRevenue: + $ref: "#/components/schemas/Currency" + oakTime: + type: string + description: Weighted average block time encoded as a Go duration string. + oakTarget: + $ref: "#/components/schemas/BlockID" + foundationSubsidyAddress: + $ref: "#/components/schemas/Address" + foundationManagementAddress: + $ref: "#/components/schemas/Address" + totalWork: + $ref: "#/components/schemas/Work" + difficulty: + $ref: "#/components/schemas/Work" + oakWork: + $ref: "#/components/schemas/Work" + elements: + $ref: "#/components/schemas/ElementAccumulator" + attestations: + type: integer + format: uint64 + required: + [ + index, + prevTimestamps, + depth, + childTarget, + siafundTaxRevenue, + oakTime, + oakTarget, + foundationSubsidyAddress, + foundationManagementAddress, + totalWork, + difficulty, + oakWork, + elements, + attestations, + ] Block: type: object - description: JSON encoding of `go.sia.tech/core/types.Block`. - additionalProperties: true + description: Block as returned by the walletd consensus endpoints. + properties: + parentID: + $ref: "#/components/schemas/BlockID" + nonce: + type: integer + format: uint64 + timestamp: + type: string + format: date-time + minerPayouts: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + transactions: + type: array + items: + $ref: "#/components/schemas/Transaction" + v2: + $ref: "#/components/schemas/V2BlockData" + required: [parentID, nonce, timestamp, minerPayouts, transactions] ConsensusCheckpointResponse: type: object @@ -1411,13 +1615,103 @@ components: Transaction: type: object - description: JSON encoding of `go.sia.tech/core/types.Transaction`, including derived IDs. - additionalProperties: true + description: Sia v1 transaction including derived identifiers returned by walletd. + properties: + id: + $ref: "#/components/schemas/TransactionID" + siacoinInputs: + type: array + items: + $ref: "#/components/schemas/SiacoinInput" + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutputWithID" + fileContracts: + type: array + items: + $ref: "#/components/schemas/FileContract" + fileContractRevisions: + type: array + items: + $ref: "#/components/schemas/FileContractRevision" + storageProofs: + type: array + items: + $ref: "#/components/schemas/StorageProof" + siafundInputs: + type: array + items: + $ref: "#/components/schemas/SiafundInput" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/SiafundOutputWithID" + minerFees: + type: array + items: + $ref: "#/components/schemas/Currency" + arbitraryData: + type: array + description: Arbitrary data entries encoded as base64 strings. + items: + type: string + format: byte + signatures: + type: array + items: + $ref: "#/components/schemas/TransactionSignature" + required: [id] V2Transaction: type: object - description: JSON encoding of `go.sia.tech/core/types.V2Transaction`, including derived IDs and proofs. - additionalProperties: true + description: Sia v2 transaction including derived identifiers returned by walletd. + properties: + id: + $ref: "#/components/schemas/TransactionID" + minerFee: + $ref: "#/components/schemas/Currency" + siacoinInputs: + type: array + items: + $ref: "#/components/schemas/V2SiacoinInput" + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutputWithID" + siafundInputs: + type: array + items: + $ref: "#/components/schemas/V2SiafundInput" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/SiafundOutputWithID" + fileContracts: + type: array + items: + $ref: "#/components/schemas/V2FileContract" + fileContractRevisions: + type: array + items: + $ref: "#/components/schemas/V2FileContractRevision" + fileContractResolutions: + type: array + items: + $ref: "#/components/schemas/V2FileContractResolution" + attestations: + type: array + items: + $ref: "#/components/schemas/Attestation" + arbitraryData: + type: string + format: byte + description: Arbitrary payload encoded as base64. + newFoundationAddress: + allOf: + - $ref: "#/components/schemas/Address" + - nullable: true + required: [id, minerFee] Balance: type: object @@ -1431,6 +1725,40 @@ components: format: uint64 required: [siacoins, immatureSiacoins, siafunds] + Work: + type: string + description: Cumulative work value represented as a base-10 stringified big integer. + example: "115792089237316195423570985008687907853269984665640564039457584007913129639935" + + ElementAccumulator: + type: object + description: Accumulator used to track Merkle proofs for chain elements. + properties: + numLeaves: + type: integer + format: uint64 + trees: + type: array + description: Roots of the accumulator trees currently populated. + items: + $ref: "#/components/schemas/Hash256" + required: [numLeaves, trees] + + V2BlockData: + type: object + description: Additional data present when a block contains v2 transactions. + properties: + height: + type: integer + format: uint64 + commitment: + $ref: "#/components/schemas/Hash256" + transactions: + type: array + items: + $ref: "#/components/schemas/V2Transaction" + required: [height, commitment, transactions] + Wallet: type: object properties: @@ -1494,6 +1822,156 @@ components: required: [type, policy] additionalProperties: true + UnlockKey: + type: string + description: Unlock key encoded as `:`. + example: ed25519:29d666f502bd8e3f83ae599434662d9ef7eed1c61fbfd83bcdca15330434353a + + UnlockConditions: + type: object + properties: + timelock: + type: integer + format: uint64 + publicKeys: + type: array + items: + $ref: "#/components/schemas/UnlockKey" + signaturesRequired: + type: integer + format: uint64 + required: [timelock, publicKeys, signaturesRequired] + + SiacoinInput: + type: object + properties: + parentID: + $ref: "#/components/schemas/SiacoinOutputID" + unlockConditions: + $ref: "#/components/schemas/UnlockConditions" + address: + $ref: "#/components/schemas/Address" + required: [parentID, unlockConditions, address] + + SiafundInput: + type: object + properties: + parentID: + $ref: "#/components/schemas/SiafundOutputID" + unlockConditions: + $ref: "#/components/schemas/UnlockConditions" + claimAddress: + $ref: "#/components/schemas/Address" + address: + $ref: "#/components/schemas/Address" + required: [parentID, unlockConditions, claimAddress, address] + + SiacoinOutputWithID: + allOf: + - $ref: "#/components/schemas/SiacoinOutput" + - type: object + properties: + id: + $ref: "#/components/schemas/SiacoinOutputID" + required: [id] + + SiafundOutputWithID: + allOf: + - $ref: "#/components/schemas/SiafundOutput" + - type: object + properties: + id: + $ref: "#/components/schemas/SiafundOutputID" + required: [id] + + StorageProof: + type: object + properties: + parentID: + $ref: "#/components/schemas/FileContractID" + leaf: + type: string + description: Hex-encoded 64 byte leaf. + proof: + type: array + items: + $ref: "#/components/schemas/Hash256" + required: [parentID, leaf, proof] + + CoveredFields: + type: object + properties: + wholeTransaction: + type: boolean + siacoinInputs: + type: array + items: + type: integer + format: uint64 + siacoinOutputs: + type: array + items: + type: integer + format: uint64 + fileContracts: + type: array + items: + type: integer + format: uint64 + fileContractRevisions: + type: array + items: + type: integer + format: uint64 + storageProofs: + type: array + items: + type: integer + format: uint64 + siafundInputs: + type: array + items: + type: integer + format: uint64 + siafundOutputs: + type: array + items: + type: integer + format: uint64 + minerFees: + type: array + items: + type: integer + format: uint64 + arbitraryData: + type: array + items: + type: integer + format: uint64 + signatures: + type: array + items: + type: integer + format: uint64 + + TransactionSignature: + type: object + properties: + parentID: + $ref: "#/components/schemas/Hash256" + publicKeyIndex: + type: integer + format: uint64 + timelock: + type: integer + format: uint64 + coveredFields: + $ref: "#/components/schemas/CoveredFields" + signature: + type: string + description: Hex-encoded signature. + required: [parentID, publicKeyIndex, coveredFields, signature] + WalletReserveRequest: type: object properties: @@ -1707,6 +2185,169 @@ components: $ref: "#/components/schemas/Currency" required: [id, stateElement, siafundOutput, claimStart] + FileContract: + type: object + properties: + filesize: + type: integer + format: uint64 + fileMerkleRoot: + $ref: "#/components/schemas/Hash256" + windowStart: + type: integer + format: uint64 + windowEnd: + type: integer + format: uint64 + payout: + $ref: "#/components/schemas/Currency" + validProofOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + missedProofOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + unlockHash: + $ref: "#/components/schemas/Address" + revisionNumber: + type: integer + format: uint64 + required: + [ + filesize, + fileMerkleRoot, + windowStart, + windowEnd, + payout, + validProofOutputs, + missedProofOutputs, + unlockHash, + revisionNumber, + ] + + FileContractElement: + type: object + properties: + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + fileContract: + $ref: "#/components/schemas/FileContract" + required: [id, stateElement, fileContract] + + FileContractRevision: + type: object + properties: + parentID: + $ref: "#/components/schemas/FileContractID" + unlockConditions: + $ref: "#/components/schemas/UnlockConditions" + filesize: + type: integer + format: uint64 + fileMerkleRoot: + $ref: "#/components/schemas/Hash256" + windowStart: + type: integer + format: uint64 + windowEnd: + type: integer + format: uint64 + validProofOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + missedProofOutputs: + type: array + items: + $ref: "#/components/schemas/SiacoinOutput" + unlockHash: + $ref: "#/components/schemas/Address" + revisionNumber: + type: integer + format: uint64 + required: + [ + parentID, + unlockConditions, + filesize, + fileMerkleRoot, + windowStart, + windowEnd, + validProofOutputs, + missedProofOutputs, + unlockHash, + revisionNumber, + ] + + V2FileContract: + type: object + properties: + capacity: + type: integer + format: uint64 + filesize: + type: integer + format: uint64 + fileMerkleRoot: + $ref: "#/components/schemas/Hash256" + proofHeight: + type: integer + format: uint64 + expirationHeight: + type: integer + format: uint64 + renterOutput: + $ref: "#/components/schemas/SiacoinOutput" + hostOutput: + $ref: "#/components/schemas/SiacoinOutput" + missedHostValue: + $ref: "#/components/schemas/Currency" + totalCollateral: + $ref: "#/components/schemas/Currency" + renterPublicKey: + $ref: "#/components/schemas/PublicKey" + hostPublicKey: + $ref: "#/components/schemas/PublicKey" + revisionNumber: + type: integer + format: uint64 + renterSignature: + $ref: "#/components/schemas/Signature" + hostSignature: + $ref: "#/components/schemas/Signature" + required: + [ + capacity, + filesize, + fileMerkleRoot, + proofHeight, + expirationHeight, + renterOutput, + hostOutput, + missedHostValue, + totalCollateral, + renterPublicKey, + hostPublicKey, + revisionNumber, + renterSignature, + hostSignature, + ] + + V2FileContractElement: + type: object + properties: + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + v2FileContract: + $ref: "#/components/schemas/V2FileContract" + required: [id, stateElement, v2FileContract] + StateElement: type: object properties: @@ -1738,6 +2379,206 @@ components: $ref: "#/components/schemas/Address" required: [value, address] + ChainIndexElement: + type: object + properties: + id: + $ref: "#/components/schemas/BlockID" + stateElement: + $ref: "#/components/schemas/StateElement" + chainIndex: + $ref: "#/components/schemas/ChainIndex" + required: [id, stateElement, chainIndex] + + Attestation: + type: object + properties: + publicKey: + $ref: "#/components/schemas/PublicKey" + key: + type: string + value: + type: string + format: byte + signature: + $ref: "#/components/schemas/Signature" + required: [publicKey, key, value, signature] + + AttestationElement: + type: object + properties: + id: + $ref: "#/components/schemas/Hash256" + stateElement: + $ref: "#/components/schemas/StateElement" + attestation: + $ref: "#/components/schemas/Attestation" + required: [id, stateElement, attestation] + + SatisfiedPolicy: + type: object + properties: + policy: + $ref: "#/components/schemas/SpendPolicy" + signatures: + type: array + items: + $ref: "#/components/schemas/Signature" + preimages: + type: array + items: + type: string + pattern: "^[0-9a-fA-F]{64}$" + required: [policy] + + V2SiacoinInput: + type: object + properties: + parent: + $ref: "#/components/schemas/SiacoinElement" + satisfiedPolicy: + $ref: "#/components/schemas/SatisfiedPolicy" + required: [parent, satisfiedPolicy] + + V2SiafundInput: + type: object + properties: + parent: + $ref: "#/components/schemas/SiafundElement" + claimAddress: + $ref: "#/components/schemas/Address" + satisfiedPolicy: + $ref: "#/components/schemas/SatisfiedPolicy" + required: [parent, claimAddress, satisfiedPolicy] + + V2FileContractRevision: + type: object + properties: + parent: + $ref: "#/components/schemas/V2FileContractElement" + revision: + $ref: "#/components/schemas/V2FileContract" + required: [parent, revision] + + V2FileContractRenewal: + type: object + properties: + finalRenterOutput: + $ref: "#/components/schemas/SiacoinOutput" + finalHostOutput: + $ref: "#/components/schemas/SiacoinOutput" + renterRollover: + $ref: "#/components/schemas/Currency" + hostRollover: + $ref: "#/components/schemas/Currency" + newContract: + $ref: "#/components/schemas/V2FileContract" + renterSignature: + $ref: "#/components/schemas/Signature" + hostSignature: + $ref: "#/components/schemas/Signature" + required: + [ + finalRenterOutput, + finalHostOutput, + renterRollover, + hostRollover, + newContract, + renterSignature, + hostSignature, + ] + + V2StorageProof: + type: object + properties: + proofIndex: + $ref: "#/components/schemas/ChainIndexElement" + leaf: + type: string + description: Hex-encoded 64 byte leaf. + proof: + type: array + items: + $ref: "#/components/schemas/Hash256" + required: [proofIndex, leaf, proof] + + V2FileContractExpiration: + type: object + description: Empty object used to signal a contract expiration event. + + V2FileContractResolution: + type: object + properties: + parent: + $ref: "#/components/schemas/V2FileContractElement" + type: + type: string + enum: [renewal, storageProof, expiration] + resolution: + oneOf: + - $ref: "#/components/schemas/V2FileContractRenewal" + - $ref: "#/components/schemas/V2StorageProof" + - $ref: "#/components/schemas/V2FileContractExpiration" + required: [parent, type, resolution] + + SiacoinElementDiff: + type: object + properties: + siacoinElement: + $ref: "#/components/schemas/SiacoinElement" + created: + type: boolean + spent: + type: boolean + required: [siacoinElement, created, spent] + + SiafundElementDiff: + type: object + properties: + siafundElement: + $ref: "#/components/schemas/SiafundElement" + created: + type: boolean + spent: + type: boolean + required: [siafundElement, created, spent] + + FileContractElementDiff: + type: object + properties: + fileContractElement: + $ref: "#/components/schemas/FileContractElement" + created: + type: boolean + revision: + allOf: + - $ref: "#/components/schemas/FileContract" + - nullable: true + resolved: + type: boolean + valid: + type: boolean + required: [fileContractElement, created, resolved, valid] + + V2FileContractElementDiff: + type: object + properties: + v2FileContractElement: + $ref: "#/components/schemas/V2FileContractElement" + created: + type: boolean + revision: + allOf: + - $ref: "#/components/schemas/V2FileContract" + - nullable: true + resolution: + nullable: true + oneOf: + - $ref: "#/components/schemas/V2FileContractRenewal" + - $ref: "#/components/schemas/V2StorageProof" + - $ref: "#/components/schemas/V2FileContractExpiration" + required: [v2FileContractElement, created] + ElementSpentResponse: type: object properties: @@ -1808,9 +2649,7 @@ components: type: object properties: update: - type: object - description: JSON encoding of `go.sia.tech/core/consensus.ApplyUpdate`. - additionalProperties: true + $ref: "#/components/schemas/ApplyUpdateData" state: $ref: "#/components/schemas/ConsensusState" block: @@ -1821,15 +2660,118 @@ components: type: object properties: update: - type: object - description: JSON encoding of `go.sia.tech/core/consensus.RevertUpdate`. - additionalProperties: true + $ref: "#/components/schemas/RevertUpdateData" state: $ref: "#/components/schemas/ConsensusState" block: $ref: "#/components/schemas/Block" required: [update, state, block] + ApplyUpdateData: + type: object + properties: + siacoinElements: + type: array + items: + $ref: "#/components/schemas/SiacoinElementDiff" + siafundElementDiffs: + type: array + items: + $ref: "#/components/schemas/SiafundElementDiff" + fileContractElementDiffs: + type: array + items: + $ref: "#/components/schemas/FileContractElementDiff" + v2FileContractElementDiffs: + type: array + items: + $ref: "#/components/schemas/V2FileContractElementDiff" + attestationElements: + type: array + items: + $ref: "#/components/schemas/AttestationElement" + chainIndexElement: + $ref: "#/components/schemas/ChainIndexElement" + updatedLeaves: + type: object + additionalProperties: + type: array + description: Updated leaf proofs keyed by tree height. + items: + $ref: "#/components/schemas/StateElement" + treeGrowth: + type: object + additionalProperties: + type: array + description: Merkle subtree hashes added during the update keyed by tree height. + items: + $ref: "#/components/schemas/Hash256" + oldNumLeaves: + type: integer + format: uint64 + numLeaves: + type: integer + format: uint64 + required: + [ + siacoinElements, + siafundElementDiffs, + fileContractElementDiffs, + v2FileContractElementDiffs, + attestationElements, + chainIndexElement, + updatedLeaves, + treeGrowth, + oldNumLeaves, + numLeaves, + ] + + RevertUpdateData: + type: object + properties: + siacoinElements: + type: array + items: + $ref: "#/components/schemas/SiacoinElementDiff" + siafundElementDiffs: + type: array + items: + $ref: "#/components/schemas/SiafundElementDiff" + fileContractElementDiffs: + type: array + items: + $ref: "#/components/schemas/FileContractElementDiff" + v2FileContractElementDiffs: + type: array + items: + $ref: "#/components/schemas/V2FileContractElementDiff" + attestationElements: + type: array + items: + $ref: "#/components/schemas/AttestationElement" + chainIndexElement: + $ref: "#/components/schemas/ChainIndexElement" + updatedLeaves: + type: object + additionalProperties: + type: array + items: + $ref: "#/components/schemas/StateElement" + numLeaves: + type: integer + format: uint64 + required: + [ + siacoinElements, + siafundElementDiffs, + fileContractElementDiffs, + v2FileContractElementDiffs, + attestationElements, + chainIndexElement, + updatedLeaves, + numLeaves, + ] + DebugMineRequest: type: object properties: From a2c71cef0cdafb141e111e0d2b1d3d7b022f9797 Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 14 Oct 2025 15:07:35 +0200 Subject: [PATCH 544/630] gh: add env variable --- .github/workflows/openapi-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/openapi-publish.yml b/.github/workflows/openapi-publish.yml index fadcc4c..fcf2f39 100644 --- a/.github/workflows/openapi-publish.yml +++ b/.github/workflows/openapi-publish.yml @@ -19,3 +19,4 @@ jobs: docs_slug: sia secrets: SCALAR_API_KEY: ${{ secrets.SCALAR_API_KEY }} + SCALAR_ACCESS_TOKEN: ${{ secrets.SCALAR_ACCESS_TOKEN }} From dc5d7ea03ece2ba6972f13315ab9cab45215b484 Mon Sep 17 00:00:00 2001 From: Peter-Jan Brone Date: Wed, 8 Oct 2025 17:03:53 +0200 Subject: [PATCH 545/630] docs: update slug --- .github/workflows/openapi-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openapi-publish.yml b/.github/workflows/openapi-publish.yml index fcf2f39..8d073a6 100644 --- a/.github/workflows/openapi-publish.yml +++ b/.github/workflows/openapi-publish.yml @@ -16,7 +16,7 @@ jobs: with: slug: walletd spec_path: openapi.yml - docs_slug: sia + docs_slug: walletd secrets: SCALAR_API_KEY: ${{ secrets.SCALAR_API_KEY }} SCALAR_ACCESS_TOKEN: ${{ secrets.SCALAR_ACCESS_TOKEN }} From 5938d8f743657b552335c10d90bdf6ffd69927f0 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 15 Oct 2025 16:59:49 +0200 Subject: [PATCH 546/630] docs: fix tags --- openapi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openapi.yml b/openapi.yml index 4978ee9..3fd1231 100644 --- a/openapi.yml +++ b/openapi.yml @@ -662,7 +662,7 @@ paths: /check/addresses: post: tags: - - addresses + - misc summary: Check address membership description: > Returns whether any supplied addresses are currently tracked by walletd. @@ -685,7 +685,7 @@ paths: /events/{id}: get: tags: - - outputs + - events summary: Fetch event by ID operationId: getEvent parameters: @@ -902,7 +902,7 @@ paths: /wallets/{id}/events: get: tags: - - events + - wallets summary: Wallet events operationId: getWalletEvents parameters: From 7f9a3b638839e5521f9c2dd7e962a784eb239a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 16:24:41 +0000 Subject: [PATCH 547/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.18.5 to 0.18.6 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.5...v0.18.6) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.18.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 080f0a6..2226428 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.18.0 - go.sia.tech/coreutils v0.18.5 + go.sia.tech/coreutils v0.18.6 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index c2b35b5..9498c47 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.18.0 h1:b4xOGdbMmbrqji5y+mHiP0TCzJADQY1okPPapiKuSag= go.sia.tech/core v0.18.0/go.mod h1:evyyK5QluEbh0QmVAgtttrbUafj3365fZ4MLePvGTM4= -go.sia.tech/coreutils v0.18.5 h1:lFKgeC2jAV5mkAcDAs13jZmaPiQ/kVxyDEx6DxzqXgU= -go.sia.tech/coreutils v0.18.5/go.mod h1:RbHOI5chN8xU28mBWJbWnsp/jAbDGJjqqgyeENpTvAc= +go.sia.tech/coreutils v0.18.6 h1:RfaA/EWmUyXrgdU8PeyQkT5zbkYvCy/vTHOSplfpSnk= +go.sia.tech/coreutils v0.18.6/go.mod h1:Z3ZfYwin15Q77z1WapaESrlocxsLOXrpvKcWRuA1wrI= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= From 88834de71c9a0675be559be123b5713d5a7dbc67 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Mon, 20 Oct 2025 16:41:14 -0400 Subject: [PATCH 548/630] Add -checkpoint flag --- README.md | 1 + cmd/walletd/main.go | 2 + cmd/walletd/node.go | 89 ++++++++++++++----------------------- config/config.go | 3 ++ go.mod | 11 +++-- go.sum | 20 ++++----- persist/sqlite/consensus.go | 6 +++ 7 files changed, 60 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 81f05fa..023d332 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ log: level: debug # override the global log level for the file path: /var/log/walletd.log format: json # human or JSON + checkpoint: 530000::0000000000000000abb98e3b587fba3a0c4e723ac1e078e9d6a4d13d1d131a2c ``` ## Building diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index 5a3d554..10dcdcd 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -204,6 +204,8 @@ func main() { rootCmd.BoolVar(&cfg.Log.File.Enabled, "log.file.enabled", cfg.Log.File.Enabled, "enable file logging") rootCmd.BoolVar(&cfg.Log.StdOut.Enabled, "log.stdout.enabled", cfg.Log.StdOut.Enabled, "enable stdout logging") + rootCmd.TextVar(&cfg.Checkpoint, "checkpoint", cfg.Checkpoint, "instant-sync to a chain index, e.g. 530000::0000000000000000abb98e3b587fba3a0c4e723ac1e078e9d6a4d13d1d131a2c") + versionCmd := flagg.New("version", versionUsage) seedCmd := flagg.New("seed", seedUsage) configCmd := flagg.New("config", "interactively configure walletd") diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 5e45e90..f9a2952 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -146,52 +146,6 @@ func loadCustomNetwork(fp string) (*consensus.Network, types.Block, error) { return &network.Network, network.Genesis, nil } -// migrateConsensusDB checks if the consensus database needs to be migrated -// to match the new v2 commitment. -func migrateConsensusDB(fp string, n *consensus.Network, genesis types.Block, log *zap.Logger) error { - bdb, err := coreutils.OpenBoltChainDB(fp) - if err != nil { - return fmt.Errorf("failed to open consensus database: %w", err) - } - defer bdb.Close() - - dbstore, tipState, err := chain.NewDBStore(bdb, n, genesis, chain.NewZapMigrationLogger(log.Named("chaindb"))) - if err != nil { - return fmt.Errorf("failed to create chain store: %w", err) - } else if tipState.Index.Height < n.HardforkV2.AllowHeight { - return nil // no migration needed, the chain is still on v1 - } - - log.Debug("checking for v2 commitment migration") - b, _, ok := dbstore.Block(tipState.Index.ID) - if !ok { - return fmt.Errorf("failed to get tip block %q", tipState.Index) - } else if b.V2 == nil { - log.Debug("tip block is not a v2 block, skipping commitment migration") - return nil - } - - parentState, ok := dbstore.State(b.ParentID) - if !ok { - return fmt.Errorf("failed to get parent state for tip block %q", b.ParentID) - } - commitment := parentState.Commitment(b.MinerPayouts[0].Address, b.Transactions, b.V2Transactions()) - log = log.With(zap.Stringer("tip", b.ID()), zap.Stringer("commitment", b.V2.Commitment), zap.Stringer("expected", commitment)) - if b.V2.Commitment == commitment { - log.Debug("tip block commitment matches parent state, no migration needed") - return nil - } - // reset the database if the commitment is not a merkle root - log.Debug("resetting consensus database for new v2 commitment") - if err := bdb.Close(); err != nil { - return fmt.Errorf("failed to close old consensus database: %w", err) - } else if err := os.RemoveAll(fp); err != nil { - return fmt.Errorf("failed to remove old consensus database: %w", err) - } - log.Debug("consensus database reset") - return nil -} - func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { @@ -219,22 +173,45 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } } - consensusPath := filepath.Join(cfg.Directory, "consensus.db") - if err := migrateConsensusDB(consensusPath, network, genesisBlock, log.Named("migrate")); err != nil { - return fmt.Errorf("failed to open consensus database: %w", err) - } - - bdb, err := coreutils.OpenBoltChainDB(consensusPath) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) if err != nil { return fmt.Errorf("failed to open consensus database: %w", err) } defer bdb.Close() - dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) - if err != nil { - return fmt.Errorf("failed to create chain store: %w", err) + var cm *chain.Manager + if cfg.Checkpoint != (types.ChainIndex{}) { + log.Info("beginning instant sync", zap.Stringer("checkpoint", cfg.Checkpoint)) + peers := append(cfg.Syncer.Peers, bootstrapPeers...) + cs, b, err := func() (consensus.State, types.Block, error) { + for _, peer := range peers { + log.Info("attempt to fetch checkpoint", zap.String("peer", peer)) + cs, b, err := syncer.SendCheckpoint(ctx, peer, cfg.Checkpoint, network, genesisBlock.ID()) + if err == nil { + return cs, b, nil + } + } + return consensus.State{}, types.Block{}, errors.New("failed to fetch checkpoint from any peer") + }() + if err != nil { + return err + } + dbstore, tipState, err := chain.NewDBStoreAtCheckpoint(bdb, cs, b, chain.NewZapMigrationLogger(log.Named("chaindb"))) + if err != nil { + return fmt.Errorf("failed to create chain store: %w", err) + } + cm = chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) + if err := store.SetCheckpoint(cfg.Checkpoint); err != nil { + return fmt.Errorf("failed to set wallet db checkpoint: %w", err) + } + log.Info("instant sync successful", zap.Stringer("tip", cm.Tip())) + } else { + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) + if err != nil { + return fmt.Errorf("failed to create chain store: %w", err) + } + cm = chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) } - cm := chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) if err != nil { diff --git a/config/config.go b/config/config.go index e6caf31..c0db577 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" "gopkg.in/yaml.v3" @@ -73,6 +74,8 @@ type ( Syncer Syncer `yaml:"syncer,omitempty"` Log Log `yaml:"log,omitempty"` Index Index `yaml:"index,omitempty"` + + Checkpoint types.ChainIndex `yaml:"checkpoint,omitempty"` } ) diff --git a/go.mod b/go.mod index 2226428..695ebf7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.18.0 - go.sia.tech/coreutils v0.18.6 + go.sia.tech/coreutils v0.18.6-0.20251017124723-95041a1930e8 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.0 @@ -20,18 +20,17 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.1 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect - go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.42.0 // indirect + golang.org/x/crypto v0.43.0 // indirect golang.org/x/mod v0.28.0 // indirect - golang.org/x/net v0.44.0 // indirect + golang.org/x/net v0.46.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.29.0 // indirect + golang.org/x/text v0.30.0 // indirect golang.org/x/tools v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index 9498c47..755201c 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= -github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -28,8 +28,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.18.0 h1:b4xOGdbMmbrqji5y+mHiP0TCzJADQY1okPPapiKuSag= go.sia.tech/core v0.18.0/go.mod h1:evyyK5QluEbh0QmVAgtttrbUafj3365fZ4MLePvGTM4= -go.sia.tech/coreutils v0.18.6 h1:RfaA/EWmUyXrgdU8PeyQkT5zbkYvCy/vTHOSplfpSnk= -go.sia.tech/coreutils v0.18.6/go.mod h1:Z3ZfYwin15Q77z1WapaESrlocxsLOXrpvKcWRuA1wrI= +go.sia.tech/coreutils v0.18.6-0.20251017124723-95041a1930e8 h1:GkQB54PzanqObQ0iDMaY4wgDd3Ncb2I0fIxBMTIbw1c= +go.sia.tech/coreutils v0.18.6-0.20251017124723-95041a1930e8/go.mod h1:k9caP/7LxEcYnXlMLNGyz6in8BEoVBbUE7EjxMJAzns= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -46,22 +46,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 125f294..833f4bf 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -224,6 +224,12 @@ func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { return } +// SetCheckpoint sets the last indexed tip to the given index. +func (s *Store) SetCheckpoint(index types.ChainIndex) error { + _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_height=$1, last_indexed_id=$2`, index.Height, encode(index.ID)) + return err +} + // ResetLastIndex resets the last indexed tip to trigger a full rescan. func (s *Store) ResetLastIndex() error { _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1`, encode(types.BlockID{})) From 0ffa8e75ecf3d6a1cd2fbd8fae47ea2741893df7 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 15 Oct 2025 09:27:27 -0700 Subject: [PATCH 549/630] add input sig hash --- ...sh_to_response_body_of_walletsidconstructv2transaction.md | 5 +++++ api/api.go | 1 + api/server.go | 1 + openapi.yml | 4 +++- 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md diff --git a/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md b/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md new file mode 100644 index 0000000..c2ea477 --- /dev/null +++ b/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Add `inputSigHash` to response body of `/wallets/:id/construct/v2/transaction` diff --git a/api/api.go b/api/api.go index 91b3102..f0c2218 100644 --- a/api/api.go +++ b/api/api.go @@ -147,6 +147,7 @@ type WalletConstructV2Response struct { ID types.TransactionID `json:"id"` Transaction types.V2Transaction `json:"transaction"` EstimatedFee types.Currency `json:"estimatedFee"` + InputSigHash types.Hash256 `json:"inputSigHash"` } // SeedSignRequest requests that a transaction be signed using the keys derived diff --git a/api/server.go b/api/server.go index 0339360..99eb53a 100644 --- a/api/server.go +++ b/api/server.go @@ -1320,6 +1320,7 @@ func (s *server) walletsConstructV2Handler(jc jape.Context) { resp.ID = txn.ID() resp.Transaction = txn + resp.InputSigHash = cs.InputSigHash(txn) sent = true // locks are released in defer jc.Encode(resp) } diff --git a/openapi.yml b/openapi.yml index 3fd1231..c7d4916 100644 --- a/openapi.yml +++ b/openapi.yml @@ -2077,7 +2077,9 @@ components: $ref: "#/components/schemas/V2Transaction" estimatedFee: $ref: "#/components/schemas/Currency" - required: [basis, id, transaction, estimatedFee] + inputSigHash: + $ref: "#/components/schemas/Hash256" + required: [basis, id, transaction, estimatedFee, inputSigHash] UnspentSiacoinElementsResponse: type: object From 2fb7f943d3f81287d2f791fc6f37ccc2227babb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 02:50:39 +0000 Subject: [PATCH 550/630] build(deps): bump golang.org/x/crypto from 0.42.0 to 0.45.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.42.0 to 0.45.0. - [Commits](https://github.com/golang/crypto/compare/v0.42.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 2226428..8cb10cc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.36.0 + golang.org/x/term v0.37.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -27,11 +27,11 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.42.0 // indirect - golang.org/x/mod v0.28.0 // indirect - golang.org/x/net v0.44.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.29.0 // indirect - golang.org/x/tools v0.37.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index 9498c47..526137e 100644 --- a/go.sum +++ b/go.sum @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From f864dd956e7536dfea4fc56604b484c1fd934335 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:50:41 +0000 Subject: [PATCH 551/630] build(deps): bump the all-dependencies group with 3 updates Bumps the all-dependencies group with 3 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.uber.org/zap](https://github.com/uber-go/zap). Updates `go.sia.tech/core` from 0.18.0 to 0.18.1 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.18.0...v0.18.1) Updates `go.sia.tech/coreutils` from 0.18.6 to 0.18.7 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.6...v0.18.7) Updates `go.uber.org/zap` from 1.27.0 to 1.27.1 - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.27.0...v1.27.1) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.18.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.18.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.uber.org/zap dependency-version: 1.27.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 11 ++++------- go.sum | 22 ++++++++++------------ 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 8cb10cc..d257e85 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 - go.sia.tech/core v0.18.0 - go.sia.tech/coreutils v0.18.6 + go.sia.tech/core v0.18.1 + go.sia.tech/coreutils v0.18.7 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.5 - go.uber.org/zap v1.27.0 + go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.37.0 gopkg.in/yaml.v3 v3.0.1 @@ -20,17 +20,14 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.1 // indirect + github.com/quic-go/quic-go v0.56.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect - go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.45.0 // indirect - golang.org/x/mod v0.29.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.38.0 // indirect diff --git a/go.sum b/go.sum index 526137e..1e2511b 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -16,8 +12,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= -github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.56.0 h1:q/TW+OLismmXAehgFLczhCDTYB3bFmua4D9lsNBWxvY= +github.com/quic-go/quic-go v0.56.0/go.mod h1:9gx5KsFQtw2oZ6GZTyh+7YEvOxWCL9WZAepnHxgAo6c= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -26,10 +22,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.18.0 h1:b4xOGdbMmbrqji5y+mHiP0TCzJADQY1okPPapiKuSag= -go.sia.tech/core v0.18.0/go.mod h1:evyyK5QluEbh0QmVAgtttrbUafj3365fZ4MLePvGTM4= -go.sia.tech/coreutils v0.18.6 h1:RfaA/EWmUyXrgdU8PeyQkT5zbkYvCy/vTHOSplfpSnk= -go.sia.tech/coreutils v0.18.6/go.mod h1:Z3ZfYwin15Q77z1WapaESrlocxsLOXrpvKcWRuA1wrI= +go.sia.tech/core v0.18.1 h1:EOxviJtUMxnMnRs9g0qFgtxSL/2Ig6Jq6jIoa4jUT7A= +go.sia.tech/core v0.18.1/go.mod h1:nrVyqhH9cSdWdd7+I+MAtUU5IUhw4Z+cLNMk6mwQ3OA= +go.sia.tech/coreutils v0.18.7 h1:Z36IoZv3EeAijNMA5RsZbzgeIuVOp9cF46tZgm8ACDc= +go.sia.tech/coreutils v0.18.7/go.mod h1:i6Kt6sYZamu2rcKNcfcrF1hF6a2rg8qMxpZIvgh6rwE= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -44,8 +40,8 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= @@ -62,6 +58,8 @@ golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 8773ae0c833e5e2b51778d4755eee844495d2fe7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:42:34 +0000 Subject: [PATCH 552/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.18.1 to 0.19.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.18.1...v0.19.0) Updates `go.sia.tech/coreutils` from 0.18.7 to 0.19.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.18.7...v0.19.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 30 ++++++++++++++++-------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index d257e85..08a35b2 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 - go.sia.tech/core v0.18.1 - go.sia.tech/coreutils v0.18.7 + go.sia.tech/core v0.19.0 + go.sia.tech/coreutils v0.19.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.1 @@ -19,8 +19,8 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect - github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.56.0 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.57.1 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect @@ -30,5 +30,5 @@ require ( golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect - golang.org/x/tools v0.38.0 // indirect + golang.org/x/tools v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index 1e2511b..79b6117 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -10,22 +12,22 @@ github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuE github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= -github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.56.0 h1:q/TW+OLismmXAehgFLczhCDTYB3bFmua4D9lsNBWxvY= -github.com/quic-go/quic-go v0.56.0/go.mod h1:9gx5KsFQtw2oZ6GZTyh+7YEvOxWCL9WZAepnHxgAo6c= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= +github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.18.1 h1:EOxviJtUMxnMnRs9g0qFgtxSL/2Ig6Jq6jIoa4jUT7A= -go.sia.tech/core v0.18.1/go.mod h1:nrVyqhH9cSdWdd7+I+MAtUU5IUhw4Z+cLNMk6mwQ3OA= -go.sia.tech/coreutils v0.18.7 h1:Z36IoZv3EeAijNMA5RsZbzgeIuVOp9cF46tZgm8ACDc= -go.sia.tech/coreutils v0.18.7/go.mod h1:i6Kt6sYZamu2rcKNcfcrF1hF6a2rg8qMxpZIvgh6rwE= +go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= +go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= +go.sia.tech/coreutils v0.19.0 h1:P2lWRGwI5/NvzhlHt83U+OK8RYR+ePD7F9uQzf6woUg= +go.sia.tech/coreutils v0.19.0/go.mod h1:BvRPC48OvX2/lKqUeNbwGYxnzAaPj4UHz5yIPjmB6tw= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -46,8 +48,8 @@ golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= @@ -60,8 +62,8 @@ golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 2d762557bacceba845c370a7a0e46be19f60a5ae Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:18:17 +0100 Subject: [PATCH 553/630] add checkpoint flag to readme --- ...nt_cli_flag_for_instant_syncing_to_a_given_chain_index.md | 5 +++++ README.md | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 .changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md diff --git a/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md b/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md new file mode 100644 index 0000000..ea0e8d8 --- /dev/null +++ b/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Add -checkpoint CLI flag for instant-syncing to a given chain index. diff --git a/README.md b/README.md index 023d332..d9ca8fd 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ Flags: p2p address to listen on (default ":9981") -bootstrap attempt to bootstrap the network (default true) + -checkpoint + instant-sync to a chain index, e.g. 530000::0000000000000000abb98e3b587fba3a0c4e723ac1e078e9d6a4d13d1d131a2c -debug enable debug mode with additional profiling and mining endpoints -dir string @@ -112,6 +114,7 @@ the working directory. All fields are optional. ```yaml directory: /etc/walletd autoOpenWebUI: true +checkpoint: 530000::0000000000000000abb98e3b587fba3a0c4e723ac1e078e9d6a4d13d1d131a2c http: address: :9980 password: sia is cool From 67c8987ae01b281bb4b0e57aa531f751dfaf4e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 16:09:22 +0000 Subject: [PATCH 554/630] build(deps): bump golang.org/x/term in the all-dependencies group Bumps the all-dependencies group with 1 update: [golang.org/x/term](https://github.com/golang/term). Updates `golang.org/x/term` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/term/compare/v0.37.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/term dependency-version: 0.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 08a35b2..99ff091 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( go.sia.tech/web/walletd v0.34.5 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.37.0 + golang.org/x/term v0.38.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -28,7 +28,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index 79b6117..452f0bf 100644 --- a/go.sum +++ b/go.sum @@ -54,10 +54,10 @@ golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= From 4354853aa4655cc6fd1aee8ba8d31edfecbfa2b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Dec 2025 19:28:11 +0000 Subject: [PATCH 555/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/coreutils` from 0.19.0 to 0.20.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.19.0...v0.20.0) Updates `go.sia.tech/web/walletd` from 0.34.5 to 0.35.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.34.5...hostd@0.35.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.20.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.35.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 99ff091..ed419c0 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.32 go.sia.tech/core v0.19.0 - go.sia.tech/coreutils v0.19.0 + go.sia.tech/coreutils v0.20.0 go.sia.tech/jape v0.14.1 - go.sia.tech/web/walletd v0.34.5 + go.sia.tech/web/walletd v0.35.0 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.38.0 @@ -26,9 +26,9 @@ require ( go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.45.0 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.39.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index 452f0bf..886554d 100644 --- a/go.sum +++ b/go.sum @@ -26,16 +26,16 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= -go.sia.tech/coreutils v0.19.0 h1:P2lWRGwI5/NvzhlHt83U+OK8RYR+ePD7F9uQzf6woUg= -go.sia.tech/coreutils v0.19.0/go.mod h1:BvRPC48OvX2/lKqUeNbwGYxnzAaPj4UHz5yIPjmB6tw= +go.sia.tech/coreutils v0.20.0 h1:7G6z+xL0VzwYueBV5S5jqzrmzdWQ5cL4NL2LAfv4PhE= +go.sia.tech/coreutils v0.20.0/go.mod h1:P/dgMZgZtpqs72HkiYafwXAcZnTLYhsg3TLq1mpvJJ4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.34.5 h1:IrS5ktvLvUBkNYcC9w56bVnqV6X+PxvxjY9vlsfAH/M= -go.sia.tech/web/walletd v0.34.5/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.35.0 h1:M1qrEIKPRhvrmojXPg1UTqi0MYr6FPd3VtO+6vgYOwc= +go.sia.tech/web/walletd v0.35.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -44,22 +44,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= From df217e41f8853c3b60c866448f80ac0ce7b3b8fc Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 18 Dec 2025 11:38:16 -0800 Subject: [PATCH 556/630] use RetrieveCheckpoint --- cmd/walletd/node.go | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index f9a2952..5edc79c 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -173,29 +173,25 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } } - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) - if err != nil { - return fmt.Errorf("failed to open consensus database: %w", err) - } - defer bdb.Close() + consensusDBPath := filepath.Join(cfg.Directory, "consensus.db") + _, existsErr := os.Open(consensusDBPath) + consensusExists := !errors.Is(existsErr, os.ErrNotExist) var cm *chain.Manager - if cfg.Checkpoint != (types.ChainIndex{}) { + if cfg.Checkpoint != (types.ChainIndex{}) && !consensusExists { log.Info("beginning instant sync", zap.Stringer("checkpoint", cfg.Checkpoint)) peers := append(cfg.Syncer.Peers, bootstrapPeers...) - cs, b, err := func() (consensus.State, types.Block, error) { - for _, peer := range peers { - log.Info("attempt to fetch checkpoint", zap.String("peer", peer)) - cs, b, err := syncer.SendCheckpoint(ctx, peer, cfg.Checkpoint, network, genesisBlock.ID()) - if err == nil { - return cs, b, nil - } - } - return consensus.State{}, types.Block{}, errors.New("failed to fetch checkpoint from any peer") - }() + cs, b, err := syncer.RetrieveCheckpoint(ctx, peers, cfg.Checkpoint, network, genesisBlock.ID()) + if err != nil { + return fmt.Errorf("failed to retrieve checkpoint: %w", err) + } + + bdb, err := coreutils.OpenBoltChainDB(consensusDBPath) if err != nil { - return err + return fmt.Errorf("failed to open consensus database: %w", err) } + defer bdb.Close() + dbstore, tipState, err := chain.NewDBStoreAtCheckpoint(bdb, cs, b, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { return fmt.Errorf("failed to create chain store: %w", err) @@ -206,6 +202,17 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } log.Info("instant sync successful", zap.Stringer("tip", cm.Tip())) } else { + if cfg.Checkpoint != (types.ChainIndex{}) { + // checkpoint specified but consensus db already exists + log.Warn("skipping instant sync. consensus database already exists") + } + + bdb, err := coreutils.OpenBoltChainDB(consensusDBPath) + if err != nil { + return fmt.Errorf("failed to open consensus database: %w", err) + } + defer bdb.Close() + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { return fmt.Errorf("failed to create chain store: %w", err) From a68edcf9a88f74ee8cdde0e05ce698c1bae195c7 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 18 Dec 2025 11:38:57 -0800 Subject: [PATCH 557/630] update changeset versions --- .changeset/update_core_dependency_from_0175_to_0180.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/update_core_dependency_from_0175_to_0180.md b/.changeset/update_core_dependency_from_0175_to_0180.md index 65dcb23..03cbcef 100644 --- a/.changeset/update_core_dependency_from_0175_to_0180.md +++ b/.changeset/update_core_dependency_from_0175_to_0180.md @@ -2,4 +2,4 @@ default: patch --- -# Update core dependency from 0.17.5 to 0.18.0. +# Update core dependency to v0.19.0 and coreutils dependency to v0.20.0. From 790537b4db0e9df2b00dee69cef2e4c99682f7c4 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 18 Dec 2025 11:47:21 -0800 Subject: [PATCH 558/630] don't panic if threadgroup closed --- wallet/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/manager.go b/wallet/manager.go index 62accc3..ec4f73d 100644 --- a/wallet/manager.go +++ b/wallet/manager.go @@ -775,7 +775,7 @@ func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) log := m.log.Named("sync") ctx, cancel, err := m.tg.AddWithContext(context.Background()) if err != nil { - log.Panic("failed to add to threadgroup", zap.Error(err)) + return } defer cancel() From 245cc821537bce50e888cb6877e58b642dfb7171 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 22 Dec 2025 15:22:24 -0500 Subject: [PATCH 559/630] refactor wallet tests --- wallet/addresses_test.go | 82 +---- wallet/manager_test.go | 14 +- wallet/wallet_test.go | 733 +++++++-------------------------------- 3 files changed, 150 insertions(+), 679 deletions(-) diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 745c19b..0513c6f 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -6,13 +6,10 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/v2/internal/testutil" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) func TestAddressUseTpool(t *testing.T) { - log := zaptest.NewLogger(t) - // mine a single payout to the wallet pk := types.GeneratePrivateKey() uc := types.StandardUnlockConditions(pk.PublicKey()) @@ -22,17 +19,10 @@ func TestAddressUseTpool(t *testing.T) { genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ {Address: addr1, Value: types.Siacoins(100)}, } - cn := testutil.NewConsensusNode(t, network, genesisBlock, log) - cm := cn.Chain - db := cn.Store - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, wm := tn.Chain, tn.manager - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement := func(t *testing.T, id types.SiacoinOutputID, value types.Currency, confirmations uint64) { t.Helper() @@ -95,29 +85,20 @@ func TestAddressUseTpool(t *testing.T) { } wm.SyncPool() // force reindexing of the tpool assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) } func TestBatchAddresses(t *testing.T) { - log := zaptest.NewLogger(t) - network, genesisBlock := testutil.V2Network() - cn := testutil.NewConsensusNode(t, network, genesisBlock, log) - cm := cn.Chain - db := cn.Store - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + wm := tn.manager // mine a bunch of payouts to different addresses addresses := make([]types.Address, 100) for i := range addresses { addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) - cn.MineBlocks(t, addresses[i], 1) + tn.MineBlocks(t, addresses[i], 1) } events, err := wm.BatchAddressEvents(addresses, 0, 1000) @@ -129,26 +110,17 @@ func TestBatchAddresses(t *testing.T) { } func TestBatchSiacoinOutputs(t *testing.T) { - log := zaptest.NewLogger(t) - network, genesisBlock := testutil.V2Network() - cn := testutil.NewConsensusNode(t, network, genesisBlock, log) - cm := cn.Chain - db := cn.Store - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + wm := tn.manager // mine a bunch of payouts to different addresses addresses := make([]types.Address, 100) for i := range addresses { addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) - cn.MineBlocks(t, addresses[i], 1) + tn.MineBlocks(t, addresses[i], 1) } - cn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) sces, _, err := wm.BatchAddressSiacoinOutputs(addresses, 0, 1000) if err != nil { @@ -159,24 +131,15 @@ func TestBatchSiacoinOutputs(t *testing.T) { } func TestBatchSiafundOutputs(t *testing.T) { - log := zaptest.NewLogger(t) - giftAddr := types.AnyoneCanSpend().Address() network, genesisBlock := testutil.V2Network() genesisBlock.Transactions[0].SiafundOutputs = []types.SiafundOutput{ {Address: giftAddr, Value: 10000}, } - cn := testutil.NewConsensusNode(t, network, genesisBlock, log) - cm := cn.Chain - db := cn.Store - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + db, cm, wm := tn.Store, tn.Chain, tn.manager - cn.WaitForSync(t) + tn.WaitForSync(t) // distribute the siafund output to multiple addresses var addresses []types.Address @@ -223,7 +186,7 @@ func TestBatchSiafundOutputs(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, txns); err != nil { t.Fatalf("failed to add pool transactions %d: %s", i, err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) } sfes, _, err := wm.BatchAddressSiafundOutputs(addresses, 0, 10000) @@ -235,24 +198,15 @@ func TestBatchSiafundOutputs(t *testing.T) { } func BenchmarkBatchAddresses(b *testing.B) { - log := zaptest.NewLogger(b) - network, genesisBlock := testutil.V2Network() - cn := testutil.NewConsensusNode(b, network, genesisBlock, log) - cm := cn.Chain - db := cn.Store - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - b.Fatal(err) - } - defer wm.Close() + tn := newTestNode(b, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + wm := tn.manager // mine a bunch of payouts to different addresses addresses := make([]types.Address, 10000) for i := range addresses { addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) - cn.MineBlocks(b, addresses[i], 1) + tn.MineBlocks(b, addresses[i], 1) } b.ResetTimer() diff --git a/wallet/manager_test.go b/wallet/manager_test.go index a1d8846..25a86ad 100644 --- a/wallet/manager_test.go +++ b/wallet/manager_test.go @@ -7,26 +7,18 @@ import ( "go.sia.tech/core/types" "go.sia.tech/walletd/v2/internal/testutil" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap/zaptest" ) func TestHealth(t *testing.T) { - log := zaptest.NewLogger(t) n, genesis := testutil.V2Network() - cn := testutil.NewConsensusNode(t, n, genesis, log) - cm := cn.Chain - - wm, err := wallet.NewManager(cm, cn.Store) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, n, genesis) + wm := tn.manager if err := wm.Health(); !errors.Is(err, wallet.ErrNotSyncing) { t.Fatalf("expected error %q, got %q", wallet.ErrNotSyncing, err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) if err := wm.Health(); err != nil { t.Fatalf("expected no error, got %v", err) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index a43cfe8..bc52d3e 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -17,7 +17,8 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/testutil" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/v2/internal/testutil" "go.sia.tech/walletd/v2/persist/sqlite" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" @@ -39,13 +40,38 @@ func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { func mineAndSync(tb testing.TB, cm *chain.Manager, ws wallet.Store, addr types.Address, n int) { tb.Helper() - for i := 0; i < n; i++ { - testutil.MineBlocks(tb, cm, addr, 1) + ctestutil.MineBlocks(tb, cm, addr, 1) waitForBlock(tb, cm, ws) } } +type testNode struct { + *testutil.ConsensusNode + log *zap.Logger + manager *wallet.Manager +} + +func newTestNode(tb testing.TB, network *consensus.Network, genesisBlock types.Block, walletOpts ...wallet.Option) *testNode { + tb.Helper() + + log := zaptest.NewLogger(tb) + cn := testutil.NewConsensusNode(tb, network, genesisBlock, log.Named("consensus")) + + opts := append([]wallet.Option{wallet.WithLogger(log.Named("wallet"))}, walletOpts...) + wm, err := wallet.NewManager(cn.Chain, cn.Store, opts...) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { wm.Close() }) + + return &testNode{ + ConsensusNode: cn, + log: log, + manager: wm, + } +} + func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { // use a modified version of Zen n, genesisBlock := chain.TestnetZen() @@ -110,32 +136,9 @@ func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr ty } func TestReserve(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - network, genesisBlock := testutil.V2Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithLockDuration(2*time.Second)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithLockDuration(2*time.Second)) + wm := tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -184,35 +187,11 @@ func TestReserve(t *testing.T) { } func TestSelectSiacoins(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testutil.Network() + network, genesisBlock := ctestutil.Network() network.InitialCoinbase = types.Siacoins(100) network.MinimumCoinbase = types.Siacoins(100) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + wm, cm := tn.manager, tn.Chain w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -238,18 +217,10 @@ func TestSelectSiacoins(t *testing.T) { t.Fatal(err) } - mineAndSync := func(t *testing.T, addr types.Address, n int) { - t.Helper() - - for i := 0; i < n; i++ { - testutil.MineBlocks(t, cm, addr, 1) - waitForBlock(t, cm, db) - } - } // mine enough utxos to ensure the pagination works - mineAndSync(t, addr, 200) + tn.MineBlocks(t, addr, 200) // mine until all the wallet's outputs are mature - mineAndSync(t, types.VoidAddress, int(cm.TipState().Network.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(cm.TipState().Network.MaturityDelay)) // check that the wallet has 200 matured outputs utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 1000) @@ -336,7 +307,7 @@ func TestSelectSiacoins(t *testing.T) { t.Fatal("transaction was already known") } - mineAndSync(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) events, err := wm.WalletEvents(w.ID, 0, 1) if err != nil { @@ -353,20 +324,6 @@ func TestSelectSiacoins(t *testing.T) { } func TestSelectSiafunds(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - sk := types.GeneratePrivateKey() uc := types.UnlockConditions{ PublicKeys: []types.UnlockKey{sk.PublicKey().UnlockKey()}, @@ -374,22 +331,12 @@ func TestSelectSiafunds(t *testing.T) { } addr := uc.UnlockHash() - network, genesisBlock := testutil.Network() + network, genesisBlock := ctestutil.Network() genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr network.InitialCoinbase = types.Siacoins(100) network.MinimumCoinbase = types.Siacoins(100) - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + wm, cm := tn.manager, tn.Chain w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -408,15 +355,7 @@ func TestSelectSiafunds(t *testing.T) { t.Fatal(err) } - mineAndSync := func(t *testing.T, addr types.Address, n int) { - t.Helper() - - for i := 0; i < n; i++ { - testutil.MineBlocks(t, cm, addr, 1) - waitForBlock(t, cm, db) - } - } - mineAndSync(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check that the wallet has a siafund utxo utxos, _, err := wm.UnspentSiafundOutputs(w.ID, 0, 1000) @@ -477,7 +416,7 @@ func TestSelectSiafunds(t *testing.T) { t.Fatal("transaction was already known") } - mineAndSync(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) events, err := wm.WalletEvents(w.ID, 0, 1) if err != nil { @@ -497,40 +436,15 @@ func TestReorg(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - setupNode := func(t *testing.T, mode wallet.IndexMode) (consensus.State, *sqlite.Store, *chain.Manager, *wallet.Manager) { + setupNode := func(t *testing.T, mode wallet.IndexMode) (*testNode, consensus.State) { t.Helper() - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { db.Close() }) - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { bdb.Close() }) - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(mode)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { wm.Close() }) - return genesisState, db, cm, wm + network, genesisBlock := testV1Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(mode)) + return tn, tn.Chain.TipState() } - testReorg := func(t *testing.T, genesisState consensus.State, db *sqlite.Store, cm *chain.Manager, wm *wallet.Manager) { + testReorg := func(t *testing.T, tn *testNode, genesisState consensus.State) { + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { t.Fatal(err) @@ -694,13 +608,13 @@ func TestReorg(t *testing.T) { } t.Run("IndexModePersonal", func(t *testing.T) { - state, db, cm, w := setupNode(t, wallet.IndexModePersonal) - testReorg(t, state, db, cm, w) + tn, state := setupNode(t, wallet.IndexModePersonal) + testReorg(t, tn, state) }) t.Run("IndexModeFull", func(t *testing.T) { - state, db, cm, w := setupNode(t, wallet.IndexModeFull) - testReorg(t, state, db, cm, w) + tn, state := setupNode(t, wallet.IndexModeFull) + testReorg(t, tn, state) }) } @@ -708,33 +622,9 @@ func TestEphemeralBalance(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV1Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -904,33 +794,9 @@ func TestEphemeralBalance(t *testing.T) { } func TestWalletAddresses(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV1Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + wm := tn.manager // Add a wallet w := wallet.Wallet{ @@ -938,7 +804,7 @@ func TestWalletAddresses(t *testing.T) { Description: "hello, world!", Metadata: json.RawMessage(`{"foo": "bar"}`), } - w, err = wm.AddWallet(w) + w, err := wm.AddWallet(w) if err != nil { t.Fatal(err) } @@ -1031,40 +897,14 @@ func TestWalletAddresses(t *testing.T) { } func TestScan(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // mine a single payout to the wallet pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - network, genesisBlock := testutil.Network() - // send the siafunds to the owned address + network, genesisBlock := ctestutil.Network() genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + genesisState := tn.Chain.TipState() + db, cm, wm := tn.Store, tn.Chain, tn.manager pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -1198,40 +1038,13 @@ func TestScan(t *testing.T) { } func TestSiafunds(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // mine a single payout to the wallet pk := types.GeneratePrivateKey() addr1 := types.StandardUnlockHash(pk.PublicKey()) - network, genesisBlock := testutil.Network() - // send the siafunds to the owned address + network, genesisBlock := ctestutil.Network() genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + db, cm, wm := tn.Store, tn.Chain, tn.manager pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -1359,35 +1172,11 @@ func TestOrphans(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + network, genesisBlock := testV1Network(types.VoidAddress) network.HardforkV2.AllowHeight = 200 network.HardforkV2.RequireHeight = 201 - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -1517,7 +1306,7 @@ func TestOrphans(t *testing.T) { t.Fatal(err) } - wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(tn.log.Named("wallet"))) if err != nil { t.Fatal(err) } @@ -1553,35 +1342,12 @@ func TestFullIndex(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - pk2 := types.GeneratePrivateKey() - addr2 := types.StandardUnlockHash(pk2.PublicKey()) - - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(addr2) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + network, genesisBlock := testV2Network(addr2) + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + db, cm, wm := tn.Store, tn.Chain, tn.manager waitForBlock(t, cm, db) @@ -1780,32 +1546,9 @@ func TestEvents(t *testing.T) { pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - network, genesisBlock := testV2Network(addr2) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + db, cm, wm := tn.Store, tn.Chain, tn.manager waitForBlock(t, cm, db) @@ -2032,37 +1775,12 @@ func TestEvents(t *testing.T) { } func TestWalletUnconfirmedEvents(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // mine a single payout to the wallet pk := types.GeneratePrivateKey() addr1 := types.StandardUnlockHash(pk.PublicKey()) - network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := ctestutil.Network() + tn := newTestNode(t, network, genesisBlock) + cm, wm := tn.Chain, tn.manager // create a wallet with no addresses w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) @@ -2076,8 +1794,8 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // mine a block sending the payout to the wallet - mineAndSync(t, cm, db, addr1, 1) - mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + tn.MineBlocks(t, addr1, 1) + tn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { @@ -2208,7 +1926,7 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } // mine the transactions - mineAndSync(t, cm, db, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check that the unconfirmed events were removed events, err = wm.WalletUnconfirmedEvents(w1.ID) @@ -2220,37 +1938,12 @@ func TestWalletUnconfirmedEvents(t *testing.T) { } func TestAddressUnconfirmedEvents(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // mine a single payout to the wallet pk := types.GeneratePrivateKey() addr1 := types.StandardUnlockHash(pk.PublicKey()) - network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := ctestutil.Network() + tn := newTestNode(t, network, genesisBlock) + cm, wm := tn.Chain, tn.manager // create a wallet with no addresses w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) @@ -2264,9 +1957,9 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // mine a block sending the payout to the wallet - mineAndSync(t, cm, db, addr1, 1) + tn.MineBlocks(t, addr1, 1) // mine until the payout matures - mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) if err != nil { @@ -2406,7 +2099,7 @@ func TestAddressUnconfirmedEvents(t *testing.T) { } // mine the transactions - mineAndSync(t, cm, db, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check that the unconfirmed events were removed events, err = wm.AddressUnconfirmedEvents(addr1) @@ -2428,33 +2121,9 @@ func TestV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV2Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -2464,7 +2133,7 @@ func TestV2(t *testing.T) { } expectedPayout := cm.TipState().BlockReward() - mineAndSync(t, cm, db, addr, 1) + tn.MineBlocks(t, addr, 1) // check that the payout was received balance, err := db.AddressBalance(addr) @@ -2485,7 +2154,7 @@ func TestV2(t *testing.T) { } // mine until the payout matures - mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) // create a v2 transaction that spends the matured payout utxos, basis, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) @@ -2512,7 +2181,7 @@ func TestV2(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - mineAndSync(t, cm, db, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check that the change was received balance, err = wm.AddressBalance(addr) @@ -2536,37 +2205,13 @@ func TestV2(t *testing.T) { } func TestScanV2(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - // mine a single payout to the wallet pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testV2Network(addr) - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesisBlock) + genesisState := tn.Chain.TipState() + db, cm, wm := tn.Store, tn.Chain, tn.manager pk2 := types.GeneratePrivateKey() addr2 := types.StandardUnlockHash(pk2.PublicKey()) @@ -2731,33 +2376,10 @@ func TestReorgV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV2Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + genesisState := tn.Chain.TipState() + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -2959,32 +2581,9 @@ func TestOrphansV2(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV2Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -3105,7 +2704,7 @@ func TestOrphansV2(t *testing.T) { t.Fatal(err) } - wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(tn.log.Named("wallet"))) if err != nil { t.Fatal(err) } @@ -3173,33 +2772,9 @@ func TestDeleteWallet(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds - - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + network, genesisBlock := testV1Network(types.VoidAddress) + tn := newTestNode(t, network, genesisBlock) + wm := tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -3857,36 +3432,11 @@ func TestSiafundClaims(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - - network, genesis := testutil.Network() - // send the siafunds to the owned address + network, genesis := ctestutil.Network() genesis.Transactions[0].SiafundOutputs[0].Address = addr siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value - - store, genesisState, err := chain.NewDBStore(bdb, network, genesis, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesis) + db, cm, wm := tn.Store, tn.Chain, tn.manager w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -3941,7 +3491,7 @@ func TestSiafundClaims(t *testing.T) { if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) @@ -3959,8 +3509,8 @@ func TestSiafundClaims(t *testing.T) { } // fund the wallet with some siacoins - testutil.MineBlocks(t, cm, addr, 5) - testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + ctestutil.MineBlocks(t, cm, addr, 5) + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) waitForBlock(t, cm, db) payout := types.Siacoins(100000) @@ -4017,7 +3567,7 @@ func TestSiafundClaims(t *testing.T) { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) cs = cm.TipState() @@ -4058,7 +3608,7 @@ func TestSiafundClaims(t *testing.T) { if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) events, err = wm.WalletEvents(w.ID, 0, 100) @@ -4083,7 +3633,7 @@ func TestSiafundClaims(t *testing.T) { } // mine until the siafund claim output is mature - testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) waitForBlock(t, cm, db) // check that the output is now spendable @@ -4103,39 +3653,14 @@ func TestV2SiafundClaims(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardAddress(pk.PublicKey()) - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), sqlite.WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) - if err != nil { - t.Fatal(err) - } - defer bdb.Close() - network, genesis := testutil.V2Network() - // send the siafunds to the owned address genesis.Transactions[0].SiafundOutputs[0].Address = addr siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value - - store, genesisState, err := chain.NewDBStore(bdb, network, genesis, nil) - if err != nil { - t.Fatal(err) - } - cm := chain.NewManager(store, genesisState) - - wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) - if err != nil { - t.Fatal(err) - } - defer wm.Close() + tn := newTestNode(t, network, genesis) + db, cm, wm := tn.Store, tn.Chain, tn.manager // activate the v2 hardfork - testutil.MineBlocks(t, cm, types.VoidAddress, 2) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 2) w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) if err != nil { @@ -4187,7 +3712,7 @@ func TestV2SiafundClaims(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) @@ -4205,8 +3730,8 @@ func TestV2SiafundClaims(t *testing.T) { } // fund the wallet with some siacoins - testutil.MineBlocks(t, cm, addr, 5) - testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + ctestutil.MineBlocks(t, cm, addr, 5) + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) waitForBlock(t, cm, db) payout := types.Siacoins(100000) @@ -4262,7 +3787,7 @@ func TestV2SiafundClaims(t *testing.T) { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) cs = cm.TipState() @@ -4299,7 +3824,7 @@ func TestV2SiafundClaims(t *testing.T) { if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - testutil.MineBlocks(t, cm, types.VoidAddress, 1) + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) waitForBlock(t, cm, db) events, err = wm.WalletEvents(w.ID, 0, 100) @@ -4324,7 +3849,7 @@ func TestV2SiafundClaims(t *testing.T) { } // mine until the siafund claim output is mature - testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) waitForBlock(t, cm, db) // check that the output is now spendable @@ -4346,7 +3871,7 @@ func TestReset(t *testing.T) { pk := types.GeneratePrivateKey() addr := types.StandardUnlockHash(pk.PublicKey()) - network, genesisBlock := testutil.Network() + network, genesisBlock := ctestutil.Network() // send the siafunds to the owned address genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr From 068f8fe504352d6f57031db8dac2fb2a7878cc87 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Fri, 19 Dec 2025 15:15:24 -0500 Subject: [PATCH 560/630] use helper to create stores --- persist/sqlite/address_test.go | 11 +---------- persist/sqlite/consensus_test.go | 21 ++++----------------- persist/sqlite/events_test.go | 8 ++------ persist/sqlite/peers_test.go | 20 ++------------------ persist/sqlite/store_test.go | 25 +++++++++++++++++++++++++ persist/sqlite/wallet_test.go | 18 +++--------------- 6 files changed, 37 insertions(+), 66 deletions(-) create mode 100644 persist/sqlite/store_test.go diff --git a/persist/sqlite/address_test.go b/persist/sqlite/address_test.go index 135b718..a9764ed 100644 --- a/persist/sqlite/address_test.go +++ b/persist/sqlite/address_test.go @@ -1,30 +1,21 @@ package sqlite import ( - "path/filepath" "testing" "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) func TestCheckAddresses(t *testing.T) { - log := zaptest.NewLogger(t) - // generate a large number of random addresses addresses := make([]types.Address, 1000) for i := range addresses { addresses[i] = frand.Entropy256() } - // create a new database - db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() + db := newTestStore(t) if known, err := db.CheckAddresses(addresses); err != nil { t.Fatal(err) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index d1b1325..f0d7b80 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -10,7 +10,6 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/testutil" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap/zaptest" ) func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { @@ -49,15 +48,9 @@ func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { } func TestPruneSiacoins(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), WithLog(log.Named("sqlite3")), WithRetainSpentElements(20)) - if err != nil { - t.Fatal(err) - } - defer db.Close() + db := newTestStore(t, WithRetainSpentElements(20)) - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) if err != nil { t.Fatal(err) } @@ -190,15 +183,9 @@ func TestPruneSiacoins(t *testing.T) { } func TestPruneSiafunds(t *testing.T) { - log := zaptest.NewLogger(t) - dir := t.TempDir() - db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() + db := newTestStore(t) - bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) if err != nil { t.Fatal(err) } diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go index 3197432..05e7194 100644 --- a/persist/sqlite/events_test.go +++ b/persist/sqlite/events_test.go @@ -2,21 +2,17 @@ package sqlite import ( "fmt" - "path/filepath" "testing" "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" + "go.uber.org/zap" "lukechampine.com/frand" ) func runBenchmarkWalletEvents(b *testing.B, name string, addresses, eventsPerAddress int) { b.Run(name, func(b *testing.B) { - db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3")) - if err != nil { - b.Fatal(err) - } - defer db.Close() + db := newTestStore(b, WithLog(zap.NewNop())) w, err := db.AddWallet(wallet.Wallet{ Name: "test", diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go index a6ec068..86b7457 100644 --- a/persist/sqlite/peers_test.go +++ b/persist/sqlite/peers_test.go @@ -2,29 +2,20 @@ package sqlite import ( "net" - "path/filepath" "testing" "time" "go.sia.tech/coreutils/syncer" - "go.uber.org/zap/zaptest" ) func TestAddPeer(t *testing.T) { - log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - + db := newTestStore(t) ps, err := NewPeerStore(db) if err != nil { t.Fatal(err) } const peer = "1.2.3.4:9981" - if err := ps.AddPeer(peer); err != nil { t.Fatal(err) } @@ -76,20 +67,13 @@ func TestAddPeer(t *testing.T) { } func TestBanPeer(t *testing.T) { - log := zaptest.NewLogger(t) - db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - + db := newTestStore(t) ps, err := NewPeerStore(db) if err != nil { t.Fatal(err) } const peer = "1.2.3.4" - if banned, err := ps.Banned(peer); err != nil || banned { t.Fatal("expected peer to not be banned", err) } diff --git a/persist/sqlite/store_test.go b/persist/sqlite/store_test.go new file mode 100644 index 0000000..23aa14a --- /dev/null +++ b/persist/sqlite/store_test.go @@ -0,0 +1,25 @@ +package sqlite + +import ( + "path/filepath" + "testing" + + "go.uber.org/zap/zaptest" +) + +// newTestStore creates a new Store for testing. It is closed automatically +// when the test completes. +func newTestStore(t testing.TB, opts ...Option) *Store { + t.Helper() + + log := zaptest.NewLogger(t) + opts = append([]Option{WithLog(log.Named("sqlite3"))}, opts...) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), opts...) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + db.Close() + }) + return db +} diff --git a/persist/sqlite/wallet_test.go b/persist/sqlite/wallet_test.go index 51228f6..68d3b9a 100644 --- a/persist/sqlite/wallet_test.go +++ b/persist/sqlite/wallet_test.go @@ -2,19 +2,16 @@ package sqlite import ( "fmt" - "path/filepath" "reflect" "testing" "go.sia.tech/core/types" "go.sia.tech/walletd/v2/wallet" - "go.uber.org/zap/zaptest" + "go.uber.org/zap" "lukechampine.com/frand" ) func TestAddAddresses(t *testing.T) { - log := zaptest.NewLogger(t) - // generate a large number of random addresses addresses := make([]wallet.Address, 1000) for i := range addresses { @@ -25,12 +22,7 @@ func TestAddAddresses(t *testing.T) { addresses[i].Description = fmt.Sprintf("address %d", i) } - // create a new database - db, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite"), WithLog(log.Named("sqlite3"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() + db := newTestStore(t) w, err := db.AddWallet(wallet.Wallet{}) if err != nil { @@ -88,11 +80,7 @@ func TestAddAddresses(t *testing.T) { } func BenchmarkAddWalletAddresses(b *testing.B) { - db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3")) - if err != nil { - b.Fatal(err) - } - defer db.Close() + db := newTestStore(b, WithLog(zap.NewNop())) addresses := make([]wallet.Address, b.N) for i := range addresses { From 7630541843c4a3bb94efe9e7975940e47fcfec35 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Mon, 5 Jan 2026 12:39:12 +0100 Subject: [PATCH 561/630] Write permissions for openapi sync --- .github/workflows/openapi-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/openapi-sync.yml b/.github/workflows/openapi-sync.yml index 7214783..2f2aade 100644 --- a/.github/workflows/openapi-sync.yml +++ b/.github/workflows/openapi-sync.yml @@ -1,7 +1,7 @@ name: Sync OpenAPI Versions permissions: - contents: read + contents: write pull-requests: write on: From e7757ce92204e2263d907b025c8f80cc914f107e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 16:08:06 +0000 Subject: [PATCH 562/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.32 to 1.14.33 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.32...v1.14.33) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.33 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ed419c0..ccc87f8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.11.0 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.32 + github.com/mattn/go-sqlite3 v1.14.33 go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.20.0 go.sia.tech/jape v0.14.1 diff --git a/go.sum b/go.sum index 886554d..3f33cc1 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= -github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0= +github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= From c23514c4f538acb0c14e2dcb8d14d62b10c622cf Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Mon, 5 Jan 2026 19:17:29 -0500 Subject: [PATCH 563/630] refactor API tests --- api/api_test.go | 586 ++++++++++++++++++++---------------------------- 1 file changed, 238 insertions(+), 348 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 44d10b0..859e52a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -25,6 +25,92 @@ import ( "lukechampine.com/frand" ) +// testNode wraps a ConsensusNode with additional fields. +type testNode struct { + *testutil.ConsensusNode + network *consensus.Network + client *api.Client + genesis types.Block + pk types.PrivateKey +} + +func (tn *testNode) fundingAddr() types.Address { + return types.StandardUnlockHash(tn.pk.PublicKey()) +} + +// newV1TestNode creates a V1 network test node with initial siacoin funding. +// If sf is true, also assigns genesis siafunds to the funding address. +func newV1TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { + tb.Helper() + + n, genesisBlock := testutil.V1Network() + fundingKey := types.GeneratePrivateKey() + fundingAddr := types.StandardUnlockHash(fundingKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: sc, + Address: fundingAddr, + } + if sf { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = fundingAddr + } + cn := testutil.NewConsensusNode(tb, n, genesisBlock, log) + c := startWalletServer(tb, cn, log, walletOpts...) + return &testNode{ + ConsensusNode: cn, + network: n, + client: c, + genesis: genesisBlock, + pk: fundingKey, + } +} + +// newV2TestNode creates a V2 network test node with initial siacoin funding. +// If sf is true, also assigns genesis siafunds to the funding address. +func newV2TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { + tb.Helper() + + n, genesisBlock := testutil.V2Network() + fundingKey := types.GeneratePrivateKey() + fundingAddr := types.StandardUnlockHash(fundingKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: sc, + Address: fundingAddr, + } + if sf { + genesisBlock.Transactions[0].SiafundOutputs[0].Address = fundingAddr + } + cn := testutil.NewConsensusNode(tb, n, genesisBlock, log) + c := startWalletServer(tb, cn, log, walletOpts...) + return &testNode{ + ConsensusNode: cn, + network: n, + client: c, + genesis: genesisBlock, + pk: fundingKey, + } +} + +// signV1Txn signs all signatures in a V1 transaction. +func signV1Txn(cs consensus.State, txn *types.Transaction, pk types.PrivateKey) { + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(*txn, sig.ParentID, 0, 0, nil) + s := pk.SignHash(sigHash) + txn.Signatures[i].Signature = s[:] + } +} + +// signV2Txn signs all siacoin and siafund inputs in a V2 transaction. +func signV2Txn(cs consensus.State, txn *types.V2Transaction, pk types.PrivateKey) { + sigHash := cs.InputSigHash(*txn) + sig := pk.SignHash(sigHash) + for i := range txn.SiacoinInputs { + txn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } +} + func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logger, walletOpts ...wallet.Option) *api.Client { tb.Helper() @@ -53,16 +139,8 @@ func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logge func TestWalletAdd(t *testing.T) { log := zaptest.NewLogger(t) - - n, genesisBlock := testutil.V1Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV1TestNode(t, log, types.Siacoins(1), false) + c := tn.client checkWalletResponse := func(wr api.WalletUpdateRequest, w wallet.Wallet, isUpdate bool) error { // check wallet @@ -200,24 +278,8 @@ func TestWalletAdd(t *testing.T) { func TestWallet(t *testing.T) { log := zaptest.NewLogger(t) - - // create syncer - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - // create chain manager - n, genesisBlock := testutil.V1Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV1TestNode(t, log, types.Siacoins(1), false) + c := tn.client w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) if err != nil { @@ -229,7 +291,7 @@ func TestWallet(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cn.WaitForSync(t) + tn.WaitForSync(t) balance, err := wc.Balance() if err != nil { @@ -275,11 +337,11 @@ func TestWallet(t *testing.T) { } // send gift to wallet - giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + giftSCOID := tn.genesis.Transactions[0].SiacoinOutputID(0) txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: giftSCOID, - UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + UnlockConditions: types.StandardUnlockConditions(tn.pk.PublicKey()), }}, SiacoinOutputs: []types.SiacoinOutput{ {Address: addr, Value: types.Siacoins(1).Div64(2)}, @@ -296,7 +358,7 @@ func TestWallet(t *testing.T) { t.Fatal(err) } - sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + sig := tn.pk.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool @@ -319,7 +381,7 @@ func TestWallet(t *testing.T) { t.Fatal("txpool should have one transaction") } // confirm the transaction - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // get new balance balance, err = wc.Balance() @@ -344,15 +406,15 @@ func TestWallet(t *testing.T) { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) - } else if basis != cn.Chain.Tip() { - t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if basis != tn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", tn.Chain.Tip(), basis) } else if outputs[0].Confirmations != 1 { t.Fatalf("expected 1 confirmation, got %v", outputs[0].Confirmations) } // mine a block to add an immature balance - expectedPayout := cn.Chain.TipState().BlockReward() - cn.MineBlocks(t, addr, 1) + expectedPayout := tn.Chain.TipState().BlockReward() + tn.MineBlocks(t, addr, 1) // get new balance balance, err = wc.Balance() @@ -366,7 +428,7 @@ func TestWallet(t *testing.T) { // mine enough blocks for the miner payout to mature expectedBalance := types.Siacoins(1).Add(expectedPayout) - cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(tn.network.MaturityDelay)) // get new balance balance, err = wc.Balance() @@ -381,17 +443,8 @@ func TestWallet(t *testing.T) { func TestAddresses(t *testing.T) { log := zaptest.NewLogger(t) - - n, genesisBlock := testutil.V1Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV1TestNode(t, log, types.Siacoins(1), false) + c := tn.client sk2 := types.GeneratePrivateKey() addr := types.StandardUnlockHash(sk2.PublicKey()) @@ -408,11 +461,11 @@ func TestAddresses(t *testing.T) { } // send gift to wallet - giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + giftSCOID := tn.genesis.Transactions[0].SiacoinOutputID(0) txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: giftSCOID, - UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + UnlockConditions: types.StandardUnlockConditions(tn.pk.PublicKey()), }}, SiacoinOutputs: []types.SiacoinOutput{ {Address: addr, Value: types.Siacoins(1).Div64(2)}, @@ -429,14 +482,14 @@ func TestAddresses(t *testing.T) { t.Fatal(err) } - sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + sig := tn.pk.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] // broadcast the transaction to the transaction pool if _, err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // get new balance balance, err := c.AddressBalance(addr) @@ -461,13 +514,13 @@ func TestAddresses(t *testing.T) { t.Fatal(err) } else if len(outputs) != 2 { t.Fatal("should have two UTXOs, got", len(outputs)) - } else if basis != cn.Chain.Tip() { - t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if basis != tn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", tn.Chain.Tip(), basis) } // mine a block to add an immature balance - expectedPayout := cn.Chain.TipState().BlockReward() - cn.MineBlocks(t, addr, 1) + expectedPayout := tn.Chain.TipState().BlockReward() + tn.MineBlocks(t, addr, 1) // get new balance balance, err = c.AddressBalance(addr) @@ -481,7 +534,7 @@ func TestAddresses(t *testing.T) { // mine enough blocks for the miner payout to mature expectedBalance := types.Siacoins(1).Add(expectedPayout) - cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + tn.MineBlocks(t, types.VoidAddress, int(tn.network.MaturityDelay)) // get new balance balance, err = c.AddressBalance(addr) @@ -518,23 +571,14 @@ func TestAddresses(t *testing.T) { func TestConsensus(t *testing.T) { log := zaptest.NewLogger(t) - - n, genesisBlock := testutil.V2Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV2TestNode(t, log, types.Siacoins(1), false) + c := tn.client // mine a block - minedBlock, ok := coreutils.MineBlock(cn.Chain, types.Address{}, time.Minute) + minedBlock, ok := coreutils.MineBlock(tn.Chain, types.Address{}, time.Minute) if !ok { t.Fatal("no block found") - } else if err := cn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { + } else if err := tn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { t.Fatal(err) } @@ -557,23 +601,14 @@ func TestConsensus(t *testing.T) { func TestConsensusCheckpoint(t *testing.T) { log := zaptest.NewLogger(t) - - n, genesisBlock := testutil.V2Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV2TestNode(t, log, types.Siacoins(1), false) + c := tn.client // mine a block - minedBlock, ok := coreutils.MineBlock(cn.Chain, types.Address{}, time.Minute) + minedBlock, ok := coreutils.MineBlock(tn.Chain, types.Address{}, time.Minute) if !ok { t.Fatal("no block found") - } else if err := cn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { + } else if err := tn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { t.Fatal(err) } @@ -591,34 +626,25 @@ func TestConsensusCheckpoint(t *testing.T) { t.Fatal(err) } else if resp.Block.ID() != minedBlock.ID() { t.Fatal("mismatch") - } else if resp.State.Index != cn.Chain.Tip() { + } else if resp.State.Index != tn.Chain.Tip() { t.Fatal("mismatch tip") } - heightResp, err := c.ConsensusCheckpointHeight(cn.Chain.Tip().Height) + heightResp, err := c.ConsensusCheckpointHeight(tn.Chain.Tip().Height) if err != nil { t.Fatal(err) } else if heightResp.Block.ID() != minedBlock.ID() { t.Fatal("mismatch") - } else if heightResp.State.Index != cn.Chain.Tip() { + } else if heightResp.State.Index != tn.Chain.Tip() { t.Fatal("mismatch tip") } } func TestConsensusUpdates(t *testing.T) { log := zaptest.NewLogger(t) - - n, genesisBlock := testutil.V1Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) - cn.MineBlocks(t, types.VoidAddress, 10) + tn := newV1TestNode(t, log, types.Siacoins(1), false) + c := tn.client + tn.MineBlocks(t, types.VoidAddress, 10) reverted, applied, err := c.ConsensusUpdates(types.ChainIndex{}, 10) if err != nil { @@ -631,36 +657,27 @@ func TestConsensusUpdates(t *testing.T) { for i, cau := range applied { // using i for height since we're testing the update contents - expected, ok := cn.Chain.BestIndex(uint64(i)) + expected, ok := tn.Chain.BestIndex(uint64(i)) if !ok { t.Fatalf("failed to get expected index for block %v", i) } else if cau.State.Index != expected { t.Fatalf("expected index %v, got %v", expected, cau.State.Index) - } else if cau.State.Network.Name != n.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. - t.Fatalf("expected network to be %q, got %q", n.Name, cau.State.Network.Name) + } else if cau.State.Network.Name != tn.network.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. + t.Fatalf("expected network to be %q, got %q", tn.network.Name, cau.State.Network.Name) } } } func TestConstructSiacoins(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV1TestNode(t, log, types.Siacoins(100), false) + c := tn.client - n, genesisBlock := testutil.V1Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() + senderAddr := tn.fundingAddr() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -681,7 +698,7 @@ func TestConstructSiacoins(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // try to construct a valid transaction with no spend policy _, err = wc.Construct([]types.SiacoinOutput{ @@ -693,10 +710,8 @@ func TestConstructSiacoins(t *testing.T) { // add the spend policy err = wc.AddAddress(wallet.Address{ - Address: senderAddr, - SpendPolicy: &types.SpendPolicy{ - Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey())), - }, + Address: senderAddr, + SpendPolicy: &senderPolicy, }) if err != nil { t.Fatal(err) @@ -739,11 +754,7 @@ func TestConstructSiacoins(t *testing.T) { } // sign the transaction - for i, sig := range resp.Transaction.Signatures { - sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) - sig := senderPrivateKey.SignHash(sigHash) - resp.Transaction.Signatures[i].Signature = sig[:] - } + signV1Txn(cs, &resp.Transaction, senderPrivateKey) if broadcastResp, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) @@ -769,7 +780,7 @@ func TestConstructSiacoins(t *testing.T) { case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -790,24 +801,14 @@ func TestConstructSiacoins(t *testing.T) { func TestConstructSiafunds(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV1TestNode(t, log, types.Siacoins(100), true) // siafunds=true + c := tn.client - n, genesisBlock := testutil.V1Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() + senderAddr := tn.fundingAddr() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -828,7 +829,7 @@ func TestConstructSiafunds(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) resp, err := wc.Construct(nil, []types.SiafundOutput{ {Value: 1, Address: receiverAddr}, @@ -860,11 +861,7 @@ func TestConstructSiafunds(t *testing.T) { } // sign the transaction - for i, sig := range resp.Transaction.Signatures { - sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) - sig := senderPrivateKey.SignHash(sigHash) - resp.Transaction.Signatures[i].Signature = sig[:] - } + signV1Txn(cs, &resp.Transaction, senderPrivateKey) if _, err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) @@ -887,7 +884,7 @@ func TestConstructSiafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -910,23 +907,14 @@ func TestConstructSiafunds(t *testing.T) { func TestConstructV2Siacoins(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), false) + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } + senderAddr := tn.fundingAddr() - cm := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cm, log) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -947,7 +935,7 @@ func TestConstructV2Siacoins(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // try to construct a transaction resp, err := wc.ConstructV2([]types.SiacoinOutput{ @@ -1003,11 +991,7 @@ func TestConstructV2Siacoins(t *testing.T) { } // sign the transaction - sigHash := cs.InputSigHash(resp.Transaction) - for i := range resp.Transaction.SiacoinInputs { - sig := senderPrivateKey.SignHash(sigHash) - resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} - } + signV2Txn(cs, &resp.Transaction, senderPrivateKey) if broadcastResp, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) @@ -1045,7 +1029,7 @@ func TestConstructV2Siacoins(t *testing.T) { t.Fatalf("expected unconfirmed event to have ID %q, got %q", sent.ID, unconfirmed[0].ID) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1066,24 +1050,14 @@ func TestConstructV2Siacoins(t *testing.T) { func TestConstructV2Siafunds(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), true) // siafunds=true + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() + senderAddr := tn.fundingAddr() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -1104,7 +1078,7 @@ func TestConstructV2Siafunds(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) resp, err := wc.ConstructV2(nil, []types.SiafundOutput{ {Value: 1, Address: receiverAddr}, @@ -1119,14 +1093,7 @@ func TestConstructV2Siafunds(t *testing.T) { } // sign the transaction - sigHash := cs.InputSigHash(resp.Transaction) - sig := senderPrivateKey.SignHash(sigHash) - for i := range resp.Transaction.SiafundInputs { - resp.Transaction.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} - } - for i := range resp.Transaction.SiafundInputs { - resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} - } + signV2Txn(cs, &resp.Transaction, senderPrivateKey) if _, err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { t.Fatal(err) @@ -1149,7 +1116,7 @@ func TestConstructV2Siafunds(t *testing.T) { case sent.SiafundOutflow()-sent.SiafundInflow() != 1: t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1173,27 +1140,17 @@ func TestConstructV2Siafunds(t *testing.T) { func TestSpentElement(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), true, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() + senderAddr := tn.fundingAddr() - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) // trigger initial scan - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) sce, basis, err := c.AddressSiacoinOutputs(senderAddr, false, 0, 100) if err != nil { @@ -1240,7 +1197,7 @@ func TestSpentElement(t *testing.T) { if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check if the element is spent spent, err = c.SpentSiacoinElement(sce[0].ID) @@ -1255,7 +1212,7 @@ func TestSpentElement(t *testing.T) { } // mine until the utxo is pruned - cn.MineBlocks(t, types.VoidAddress, 144) + tn.MineBlocks(t, types.VoidAddress, 144) _, err = c.SpentSiacoinElement(sce[0].ID) if !strings.Contains(err.Error(), "not found") { @@ -1308,7 +1265,7 @@ func TestSpentElement(t *testing.T) { if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // check if the element is spent spent, err = c.SpentSiafundElement(sfe[0].ID) @@ -1323,7 +1280,7 @@ func TestSpentElement(t *testing.T) { } // mine until the utxo is pruned - cn.MineBlocks(t, types.VoidAddress, 144) + tn.MineBlocks(t, types.VoidAddress, 144) _, err = c.SpentSiafundElement(sfe[0].ID) if !strings.Contains(err.Error(), "not found") { @@ -1333,13 +1290,10 @@ func TestSpentElement(t *testing.T) { func TestDebugMine(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testutil.V1Network() - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV1TestNode(t, log, types.ZeroCurrency, false) jc := jape.Client{ - BaseURL: c.BaseURL(), + BaseURL: tn.client.BaseURL(), Password: "password", } @@ -1350,9 +1304,9 @@ func TestDebugMine(t *testing.T) { if err != nil { t.Fatal(err) } - cn.WaitForSync(t) + tn.WaitForSync(t) - tip, err := c.ConsensusTip() + tip, err := tn.client.ConsensusTip() if err != nil { t.Fatal(err) } else if tip.Height != 5 { @@ -1449,12 +1403,10 @@ func TestAPISecurity(t *testing.T) { func TestAPINoContent(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testutil.V1Network() + tn := newV1TestNode(t, log, types.ZeroCurrency, false) + c := tn.client - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) - - buf, err := json.Marshal(cn.Chain.Tip().Height) + buf, err := json.Marshal(tn.Chain.Tip().Height) if err != nil { t.Fatal(err) } @@ -1476,10 +1428,8 @@ func TestAPINoContent(t *testing.T) { func TestV2TransactionUpdateBasis(t *testing.T) { log := zaptest.NewLogger(t) - n, genesisBlock := testutil.V2Network() - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + tn := newV2TestNode(t, log, types.ZeroCurrency, false) + c := tn.client // create a wallet w, err := c.AddWallet(api.WalletUpdateRequest{ @@ -1504,7 +1454,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { } // fund the wallet - cn.MineBlocks(t, addr, 5+int(n.MaturityDelay)) + tn.MineBlocks(t, addr, 5+int(tn.network.MaturityDelay)) resp, err := wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(100), Address: addr}, @@ -1529,7 +1479,7 @@ func TestV2TransactionUpdateBasis(t *testing.T) { if _, err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) // create a child transaction sce := parentTxn.EphemeralSiacoinOutput(0) @@ -1570,24 +1520,17 @@ func TestV2TransactionUpdateBasis(t *testing.T) { if _, err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { t.Fatal(err) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) } func TestAddressTPool(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), false, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client - pk := types.GeneratePrivateKey() + pk := tn.pk uc := types.StandardUnlockConditions(pk.PublicKey()) - addr1 := types.StandardUnlockHash(pk.PublicKey()) - - n, genesisBlock := testutil.V2Network() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: addr1, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + addr1 := tn.fundingAddr() assertSiacoinElement := func(t *testing.T, id types.SiacoinOutputID, value types.Currency, confirmations uint64) { t.Helper() @@ -1609,9 +1552,9 @@ func TestAddressTPool(t *testing.T) { t.Fatalf("expected siacoin element with ID %q not found", id) } - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) - airdropID := genesisBlock.Transactions[0].SiacoinOutputID(0) + airdropID := tn.genesis.Transactions[0].SiacoinOutputID(0) assertSiacoinElement(t, airdropID, types.Siacoins(100), 2) utxos, basis, err := c.AddressSiacoinOutputs(addr1, true, 0, 100) @@ -1655,28 +1598,22 @@ func TestAddressTPool(t *testing.T) { } assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 0) - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) assertSiacoinElement(t, txn.SiacoinOutputID(txn.ID(), 1), types.Siacoins(75), 1) } func TestEphemeralTransactions(t *testing.T) { log := zaptest.NewLogger(t) - pk := types.GeneratePrivateKey() + tn := newV2TestNode(t, log, types.Siacoins(100), false, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client + + pk := tn.pk sp := types.SpendPolicy{ Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), } - addr1 := sp.Address() + addr1 := tn.fundingAddr() - n, genesisBlock := testutil.V2Network() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: addr1, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) - - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) sces, basis, err := c.AddressSiacoinOutputs(addr1, true, 0, 100) if err != nil { @@ -1747,7 +1684,7 @@ func TestEphemeralTransactions(t *testing.T) { txn2.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} // mine a block so the basis is behind - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) sces, _, err = c.AddressSiacoinOutputs(addr1, true, 0, 100) if err != nil { @@ -1776,20 +1713,14 @@ func TestBroadcastRace(t *testing.T) { t.Skip("NDF") // TODO: fix log := zap.NewNop() - pk := types.GeneratePrivateKey() + tn := newV2TestNode(t, log, types.Siacoins(100000), false, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client + + pk := tn.pk sp := types.SpendPolicy{ Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), } - addr1 := sp.Address() - - n, genesisBlock := testutil.V2Network() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100000), - Address: addr1, - } - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + addr1 := tn.fundingAddr() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1800,7 +1731,7 @@ func TestBroadcastRace(t *testing.T) { case <-ctx.Done(): return default: - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) } } }() @@ -1855,23 +1786,14 @@ func TestBroadcastRace(t *testing.T) { func TestTxPoolOverwriteProofs(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), false, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } + senderAddr := tn.fundingAddr() - cm := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cm, log, wallet.WithIndexMode(wallet.IndexModeFull)) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -1889,7 +1811,7 @@ func TestTxPoolOverwriteProofs(t *testing.T) { if err != nil { t.Fatal(err) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) resp, err := wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(1), Address: receiverAddr}, @@ -1910,7 +1832,7 @@ func TestTxPoolOverwriteProofs(t *testing.T) { } // assert the transaction is valid - cs, ok := cm.Chain.State(resp.Basis.ID) + cs, ok := tn.Chain.State(resp.Basis.ID) if !ok { t.Fatal("failed to get state") } @@ -1952,7 +1874,7 @@ func TestTxPoolOverwriteProofs(t *testing.T) { case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -1973,23 +1895,14 @@ func TestTxPoolOverwriteProofs(t *testing.T) { func TestTxPoolOverwriteProofsEphemeral(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), false, wallet.WithIndexMode(wallet.IndexModeFull)) + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - receiverPrivateKey := types.GeneratePrivateKey() - receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} - receiverAddr := receiverPolicy.Address() + senderAddr := tn.fundingAddr() - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - - cm := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cm, log, wallet.WithIndexMode(wallet.IndexModeFull)) + receiverAddr := types.StandardUnlockHash(types.GeneratePrivateKey().PublicKey()) w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -2007,7 +1920,7 @@ func TestTxPoolOverwriteProofsEphemeral(t *testing.T) { if err != nil { t.Fatal(err) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) resp, err := wc.ConstructV2([]types.SiacoinOutput{ {Value: types.Siacoins(1), Address: senderAddr}, @@ -2065,7 +1978,7 @@ func TestTxPoolOverwriteProofsEphemeral(t *testing.T) { } else if len(unconfirmed) != 2 { t.Fatalf("expected 2 unconfirmed events, got %v", len(unconfirmed)) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) confirmed, err := wc.Events(0, 5) if err != nil { @@ -2077,26 +1990,10 @@ func TestTxPoolOverwriteProofsEphemeral(t *testing.T) { func TestWalletConfirmations(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV1TestNode(t, log, types.Siacoins(1), true) + c := tn.client - // create syncer - syncerListener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - defer syncerListener.Close() - - // create chain manager - n, genesisBlock := testutil.V1Network() - giftPrivateKey := types.GeneratePrivateKey() - giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(1), - Address: giftAddress, - } - genesisBlock.Transactions[0].SiafundOutputs[0].Address = giftAddress - - cn := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cn, log) + giftPrivateKey := tn.pk w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -2121,7 +2018,7 @@ func TestWalletConfirmations(t *testing.T) { c.Rescan(0) // send gift to wallet - giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + giftSCOID := tn.genesis.Transactions[0].SiacoinOutputID(0) txn := types.Transaction{ SiacoinInputs: []types.SiacoinInput{{ ParentID: giftSCOID, @@ -2131,17 +2028,17 @@ func TestWalletConfirmations(t *testing.T) { {Address: addr, Value: types.Siacoins(1)}, }, SiafundInputs: []types.SiafundInput{{ - ParentID: genesisBlock.Transactions[0].SiafundOutputID(0), + ParentID: tn.genesis.Transactions[0].SiafundOutputID(0), UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), }}, SiafundOutputs: []types.SiafundOutput{ - {Address: addr, Value: genesisBlock.Transactions[0].SiafundOutputs[0].Value}, + {Address: addr, Value: tn.genesis.Transactions[0].SiafundOutputs[0].Value}, }, Signatures: []types.TransactionSignature{{ ParentID: types.Hash256(giftSCOID), CoveredFields: types.CoveredFields{WholeTransaction: true}, }, { - ParentID: types.Hash256(genesisBlock.Transactions[0].SiafundOutputID(0)), + ParentID: types.Hash256(tn.genesis.Transactions[0].SiafundOutputID(0)), CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } @@ -2153,7 +2050,7 @@ func TestWalletConfirmations(t *testing.T) { sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] - sig2 := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(genesisBlock.Transactions[0].SiafundOutputID(0)), 0, 0, nil)) + sig2 := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(tn.genesis.Transactions[0].SiafundOutputID(0)), 0, 0, nil)) txn.Signatures[1].Signature = sig2[:] // broadcast the transaction to the transaction pool @@ -2162,7 +2059,7 @@ func TestWalletConfirmations(t *testing.T) { } // confirm the transaction - cn.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) assertConfirmations := func(t *testing.T, n uint64) { t.Helper() @@ -2172,8 +2069,8 @@ func TestWalletConfirmations(t *testing.T) { t.Fatal(err) } else if len(outputs) != 1 { t.Fatal("should have one UTXOs, got", len(outputs)) - } else if basis != cn.Chain.Tip() { - t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if basis != tn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", tn.Chain.Tip(), basis) } else if outputs[0].Confirmations != n { t.Fatalf("expected %d confirmation, got %v", n, outputs[0].Confirmations) } @@ -2183,33 +2080,26 @@ func TestWalletConfirmations(t *testing.T) { t.Fatal(err) } else if len(sfe) != 1 { t.Fatal("should have one siafund output, got", len(sfe)) - } else if basis != cn.Chain.Tip() { - t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) + } else if basis != tn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", tn.Chain.Tip(), basis) } else if sfe[0].Confirmations != n { t.Fatalf("expected %d confirmation, got %v", n, sfe[0].Confirmations) } } assertConfirmations(t, 1) - cn.MineBlocks(t, types.VoidAddress, 10) + tn.MineBlocks(t, types.VoidAddress, 10) assertConfirmations(t, 11) } func TestTxPoolAllowVoid(t *testing.T) { log := zaptest.NewLogger(t) + tn := newV2TestNode(t, log, types.Siacoins(100), false) + c := tn.client - n, genesisBlock := testutil.V2Network() - senderPrivateKey := types.GeneratePrivateKey() + senderPrivateKey := tn.pk senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} - senderAddr := senderPolicy.Address() - - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: types.Siacoins(100), - Address: senderAddr, - } - - cm := testutil.NewConsensusNode(t, n, genesisBlock, log) - c := startWalletServer(t, cm, log) + senderAddr := tn.fundingAddr() w, err := c.AddWallet(api.WalletUpdateRequest{ Name: "primary", @@ -2230,7 +2120,7 @@ func TestTxPoolAllowVoid(t *testing.T) { if err := c.Rescan(0); err != nil { t.Fatal(err) } - cm.MineBlocks(t, types.VoidAddress, 1) + tn.MineBlocks(t, types.VoidAddress, 1) sces, basis, err := wc.SiacoinOutputs(0, 100) if err != nil { From be504f08322447ad765f2116eb895370af80e256 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Tue, 6 Jan 2026 08:35:03 -0500 Subject: [PATCH 564/630] simplify test node creation functions --- api/api_test.go | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 859e52a..2d7d64a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -38,12 +38,11 @@ func (tn *testNode) fundingAddr() types.Address { return types.StandardUnlockHash(tn.pk.PublicKey()) } -// newV1TestNode creates a V1 network test node with initial siacoin funding. +// newCustomTestNode creates a test node with the given network and genesis block. // If sf is true, also assigns genesis siafunds to the funding address. -func newV1TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { +func newCustomTestNode(tb testing.TB, log *zap.Logger, n *consensus.Network, genesisBlock types.Block, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { tb.Helper() - n, genesisBlock := testutil.V1Network() fundingKey := types.GeneratePrivateKey() fundingAddr := types.StandardUnlockHash(fundingKey.PublicKey()) genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ @@ -64,30 +63,20 @@ func newV1TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, w } } +// newV1TestNode creates a V1 network test node with initial siacoin funding. +// If sf is true, also assigns genesis siafunds to the funding address. +func newV1TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { + tb.Helper() + n, genesisBlock := testutil.V1Network() + return newCustomTestNode(tb, log, n, genesisBlock, sc, sf, walletOpts...) +} + // newV2TestNode creates a V2 network test node with initial siacoin funding. // If sf is true, also assigns genesis siafunds to the funding address. func newV2TestNode(tb testing.TB, log *zap.Logger, sc types.Currency, sf bool, walletOpts ...wallet.Option) *testNode { tb.Helper() - n, genesisBlock := testutil.V2Network() - fundingKey := types.GeneratePrivateKey() - fundingAddr := types.StandardUnlockHash(fundingKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ - Value: sc, - Address: fundingAddr, - } - if sf { - genesisBlock.Transactions[0].SiafundOutputs[0].Address = fundingAddr - } - cn := testutil.NewConsensusNode(tb, n, genesisBlock, log) - c := startWalletServer(tb, cn, log, walletOpts...) - return &testNode{ - ConsensusNode: cn, - network: n, - client: c, - genesis: genesisBlock, - pk: fundingKey, - } + return newCustomTestNode(tb, log, n, genesisBlock, sc, sf, walletOpts...) } // signV1Txn signs all signatures in a V1 transaction. From 0628f87a4a3979e074359e8f9e4159c6a68e0fa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 20:35:29 +0000 Subject: [PATCH 565/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/coreutils` from 0.20.0 to 0.20.1 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.20.0...v0.20.1) Updates `golang.org/x/term` from 0.38.0 to 0.39.0 - [Commits](https://github.com/golang/term/compare/v0.38.0...v0.39.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.20.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.39.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 18 ++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index ccc87f8..9f0653f 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.33 go.sia.tech/core v0.19.0 - go.sia.tech/coreutils v0.20.0 + go.sia.tech/coreutils v0.20.1 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.35.0 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.38.0 + golang.org/x/term v0.39.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -20,7 +20,7 @@ require ( require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.1 // indirect + github.com/quic-go/quic-go v0.58.0 // indirect github.com/quic-go/webtransport-go v0.9.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect @@ -28,7 +28,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.39.0 // indirect + golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.39.0 // indirect ) diff --git a/go.sum b/go.sum index 3f33cc1..0f6745e 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.1 h1:25KAAR9QR8KZrCZRThWMKVAwGoiHIrNbT72ULHTuI10= -github.com/quic-go/quic-go v0.57.1/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/quic-go/quic-go v0.58.0 h1:ggY2pvZaVdB9EyojxL1p+5mptkuHyX5MOSv4dgWF4Ug= +github.com/quic-go/quic-go v0.58.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -26,8 +26,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= -go.sia.tech/coreutils v0.20.0 h1:7G6z+xL0VzwYueBV5S5jqzrmzdWQ5cL4NL2LAfv4PhE= -go.sia.tech/coreutils v0.20.0/go.mod h1:P/dgMZgZtpqs72HkiYafwXAcZnTLYhsg3TLq1mpvJJ4= +go.sia.tech/coreutils v0.20.1 h1:KrvR4BJohqgP3C+HPk/lIx6joTuatmif5q3+6lkJQgQ= +go.sia.tech/coreutils v0.20.1/go.mod h1:1UglfaKEcW3lwPkMOD6HbM9xQ+Qt4r+9vBdlok7kT/U= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -54,14 +54,12 @@ golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 30decca7fb9727b4756be4ddafb72174b24c3c08 Mon Sep 17 00:00:00 2001 From: Druffib <126788714+Druffib@users.noreply.github.com> Date: Wed, 14 Jan 2026 09:03:21 -0500 Subject: [PATCH 566/630] Update README.md Remove checkpoint from log.file field --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d9ca8fd..96e8738 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,6 @@ log: level: debug # override the global log level for the file path: /var/log/walletd.log format: json # human or JSON - checkpoint: 530000::0000000000000000abb98e3b587fba3a0c4e723ac1e078e9d6a4d13d1d131a2c ``` ## Building From 83867eeb3b238b1cee6a601fb31dd986ae64021c Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 1 Feb 2026 20:04:41 -0800 Subject: [PATCH 567/630] add utxo source --- ..._siacoin_input_origin_to_consensusblock.md | 5 + api/api.go | 81 +++++ api/api_test.go | 8 +- api/server.go | 29 +- cmd/walletd/node.go | 2 +- persist/sqlite/consensus.go | 139 +++++++- persist/sqlite/consensus_test.go | 320 ++++++++++++++++++ persist/sqlite/encoding.go | 17 + persist/sqlite/init.sql | 6 +- persist/sqlite/migrations.go | 8 + wallet/update.go | 180 ++++++++-- 11 files changed, 756 insertions(+), 39 deletions(-) create mode 100644 .changeset/added_siacoin_input_origin_to_consensusblock.md diff --git a/.changeset/added_siacoin_input_origin_to_consensusblock.md b/.changeset/added_siacoin_input_origin_to_consensusblock.md new file mode 100644 index 0000000..e9981df --- /dev/null +++ b/.changeset/added_siacoin_input_origin_to_consensusblock.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Added Siacoin input origin to consensus/block diff --git a/api/api.go b/api/api.go index f0c2218..83d6436 100644 --- a/api/api.go +++ b/api/api.go @@ -255,3 +255,84 @@ type ElementSpentResponse struct { type BatchAddressesRequest struct { Addresses []types.Address `json:"addresses"` } + +type ( + // ConsensusSiacoinInput represents a siacoin input along with its origin + // information. + ConsensusSiacoinInput struct { + ParentID types.SiacoinOutputID `json:"parentID"` + UnlockConditions types.UnlockConditions `json:"unlockConditions"` + + // analogous to txnid:vout in bitcoin + Origin wallet.SiacoinOrigin `json:"origin"` + } + + // ConsensusV2SiacoinInput represents a v2 siacoin input along with its origin + // information. + ConsensusV2SiacoinInput struct { + Parent types.SiacoinElement `json:"parent"` + SatisfiedPolicy types.SatisfiedPolicy `json:"satisfiedPolicy"` + + // analogous to txnid:vout in bitcoin + Origin wallet.SiacoinOrigin `json:"origin"` + } + + // ConsensusSiacoinOutput represents a siacoin output along with its ID. + ConsensusSiacoinOutput struct { + ID types.SiacoinOutputID `json:"id"` + Value types.Currency `json:"value"` + Address types.Address `json:"address"` + } + + // ConsensusTransaction represents a transaction along with its + // decorated inputs and outputs. + ConsensusTransaction struct { + ID types.TransactionID `json:"id"` + SiacoinInputs []ConsensusSiacoinInput `json:"siacoinInputs,omitempty"` + SiacoinOutputs []ConsensusSiacoinOutput `json:"siacoinOutputs,omitempty"` + FileContracts []types.FileContract `json:"fileContracts,omitempty"` + FileContractRevisions []types.FileContractRevision `json:"fileContractRevisions,omitempty"` + StorageProofs []types.StorageProof `json:"storageProofs,omitempty"` + SiafundInputs []types.SiafundInput `json:"siafundInputs,omitempty"` + SiafundOutputs []types.SiafundOutput `json:"siafundOutputs,omitempty"` + MinerFees []types.Currency `json:"minerFees,omitempty"` + ArbitraryData [][]byte `json:"arbitraryData,omitempty"` + Signatures []types.TransactionSignature `json:"signatures,omitempty"` + } + + // ConsensusV2Transaction represents a v2 transaction along with its + // decorated inputs and outputs. + ConsensusV2Transaction struct { + ID types.TransactionID `json:"id"` + SiacoinInputs []ConsensusV2SiacoinInput `json:"siacoinInputs,omitempty"` + SiacoinOutputs []ConsensusSiacoinOutput `json:"siacoinOutputs,omitempty"` + SiafundInputs []types.V2SiafundInput `json:"siafundInputs,omitempty"` + SiafundOutputs []types.SiafundOutput `json:"siafundOutputs,omitempty"` + FileContracts []types.V2FileContract `json:"fileContracts,omitempty"` + FileContractRevisions []types.V2FileContractRevision `json:"fileContractRevisions,omitempty"` + FileContractResolutions []types.V2FileContractResolution `json:"fileContractResolutions,omitempty"` + Attestations []types.Attestation `json:"attestations,omitempty"` + ArbitraryData []byte `json:"arbitraryData,omitempty"` + NewFoundationAddress *types.Address `json:"newFoundationAddress,omitempty"` + MinerFee types.Currency `json:"minerFee"` + } + + // ConsensusV2BlockData contains additional data for v2 blocks. + ConsensusV2BlockData struct { + Height uint64 `json:"height"` + Commitment types.Hash256 `json:"commitment"` + Transactions []ConsensusV2Transaction `json:"transactions"` + } + + // ConsensusBlock represents a block along with its decorated transactions. + ConsensusBlock struct { + ID types.BlockID `json:"id"` + ParentID types.BlockID `json:"parentID"` + Nonce uint64 `json:"nonce"` + Timestamp time.Time `json:"timestamp"` + MinerPayouts []types.SiacoinOutput `json:"minerPayouts"` + Transactions []ConsensusTransaction `json:"transactions"` + + V2 *ConsensusV2BlockData `json:"v2,omitempty"` + } +) diff --git a/api/api_test.go b/api/api_test.go index 2d7d64a..7621bca 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -116,7 +116,7 @@ func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logge tb.Cleanup(func() { wm.Close() }) server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(log)), + Handler: api.NewServer(cn.Store, cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(log)), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -139,7 +139,7 @@ func TestWalletAdd(t *testing.T) { return fmt.Errorf("expected wallet description to be %v, got %v", wr.Description, w.Description) } else if w.DateCreated.After(time.Now()) { return fmt.Errorf("expected wallet creation date to be in the past, got %v", w.DateCreated) - } else if isUpdate && w.DateCreated == w.LastUpdated { + } else if isUpdate && w.DateCreated.Equal(w.LastUpdated) { return fmt.Errorf("expected wallet last updated date to be after creation %v, got %v", w.DateCreated, w.LastUpdated) } @@ -1321,7 +1321,7 @@ func TestAPISecurity(t *testing.T) { defer httpListener.Close() server := &http.Server{ - Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + Handler: api.NewServer(cn.Store, cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, } @@ -1329,7 +1329,7 @@ func TestAPISecurity(t *testing.T) { go server.Serve(httpListener) replaceHandler := func(apiOpts ...api.ServerOption) { - server.Handler = api.NewServer(cn.Chain, cn.Syncer, wm, apiOpts...) + server.Handler = api.NewServer(cn.Store, cn.Chain, cn.Syncer, wm, apiOpts...) } // create a client with correct credentials diff --git a/api/server.go b/api/server.go index 99eb53a..ad84195 100644 --- a/api/server.go +++ b/api/server.go @@ -87,6 +87,11 @@ type ( BroadcastV2BlockOutline(bo gateway.V2BlockOutline) error } + // A Store provides access to persistent storage. + Store interface { + DecorateConsensusBlock(types.Block) (ConsensusBlock, error) + } + // A WalletManager manages wallets, keyed by name. WalletManager interface { Health() error @@ -160,10 +165,11 @@ type server struct { publicEndpoints bool password string - log *zap.Logger - cm ChainManager - s Syncer - wm WalletManager + log *zap.Logger + cm ChainManager + s Syncer + wm WalletManager + store Store scanMu sync.Mutex // for resubscribe scanInProgress bool @@ -266,7 +272,11 @@ func (s *server) consensusBlocksIDHandler(jc jape.Context) { jc.Error(errors.New("couldn't find block"), http.StatusNotFound) return } - jc.Encode(block) + cb, err := s.store.DecorateConsensusBlock(block) + if jc.Check("couldn't decorate block", err) != nil { + return + } + jc.Encode(cb) } func (s *server) consensusIndexHeightHandler(jc jape.Context) { @@ -1627,16 +1637,17 @@ func (s *server) pprofHandler(jc jape.Context) { } // NewServer returns an HTTP handler that serves the walletd API. -func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { +func NewServer(store Store, cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { srv := server{ log: zap.NewNop(), debugEnabled: false, publicEndpoints: false, startTime: time.Now(), - cm: cm, - s: s, - wm: wm, + cm: cm, + s: s, + wm: wm, + store: store, } for _, opt := range opts { opt(&srv) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 5edc79c..3873d6d 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -296,7 +296,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { if cfg.Debug { apiOpts = append(apiOpts, api.WithDebug()) } - api := api.NewServer(cm, s, wm, apiOpts...) + api := api.NewServer(store, cm, s, wm, apiOpts...) web := walletd.Handler() server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 833f4bf..d8a5409 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -9,6 +9,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/v2/api" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" ) @@ -561,7 +562,7 @@ func revertMatureSiacoinBalance(tx *txn, index types.ChainIndex) error { return nil } -func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { +func addSiacoinElements(tx *txn, elements []wallet.CreatedSiacoinElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { if len(elements) == 0 { return nil } @@ -579,7 +580,7 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, defer existsStmt.Close() // ignore elements already in the database. - insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) + insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) if err != nil { return fmt.Errorf("failed to prepare insert statement: %w", err) } @@ -606,7 +607,7 @@ func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, se.StateElement.MerkleProof = nil } - _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID, se.Origin.Source, encode(se.Origin.ID), se.Origin.Index) if err != nil { return fmt.Errorf("failed to execute statement: %w", err) } @@ -1440,3 +1441,135 @@ func addressRefStmt(tx *txn) (func(types.Address) (addressRef, error), func() er return ref, nil }, stmt.Close, nil } + +// DecorateConsensusBlock converts a types.Block into an api.ConsensusBlock by +// decorating its transactions with additional information such as siacoin input +// origins. +func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, error) { + cb := api.ConsensusBlock{ + ID: block.ID(), + ParentID: block.ParentID, + Nonce: block.Nonce, + Timestamp: block.Timestamp, + MinerPayouts: block.MinerPayouts, + Transactions: make([]api.ConsensusTransaction, 0, len(block.Transactions)), + } + + if block.V2 != nil { + cb.V2 = &api.ConsensusV2BlockData{ + Height: block.V2.Height, + Commitment: block.V2.Commitment, + Transactions: make([]api.ConsensusV2Transaction, 0, len(block.V2Transactions())), + } + } + + err := s.transaction(func(tx *txn) error { + stmt, err := tx.Prepare(`SELECT origin_source, origin_transaction_id, origin_transaction_index FROM siacoin_elements WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + getUTXOOrigin := func(id types.SiacoinOutputID) (wallet.SiacoinOrigin, error) { + var source sql.NullString + var originID nullDecodable[types.Hash256] + var index sql.NullInt64 + err := stmt.QueryRow(encode(id)).Scan(&source, &originID, &index) + if err != nil { + return wallet.SiacoinOrigin{}, fmt.Errorf("failed to query siacoin input source for %q: %w", id, err) + } else if !source.Valid || !originID.Valid || !index.Valid { + // don't allow partially null origins + return wallet.SiacoinOrigin{}, nil + } + return wallet.SiacoinOrigin{ + Source: source.String, + ID: originID.V, + Index: uint64(index.Int64), + }, nil + } + + for _, txn := range block.Transactions { + apiTx := api.ConsensusTransaction{ + ID: txn.ID(), + MinerFees: txn.MinerFees, + ArbitraryData: txn.ArbitraryData, + SiacoinInputs: make([]api.ConsensusSiacoinInput, 0, len(txn.SiacoinInputs)), + SiacoinOutputs: func() []api.ConsensusSiacoinOutput { + outputs := make([]api.ConsensusSiacoinOutput, 0, len(txn.SiacoinOutputs)) + for i, sco := range txn.SiacoinOutputs { + outputs = append(outputs, api.ConsensusSiacoinOutput{ + ID: txn.SiacoinOutputID(i), + Address: sco.Address, + Value: sco.Value, + }) + } + return outputs + }(), + SiafundInputs: txn.SiafundInputs, + SiafundOutputs: txn.SiafundOutputs, + FileContracts: txn.FileContracts, + Signatures: txn.Signatures, + } + + for _, sci := range txn.SiacoinInputs { + origin, err := getUTXOOrigin(sci.ParentID) + if err != nil { + return fmt.Errorf("failed to get siacoin input source for %q: %w", sci.ParentID, err) + } + + apiTx.SiacoinInputs = append(apiTx.SiacoinInputs, api.ConsensusSiacoinInput{ + ParentID: sci.ParentID, + UnlockConditions: sci.UnlockConditions, + Origin: origin, + }) + } + + cb.Transactions = append(cb.Transactions, apiTx) + } + + for _, txn := range block.V2Transactions() { + txnID := txn.ID() + apiTx := api.ConsensusV2Transaction{ + ID: txnID, + SiacoinInputs: make([]api.ConsensusV2SiacoinInput, 0, len(txn.SiacoinInputs)), + SiacoinOutputs: func() []api.ConsensusSiacoinOutput { + outputs := make([]api.ConsensusSiacoinOutput, 0, len(txn.SiacoinOutputs)) + for i, sco := range txn.SiacoinOutputs { + outputs = append(outputs, api.ConsensusSiacoinOutput{ + ID: txn.SiacoinOutputID(txnID, i), + Address: sco.Address, + Value: sco.Value, + }) + } + return outputs + }(), + SiafundInputs: txn.SiafundInputs, + SiafundOutputs: txn.SiafundOutputs, + FileContracts: txn.FileContracts, + FileContractRevisions: txn.FileContractRevisions, + FileContractResolutions: txn.FileContractResolutions, + Attestations: txn.Attestations, + ArbitraryData: txn.ArbitraryData, + NewFoundationAddress: txn.NewFoundationAddress, + MinerFee: txn.MinerFee, + } + + for _, sci := range txn.SiacoinInputs { + origin, err := getUTXOOrigin(sci.Parent.ID) + if err != nil { + return fmt.Errorf("failed to get siacoin input source for %q: %w", sci.Parent.ID, err) + } + + apiTx.SiacoinInputs = append(apiTx.SiacoinInputs, api.ConsensusV2SiacoinInput{ + Parent: sci.Parent, + SatisfiedPolicy: sci.SatisfiedPolicy, + Origin: origin, + }) + } + + cb.V2.Transactions = append(cb.V2.Transactions, apiTx) + } + return nil + }) + return cb, err +} diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index f0d7b80..a30c582 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -2,6 +2,7 @@ package sqlite import ( "path/filepath" + "strings" "testing" "go.sia.tech/core/consensus" @@ -300,3 +301,322 @@ func TestPruneSiafunds(t *testing.T) { syncDB(t, db, cm) assertUTXOs(0, 0) } + +func TestDecorateConsensusBlock(t *testing.T) { + db := newTestStore(t) + addr := types.VoidAddress + + t.Run("NullOriginFields", func(t *testing.T) { + // Manually insert a siacoin element without setting origin fields + // This simulates an older database or an element that doesn't have origin tracking + outputID := types.SiacoinOutputID{1, 2, 3} + value := types.Siacoins(100) + + err := db.transaction(func(tx *txn) error { + // First insert a chain index + var indexID int64 + err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, + encode(types.BlockID{}), 0).Scan(&indexID) + if err != nil { + return err + } + + // Insert an address + var addressID int64 + err = tx.QueryRow(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) + VALUES ($1, $2, $3, $4) RETURNING id`, + encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency), 0).Scan(&addressID) + if err != nil { + return err + } + + // Insert a siacoin element with NULL origin_transaction_id and origin_transaction_index. + // Hacky to use queries directly, but tests backwards compatibility with existing databases. + _, err = tx.Exec(`INSERT INTO siacoin_elements + (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL)`, + encode(outputID), encode(value), encode([]types.Hash256{}), 0, 0, addressID, true, indexID, "miner_payout") + return err + }) + if err != nil { + t.Fatal(err) + } + + // Create a block with a transaction that spends the element + pk := types.GeneratePrivateKey() + block := types.Block{ + Transactions: []types.Transaction{{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: outputID, + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + }}, + } + + // Call DecorateConsensusBlock - this should not error even though origin fields are NULL + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatalf("DecorateConsensusBlock failed with NULL origin fields: %v", err) + } + + // Verify the transaction was decorated + if len(decorated.Transactions) != 1 { + t.Fatalf("expected 1 transaction, got %d", len(decorated.Transactions)) + } + + // Verify the siacoin input was decorated + if len(decorated.Transactions[0].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.Transactions[0].SiacoinInputs)) + } + + // Verify the origin is empty (zero value) when origin fields are NULL + origin := decorated.Transactions[0].SiacoinInputs[0].Origin + if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { + t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) + } + }) + + t.Run("CompleteOriginFields", func(t *testing.T) { + // Test that elements with complete origin fields decorate correctly + outputID := types.SiacoinOutputID{4, 5, 6} + value := types.Siacoins(200) + originTxnID := types.TransactionID{7, 8, 9} + originIndex := uint64(2) + + err := db.transaction(func(tx *txn) error { + // Insert a chain index for the second element + var indexID int64 + err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, + encode(types.BlockID{1}), 1).Scan(&indexID) + if err != nil { + return err + } + + // Reuse the same address + var addressID int64 + err = tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&addressID) + if err != nil { + return err + } + + // Insert a siacoin element with ALL origin fields properly set + // This simulates what UpdateChainState would do for a transaction output + _, err = tx.Exec(`INSERT INTO siacoin_elements + (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + encode(outputID), encode(value), encode([]types.Hash256{}), 1, 0, addressID, true, indexID, "transaction", encode(originTxnID), originIndex) + return err + }) + if err != nil { + t.Fatal(err) + } + + // Create a block that spends the element with complete origin + pk := types.GeneratePrivateKey() + block := types.Block{ + Transactions: []types.Transaction{{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: outputID, + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + }}, + } + + // Decorate the block - should properly retrieve origin information + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatalf("DecorateConsensusBlock failed with complete origin fields: %v", err) + } + + // Verify the origin is properly populated + if len(decorated.Transactions) != 1 || len(decorated.Transactions[0].SiacoinInputs) != 1 { + t.Fatal("unexpected transaction structure") + } + + origin := decorated.Transactions[0].SiacoinInputs[0].Origin + if origin.Source != "transaction" { + t.Errorf("expected origin source 'transaction', got %q", origin.Source) + } + if origin.ID != types.Hash256(originTxnID) { + t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) + } + if origin.Index != originIndex { + t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) + } + }) + + t.Run("V2NullOriginFields", func(t *testing.T) { + // Test V2 transactions with NULL origin fields + outputID := types.SiacoinOutputID{10, 11, 12} + value := types.Siacoins(300) + + err := db.transaction(func(tx *txn) error { + var indexID int64 + err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, + encode(types.BlockID{2}), 2).Scan(&indexID) + if err != nil { + return err + } + + var addressID int64 + err = tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&addressID) + if err != nil { + return err + } + + _, err = tx.Exec(`INSERT INTO siacoin_elements + (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL)`, + encode(outputID), encode(value), encode([]types.Hash256{}), 2, 0, addressID, true, indexID, "miner_payout") + return err + }) + if err != nil { + t.Fatal(err) + } + + // Create a V2 block with a transaction that spends the element + block := types.Block{ + V2: &types.V2BlockData{ + Height: 1, + Commitment: types.Hash256{}, + Transactions: []types.V2Transaction{{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: types.SiacoinElement{ + ID: outputID, + SiacoinOutput: types.SiacoinOutput{ + Address: addr, + Value: value, + }, + }, + SatisfiedPolicy: types.SatisfiedPolicy{}, + }}, + }}, + }, + } + + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatalf("DecorateConsensusBlock failed with NULL origin fields on V2: %v", err) + } + + // Verify V2 transaction was decorated + if decorated.V2 == nil || len(decorated.V2.Transactions) != 1 { + t.Fatalf("expected 1 V2 transaction, got %d", len(decorated.V2.Transactions)) + } + + if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) + } + + // Verify the origin is empty for V2 transactions with NULL fields + origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin + if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { + t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) + } + }) + + t.Run("V2CompleteOriginFields", func(t *testing.T) { + // Test V2 transactions with complete origin fields + outputID := types.SiacoinOutputID{13, 14, 15} + value := types.Siacoins(400) + originTxnID := types.TransactionID{16, 17, 18} + originIndex := uint64(3) + + err := db.transaction(func(tx *txn) error { + var indexID int64 + err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, + encode(types.BlockID{3}), 3).Scan(&indexID) + if err != nil { + return err + } + + var addressID int64 + err = tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&addressID) + if err != nil { + return err + } + + _, err = tx.Exec(`INSERT INTO siacoin_elements + (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + encode(outputID), encode(value), encode([]types.Hash256{}), 3, 0, addressID, true, indexID, "transaction", encode(originTxnID), originIndex) + return err + }) + if err != nil { + t.Fatal(err) + } + + // Create a V2 block with a transaction that spends the element + block := types.Block{ + V2: &types.V2BlockData{ + Height: 2, + Commitment: types.Hash256{}, + Transactions: []types.V2Transaction{{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: types.SiacoinElement{ + ID: outputID, + SiacoinOutput: types.SiacoinOutput{ + Address: addr, + Value: value, + }, + }, + SatisfiedPolicy: types.SatisfiedPolicy{}, + }}, + }}, + }, + } + + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatalf("DecorateConsensusBlock failed with complete origin fields on V2: %v", err) + } + + // Verify V2 transaction was decorated + if decorated.V2 == nil || len(decorated.V2.Transactions) != 1 { + t.Fatalf("expected 1 V2 transaction, got %d", len(decorated.V2.Transactions)) + } + + if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) + } + + // Verify the origin is properly populated for V2 transactions + origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin + if origin.Source != "transaction" { + t.Errorf("expected origin source 'transaction', got %q", origin.Source) + } + if origin.ID != types.Hash256(originTxnID) { + t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) + } + if origin.Index != originIndex { + t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) + } + }) + + t.Run("MissingElement", func(t *testing.T) { + // Test error handling when element doesn't exist in database + nonExistentID := types.SiacoinOutputID{99, 99, 99} + + pk := types.GeneratePrivateKey() + block := types.Block{ + Transactions: []types.Transaction{{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: nonExistentID, + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + }}, + } + + // Should return an error when the element doesn't exist + _, err := db.DecorateConsensusBlock(block) + if err == nil { + t.Fatal("expected error when decorating block with missing element, got nil") + } + + // Verify it's the expected error about the missing element + expectedErrMsg := "failed to query siacoin input source" + if !strings.Contains(err.Error(), expectedErrMsg) { + t.Errorf("expected error containing %q, got %q", expectedErrMsg, err.Error()) + } + }) +} diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go index 48b9e25..79c1299 100644 --- a/persist/sqlite/encoding.go +++ b/persist/sqlite/encoding.go @@ -93,3 +93,20 @@ func (d *decodable) Scan(src any) error { func decode(obj any) sql.Scanner { return &decodable{obj} } + +type nullDecodable[T any] struct { + V T + Valid bool +} + +// Scan implements the sql.Scanner interface. +func (d *nullDecodable[T]) Scan(src any) error { + if src == nil { + d.Valid = false + return nil + } + + err := decode(&d.V).Scan(src) + d.Valid = err == nil + return err +} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql index d3afb4c..885eafc 100644 --- a/persist/sqlite/init.sql +++ b/persist/sqlite/init.sql @@ -23,7 +23,11 @@ CREATE TABLE siacoin_elements ( matured BOOLEAN NOT NULL, -- tracks whether the value has been added to the address balance chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), spent_index_id INTEGER REFERENCES chain_indices (id), -- soft delete - spent_event_id INTEGER REFERENCES events (id) -- atomic swap tracking + spent_event_id INTEGER REFERENCES events (id), -- atomic swap tracking + + origin_source TEXT NOT NULL DEFAULT 'unknown', -- source of the UTXO (e.g. miner payout, contract payout, foundation subsidy, transaction) + origin_transaction_id BLOB, -- transaction that created the UTXO if source is 'transaction' + origin_transaction_index INTEGER -- index of the output in the origin transaction (vout equivalent) ); CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go index 77a747e..3541578 100644 --- a/persist/sqlite/migrations.go +++ b/persist/sqlite/migrations.go @@ -7,6 +7,13 @@ import ( "go.uber.org/zap" ) +func migrateVersion9(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`ALTER TABLE siacoin_elements ADD COLUMN origin_source TEXT NOT NULL DEFAULT 'unknown'; +ALTER TABLE siacoin_elements ADD COLUMN origin_transaction_id BLOB; +ALTER TABLE siacoin_elements ADD COLUMN origin_transaction_index INTEGER;`) + return err +} + func migrateVersion8(tx *txn, _ *zap.Logger) error { _, err := tx.Exec(`CREATE TABLE signing_keys ( public_key BLOB PRIMARY KEY, @@ -209,4 +216,5 @@ var migrations = []func(tx *txn, log *zap.Logger) error{ migrateVersion6, migrateVersion7, migrateVersion8, + migrateVersion9, } diff --git a/wallet/update.go b/wallet/update.go index da9544d..4c91970 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -8,7 +8,31 @@ import ( "go.uber.org/zap" ) +const ( + // ElementSourceTransaction indicates that a siacoin element originated + // from a transaction output. + ElementSourceTransaction SiacoinElementSource = "transaction" + // ElementSourceMiner indicates that a siacoin element originated from a + // miner payout. + ElementSourceMiner SiacoinElementSource = "minerPayout" + // ElementSourceContract indicates that a siacoin element originated from a + // file contract output. + ElementSourceContract SiacoinElementSource = "contractPayout" + // ElementSourceSiafund indicates that a siacoin element originated from a + // siafund claim. + ElementSourceSiafund SiacoinElementSource = "siafundClaim" + // ElementSourceFoundationSubsidy indicates that a siacoin element originated + // from a foundation subsidy payout. + ElementSourceFoundationSubsidy SiacoinElementSource = "foundationSubsidy" + // ElementSourceUnknown indicates that the source of a siacoin element is + // unknown. + ElementSourceUnknown SiacoinElementSource = "unknown" +) + type ( + // SiacoinElementSource indicates the source of a siacoin element. + SiacoinElementSource = string + // A stateTreeUpdater is an interface for applying and reverting // Merkle tree updates. stateTreeUpdater interface { @@ -27,6 +51,14 @@ type ( Balance } + // A SiacoinOrigin is analogous to txnid:vout in Bitcoin, indicating the + // origin of a siacoin output. + SiacoinOrigin struct { + Source string + ID types.Hash256 `json:"id"` + Index uint64 `json:"index"` + } + // SpentSiacoinElement pairs a spent siacoin element with the ID of the // transaction that spent it. SpentSiacoinElement struct { @@ -41,12 +73,19 @@ type ( EventID types.TransactionID } + // CreatedSiacoinElement pairs a created siacoin element with its source + // and the ID of the transaction that created it. + CreatedSiacoinElement struct { + types.SiacoinElement + Origin SiacoinOrigin + } + // AppliedState contains all state changes made to a store after applying a chain // update. AppliedState struct { NumLeaves uint64 Events []Event - CreatedSiacoinElements []types.SiacoinElement + CreatedSiacoinElements []CreatedSiacoinElement SpentSiacoinElements []SpentSiacoinElement CreatedSiafundElements []types.SiafundElement SpentSiafundElements []SpentSiafundElement @@ -106,14 +145,29 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e NumLeaves: cau.State.Elements.NumLeaves, } + scoOrigins := make(map[types.SiacoinOutputID]SiacoinOrigin) spentEventIDs := make(map[types.Hash256]types.TransactionID) for _, txn := range cau.Block.Transactions { txnID := txn.ID() for _, input := range txn.SiacoinInputs { spentEventIDs[types.Hash256(input.ParentID)] = txnID } - for _, input := range txn.SiafundInputs { + for i, input := range txn.SiafundInputs { spentEventIDs[types.Hash256(input.ParentID)] = txnID + scoOrigins[input.ParentID.ClaimOutputID()] = SiacoinOrigin{ + Source: ElementSourceSiafund, + ID: types.Hash256(txnID), + Index: uint64(i), + } + } + // add sources for siacoin utxos + for i := range txn.SiacoinOutputs { + scoID := txn.SiacoinOutputID(i) + scoOrigins[scoID] = SiacoinOrigin{ + Source: ElementSourceTransaction, + ID: types.Hash256(txnID), + Index: uint64(i), + } } } for _, txn := range cau.Block.V2Transactions() { @@ -121,34 +175,91 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e for _, input := range txn.SiacoinInputs { spentEventIDs[types.Hash256(input.Parent.ID)] = txnID } - for _, input := range txn.SiafundInputs { + for i, input := range txn.SiafundInputs { spentEventIDs[types.Hash256(input.Parent.ID)] = txnID + scoOrigins[input.Parent.ID.ClaimOutputID()] = SiacoinOrigin{ + Source: ElementSourceSiafund, + ID: types.Hash256(txnID), + Index: uint64(i), + } + } + + // add sources for siacoin utxos + for i := range txn.SiacoinOutputs { + scoID := txn.SiacoinOutputID(txnID, i) + scoOrigins[scoID] = SiacoinOrigin{ + Source: ElementSourceTransaction, + ID: types.Hash256(txnID), + Index: uint64(i), + } } } - // add new siacoin elements to the store - for _, sced := range cau.SiacoinElementDiffs() { - sce := sced.SiacoinElement - if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { - continue - } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { - panic(err) - } else if !relevant { + // determine sources for miner payout utxos + blockID := cau.Block.ID() + for i := range cau.Block.MinerPayouts { + scoID := blockID.MinerOutputID(i) + scoOrigins[scoID] = SiacoinOrigin{ + Source: ElementSourceMiner, + ID: types.Hash256(blockID), + Index: uint64(i), + } + } + + // source for possible foundation subsidy utxo + scoOrigins[blockID.FoundationOutputID()] = SiacoinOrigin{ + Source: ElementSourceFoundationSubsidy, + ID: types.Hash256(blockID), + Index: 0, + } + // determine sources for file contract utxos + for _, diff := range cau.FileContractElementDiffs() { + if !diff.Resolved { continue } - if sced.Spent { - spentTxnID, ok := spentEventIDs[types.Hash256(sce.ID)] - if !ok { - panic(fmt.Errorf("missing transaction ID for spent siacoin element %v", sce.ID)) + + fce := diff.FileContractElement + if rev, ok := diff.RevisionElement(); ok { + fce = rev + } + + for i := range fce.FileContract.ValidProofOutputs { + scoID := fce.ID.ValidOutputID(i) + scoOrigins[scoID] = SiacoinOrigin{ + Source: ElementSourceContract, + ID: types.Hash256(fce.ID), + Index: uint64(i), } - applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, SpentSiacoinElement{ - SiacoinElement: sce, - EventID: spentTxnID, - }) - } else { - applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, sce) + } + for i := range fce.FileContract.MissedProofOutputs { + scoID := fce.ID.MissedOutputID(i) + scoOrigins[scoID] = SiacoinOrigin{ + Source: ElementSourceContract, + ID: types.Hash256(fce.ID), + Index: uint64(i), + } + } + } + + // determine sources for V2 file contract utxos + for _, diff := range cau.V2FileContractElementDiffs() { + if diff.Resolution != nil { + continue + } + + scoOrigins[diff.V2FileContractElement.ID.V2HostOutputID()] = SiacoinOrigin{ + Source: ElementSourceContract, + ID: types.Hash256(diff.V2FileContractElement.ID), + Index: 0, + } + scoOrigins[diff.V2FileContractElement.ID.V2RenterOutputID()] = SiacoinOrigin{ + Source: ElementSourceContract, + ID: types.Hash256(diff.V2FileContractElement.ID), + Index: 1, } } + + // add new siafund elements to the store for _, sfed := range cau.SiafundElementDiffs() { sfe := sfed.SiafundElement if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { @@ -172,6 +283,33 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e } } + // add new siacoin elements to the store + for _, sced := range cau.SiacoinElementDiffs() { + sce := sced.SiacoinElement + if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { + continue + } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { + panic(err) + } else if !relevant { + continue + } + if sced.Spent { + spentTxnID, ok := spentEventIDs[types.Hash256(sce.ID)] + if !ok { + panic(fmt.Errorf("missing transaction ID for spent siacoin element %v", sce.ID)) + } + applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, SpentSiacoinElement{ + SiacoinElement: sce, + EventID: spentTxnID, + }) + } else { + applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, CreatedSiacoinElement{ + SiacoinElement: sce, + Origin: scoOrigins[sce.ID], + }) + } + } + // add events relevant := func(addr types.Address) bool { relevant, err := tx.AddressRelevant(addr) From d227c0c69c6012dffaf7a2c9b10af437e77d9ba5 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 17:08:46 -0800 Subject: [PATCH 568/630] add thorough test --- api/api_test.go | 2 +- api/client.go | 4 +- persist/sqlite/consensus_test.go | 57 +-- wallet/update.go | 17 +- wallet/update_test.go | 749 +++++++++++++++++++++++++++++++ 5 files changed, 771 insertions(+), 58 deletions(-) create mode 100644 wallet/update_test.go diff --git a/api/api_test.go b/api/api_test.go index 7621bca..c5a7fad 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -583,7 +583,7 @@ func TestConsensus(t *testing.T) { b, err := c.ConsensusBlocksID(minedBlock.ID()) if err != nil { t.Fatal(err) - } else if b.ID() != minedBlock.ID() { + } else if b.ID != minedBlock.ID() { t.Fatal("mismatch") } } diff --git a/api/client.go b/api/client.go index eff6c70..3ff885e 100644 --- a/api/client.go +++ b/api/client.go @@ -106,13 +106,13 @@ func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { } // ConsensusBlocksID returns the block with the given id. -func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err error) { +func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp ConsensusBlock, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/blocks/%v", bid), &resp) return } // ConsensusBlocksHeight returns the block with the given height. -func (c *Client) ConsensusBlocksHeight(height uint64) (resp types.Block, err error) { +func (c *Client) ConsensusBlocksHeight(height uint64) (resp ConsensusBlock, err error) { err = c.c.GET(context.Background(), fmt.Sprintf("/consensus/blocks/%d", height), &resp) return } diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index a30c582..fdad7d2 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -307,13 +307,10 @@ func TestDecorateConsensusBlock(t *testing.T) { addr := types.VoidAddress t.Run("NullOriginFields", func(t *testing.T) { - // Manually insert a siacoin element without setting origin fields - // This simulates an older database or an element that doesn't have origin tracking outputID := types.SiacoinOutputID{1, 2, 3} value := types.Siacoins(100) err := db.transaction(func(tx *txn) error { - // First insert a chain index var indexID int64 err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, encode(types.BlockID{}), 0).Scan(&indexID) @@ -321,7 +318,6 @@ func TestDecorateConsensusBlock(t *testing.T) { return err } - // Insert an address var addressID int64 err = tx.QueryRow(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, $4) RETURNING id`, @@ -330,8 +326,7 @@ func TestDecorateConsensusBlock(t *testing.T) { return err } - // Insert a siacoin element with NULL origin_transaction_id and origin_transaction_index. - // Hacky to use queries directly, but tests backwards compatibility with existing databases. + // hacky to use queries directly, but tests backwards compatibility with existing databases. _, err = tx.Exec(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL)`, @@ -342,7 +337,7 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatal(err) } - // Create a block with a transaction that spends the element + // create a block with a transaction that spends the element pk := types.GeneratePrivateKey() block := types.Block{ Transactions: []types.Transaction{{ @@ -353,23 +348,17 @@ func TestDecorateConsensusBlock(t *testing.T) { }}, } - // Call DecorateConsensusBlock - this should not error even though origin fields are NULL decorated, err := db.DecorateConsensusBlock(block) if err != nil { t.Fatalf("DecorateConsensusBlock failed with NULL origin fields: %v", err) } - // Verify the transaction was decorated if len(decorated.Transactions) != 1 { t.Fatalf("expected 1 transaction, got %d", len(decorated.Transactions)) - } - - // Verify the siacoin input was decorated - if len(decorated.Transactions[0].SiacoinInputs) != 1 { + } else if len(decorated.Transactions[0].SiacoinInputs) != 1 { t.Fatalf("expected 1 siacoin input, got %d", len(decorated.Transactions[0].SiacoinInputs)) } - // Verify the origin is empty (zero value) when origin fields are NULL origin := decorated.Transactions[0].SiacoinInputs[0].Origin if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) @@ -377,14 +366,12 @@ func TestDecorateConsensusBlock(t *testing.T) { }) t.Run("CompleteOriginFields", func(t *testing.T) { - // Test that elements with complete origin fields decorate correctly outputID := types.SiacoinOutputID{4, 5, 6} value := types.Siacoins(200) originTxnID := types.TransactionID{7, 8, 9} originIndex := uint64(2) err := db.transaction(func(tx *txn) error { - // Insert a chain index for the second element var indexID int64 err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) RETURNING id`, encode(types.BlockID{1}), 1).Scan(&indexID) @@ -392,15 +379,13 @@ func TestDecorateConsensusBlock(t *testing.T) { return err } - // Reuse the same address var addressID int64 err = tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&addressID) if err != nil { return err } - // Insert a siacoin element with ALL origin fields properly set - // This simulates what UpdateChainState would do for a transaction output + // simulate UpdateChainState for a transaction output _, err = tx.Exec(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, @@ -411,7 +396,6 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatal(err) } - // Create a block that spends the element with complete origin pk := types.GeneratePrivateKey() block := types.Block{ Transactions: []types.Transaction{{ @@ -422,13 +406,11 @@ func TestDecorateConsensusBlock(t *testing.T) { }}, } - // Decorate the block - should properly retrieve origin information decorated, err := db.DecorateConsensusBlock(block) if err != nil { t.Fatalf("DecorateConsensusBlock failed with complete origin fields: %v", err) } - // Verify the origin is properly populated if len(decorated.Transactions) != 1 || len(decorated.Transactions[0].SiacoinInputs) != 1 { t.Fatal("unexpected transaction structure") } @@ -436,17 +418,14 @@ func TestDecorateConsensusBlock(t *testing.T) { origin := decorated.Transactions[0].SiacoinInputs[0].Origin if origin.Source != "transaction" { t.Errorf("expected origin source 'transaction', got %q", origin.Source) - } - if origin.ID != types.Hash256(originTxnID) { + } else if origin.ID != types.Hash256(originTxnID) { t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) - } - if origin.Index != originIndex { + } else if origin.Index != originIndex { t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) } }) t.Run("V2NullOriginFields", func(t *testing.T) { - // Test V2 transactions with NULL origin fields outputID := types.SiacoinOutputID{10, 11, 12} value := types.Siacoins(300) @@ -474,7 +453,6 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatal(err) } - // Create a V2 block with a transaction that spends the element block := types.Block{ V2: &types.V2BlockData{ Height: 1, @@ -499,16 +477,12 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatalf("DecorateConsensusBlock failed with NULL origin fields on V2: %v", err) } - // Verify V2 transaction was decorated if decorated.V2 == nil || len(decorated.V2.Transactions) != 1 { t.Fatalf("expected 1 V2 transaction, got %d", len(decorated.V2.Transactions)) - } - - if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { + } else if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) } - // Verify the origin is empty for V2 transactions with NULL fields origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) @@ -516,7 +490,6 @@ func TestDecorateConsensusBlock(t *testing.T) { }) t.Run("V2CompleteOriginFields", func(t *testing.T) { - // Test V2 transactions with complete origin fields outputID := types.SiacoinOutputID{13, 14, 15} value := types.Siacoins(400) originTxnID := types.TransactionID{16, 17, 18} @@ -546,7 +519,6 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatal(err) } - // Create a V2 block with a transaction that spends the element block := types.Block{ V2: &types.V2BlockData{ Height: 2, @@ -571,30 +543,23 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatalf("DecorateConsensusBlock failed with complete origin fields on V2: %v", err) } - // Verify V2 transaction was decorated if decorated.V2 == nil || len(decorated.V2.Transactions) != 1 { t.Fatalf("expected 1 V2 transaction, got %d", len(decorated.V2.Transactions)) - } - - if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { + } else if len(decorated.V2.Transactions[0].SiacoinInputs) != 1 { t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) } - // Verify the origin is properly populated for V2 transactions origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin if origin.Source != "transaction" { t.Errorf("expected origin source 'transaction', got %q", origin.Source) - } - if origin.ID != types.Hash256(originTxnID) { + } else if origin.ID != types.Hash256(originTxnID) { t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) - } - if origin.Index != originIndex { + } else if origin.Index != originIndex { t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) } }) t.Run("MissingElement", func(t *testing.T) { - // Test error handling when element doesn't exist in database nonExistentID := types.SiacoinOutputID{99, 99, 99} pk := types.GeneratePrivateKey() @@ -607,13 +572,11 @@ func TestDecorateConsensusBlock(t *testing.T) { }}, } - // Should return an error when the element doesn't exist _, err := db.DecorateConsensusBlock(block) if err == nil { t.Fatal("expected error when decorating block with missing element, got nil") } - // Verify it's the expected error about the missing element expectedErrMsg := "failed to query siacoin input source" if !strings.Contains(err.Error(), expectedErrMsg) { t.Errorf("expected error containing %q, got %q", expectedErrMsg, err.Error()) diff --git a/wallet/update.go b/wallet/update.go index 4c91970..5c1912e 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -24,9 +24,6 @@ const ( // ElementSourceFoundationSubsidy indicates that a siacoin element originated // from a foundation subsidy payout. ElementSourceFoundationSubsidy SiacoinElementSource = "foundationSubsidy" - // ElementSourceUnknown indicates that the source of a siacoin element is - // unknown. - ElementSourceUnknown SiacoinElementSource = "unknown" ) type ( @@ -54,7 +51,7 @@ type ( // A SiacoinOrigin is analogous to txnid:vout in Bitcoin, indicating the // origin of a siacoin output. SiacoinOrigin struct { - Source string + Source string `json:"source"` ID types.Hash256 `json:"id"` Index uint64 `json:"index"` } @@ -74,7 +71,7 @@ type ( } // CreatedSiacoinElement pairs a created siacoin element with its source - // and the ID of the transaction that created it. + // and an origin ID. CreatedSiacoinElement struct { types.SiacoinElement Origin SiacoinOrigin @@ -177,7 +174,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e } for i, input := range txn.SiafundInputs { spentEventIDs[types.Hash256(input.Parent.ID)] = txnID - scoOrigins[input.Parent.ID.ClaimOutputID()] = SiacoinOrigin{ + scoOrigins[input.Parent.ID.V2ClaimOutputID()] = SiacoinOrigin{ Source: ElementSourceSiafund, ID: types.Hash256(txnID), Index: uint64(i), @@ -243,7 +240,7 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e // determine sources for V2 file contract utxos for _, diff := range cau.V2FileContractElementDiffs() { - if diff.Resolution != nil { + if diff.Resolution == nil { continue } @@ -303,9 +300,13 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e EventID: spentTxnID, }) } else { + origin, ok := scoOrigins[sce.ID] + if !ok { + panic("missing origin for created siacoin element " + sce.ID.String()) + } applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, CreatedSiacoinElement{ SiacoinElement: sce, - Origin: scoOrigins[sce.ID], + Origin: origin, }) } } diff --git a/wallet/update_test.go b/wallet/update_test.go new file mode 100644 index 0000000..a95eab9 --- /dev/null +++ b/wallet/update_test.go @@ -0,0 +1,749 @@ +package wallet_test + +import ( + "testing" + + proto2 "go.sia.tech/core/rhp/v2" + proto4 "go.sia.tech/core/rhp/v4" + "go.sia.tech/core/types" + ctestutil "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/v2/internal/testutil" + "go.sia.tech/walletd/v2/wallet" + "lukechampine.com/frand" +) + +func TestDecorateBlock(t *testing.T) { + testOrigin := func(t *testing.T, tn *testNode, pk types.PrivateKey, uc types.UnlockConditions, expected wallet.SiacoinOrigin) { + t.Helper() + cm, db := tn.Chain, tn.Store + addr := uc.UnlockHash() + utxos, _, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatal("expected exactly one utxo") + } + + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + {ParentID: utxos[0].ID, UnlockConditions: uc}, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, txn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // check the last block contains the decorated input + block, ok := cm.Block(cm.Tip().ID) + if !ok { + t.Fatal("could not retrieve block") + } + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatal(err) + } else if len(decorated.Transactions) != 1 { + t.Fatalf("expected 1 transaction, got %d", len(decorated.Transactions)) + } else if len(decorated.Transactions[0].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.Transactions[0].SiacoinInputs)) + } else if decorated.Transactions[0].SiacoinInputs[0].Origin != expected { + t.Fatalf("expected origin %v, got %v", expected, decorated.Transactions[0].SiacoinInputs[0].Origin) + } + } + + testV2Origin := func(t *testing.T, tn *testNode, pk types.PrivateKey, sp types.SpendPolicy, expected wallet.SiacoinOrigin) { + t.Helper() + + cm, db := tn.Chain, tn.Store + addr := sp.Address() + utxos, tip, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatal("expected exactly one utxo") + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if _, err := cm.AddV2PoolTransactions(tip, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // check the last block contains the decorated input + block, ok := cm.Block(cm.Tip().ID) + if !ok { + t.Fatal("could not retrieve block") + } + + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatal(err) + } else if len(decorated.V2.Transactions) != 2 { + // one "coinbase" txn + the test txn + t.Fatalf("expected 2 transactions, got %d", len(decorated.V2.Transactions)) + } else if len(decorated.V2.Transactions[1].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) + } else if decorated.V2.Transactions[1].SiacoinInputs[0].Origin != expected { + t.Fatalf("expected origin %v, got %v", expected, decorated.V2.Transactions[1].SiacoinInputs[0].Origin) + } + } + + t.Run("transaction", func(t *testing.T) { + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr := uc.UnlockHash() + + network, genesisBlock := testutil.V1Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: types.VoidAddress, Value: types.Siacoins(2)}, + {Address: addr, Value: types.Siacoins(100)}, // gift output is index 2 + } + giftTxnID := genesisBlock.Transactions[0].ID() + + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + testOrigin(t, tn, pk, uc, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceTransaction, + ID: types.Hash256(giftTxnID), + Index: 2, + }) + }) + + t.Run("v2 transaction", func(t *testing.T) { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: types.VoidAddress, Value: types.Siacoins(2)}, + {Address: addr, Value: types.Siacoins(100)}, // gift output is index 2 + } + giftTxnID := genesisBlock.Transactions[0].ID() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceTransaction, + ID: types.Hash256(giftTxnID), + Index: 2, + }) + }) + + t.Run("miner", func(t *testing.T) { + // Create a UTXO from a miner payout + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesisBlock := testutil.V2Network() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, addr, 1) + waitForBlock(t, cm, db) + + minerBlock := cm.Tip() + + // mine until it matures + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceMiner, + ID: types.Hash256(minerBlock.ID), + Index: 0, + }) + }) + + t.Run("siafund", func(t *testing.T) { + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr := uc.UnlockHash() + + network, genesisBlock := testutil.V1Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1000)}, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + var sector [proto4.SectorSize]byte + frand.Read(sector[:]) + roots := []types.Hash256{proto4.SectorRoot(§or)} + + payout := types.Siacoins(500) + fc := types.FileContract{ + UnlockHash: addr, + Filesize: proto4.SectorSize, + FileMerkleRoot: proto2.MetaRoot(roots), + Payout: taxAdjustedPayout(payout), + WindowStart: cm.Tip().Height + 10, + WindowEnd: cm.Tip().Height + 20, + ValidProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + } + + fcTxn := types.Transaction{ + FileContracts: []types.FileContract{fc}, + } + + utxos, _, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) == 0 { + t.Fatal("expected at least one utxo") + } + + fcTxn.SiacoinInputs = []types.SiacoinInput{ + {ParentID: utxos[0].ID, UnlockConditions: uc}, + } + change := utxos[0].SiacoinOutput.Value.Sub(fc.Payout) + fcTxn.SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: change}, // burn the rest for easy testing + } + fcTxn.Signatures = []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + } + + sigHash := cm.TipState().WholeSigHash(fcTxn, fcTxn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + fcTxn.Signatures[0].Signature = sig[:] + + // confirm the contract + if _, err := cm.AddPoolTransactions([]types.Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // claim the siafund tax revenue + sfUtxos, _, err := tn.manager.AddressSiafundOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfUtxos) == 0 { + t.Fatal("expected at least one siafund utxo") + } + + claimTxn := types.Transaction{ + SiafundInputs: []types.SiafundInput{ + { + ParentID: sfUtxos[0].ID, + UnlockConditions: uc, + ClaimAddress: addr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sfUtxos[0].SiafundOutput.Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(sfUtxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash = cm.TipState().WholeSigHash(claimTxn, claimTxn.Signatures[0].ParentID, 0, 0, nil) + sig = pk.SignHash(sigHash) + claimTxn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{claimTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay+1)) + waitForBlock(t, cm, db) + + testOrigin(t, tn, pk, uc, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceSiafund, + ID: types.Hash256(claimTxn.ID()), + Index: 0, + }) + }) + + t.Run("v2 siafund", func(t *testing.T) { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesis := ctestutil.V2Network() + genesis.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1000)}, + } + genesis.Transactions[0].SiafundOutputs[0].Address = addr + tn := newTestNode(t, network, genesis, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + renterKey, hostKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + cs := cm.TipState() + + // generate tax revenue by creating and funding a file contract + fc := types.V2FileContract{ + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.Siacoins(250), + }, + RenterOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.Siacoins(250), + }, + RenterPublicKey: renterKey.PublicKey(), + HostPublicKey: hostKey.PublicKey(), + ProofHeight: cs.Index.Height + 10, + ExpirationHeight: cs.Index.Height + 20, + } + fc.RenterSignature = renterKey.SignHash(cs.ContractSigHash(fc)) + fc.HostSignature = hostKey.SignHash(cs.ContractSigHash(fc)) + + utxos, basis, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) == 0 { + t.Fatal("expected at least one utxo") + } + + fundAmount := types.Siacoins(500).Add(cs.V2FileContractTax(fc)) + fcTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(fundAmount)}, // burn the rest for easy testing + }, + FileContracts: []types.V2FileContract{fc}, + } + fcTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(fcTxn))} + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // claim the siafunds + sfUtxos, basis, err := tn.manager.AddressSiafundOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfUtxos) == 0 { + t.Fatal("expected at least one siafund utxo") + } + sfClaimTxn := types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfUtxos[0].SiafundElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + ClaimAddress: addr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sfUtxos[0].SiafundOutput.Value}, + }, + } + sfClaimTxn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(sfClaimTxn))} + + // mine until the claim matures + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{sfClaimTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)+1) + waitForBlock(t, cm, db) + + testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceSiafund, + ID: types.Hash256(sfClaimTxn.ID()), + Index: 0, + }) + }) + + t.Run("valid contract", func(t *testing.T) { + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr := uc.UnlockHash() + + network, genesisBlock := testutil.V1Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: types.VoidAddress, Value: types.Siacoins(2)}, + {Address: addr, Value: types.Siacoins(150)}, // gift output is index 2 + } + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + var sector [proto4.SectorSize]byte + frand.Read(sector[:]) + roots := []types.Hash256{proto4.SectorRoot(§or)} + + cs := cm.TipState() + fc := types.FileContract{ + UnlockHash: addr, + Filesize: proto4.SectorSize, + FileMerkleRoot: proto2.MetaRoot(roots), + Payout: taxAdjustedPayout(types.Siacoins(3)), + WindowStart: cm.Tip().Height + 10, + WindowEnd: cm.Tip().Height + 20, + ValidProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: addr, Value: types.Siacoins(2)}, // origin index 1 + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(3)}, + }, + } + + fcTxn := types.Transaction{ + FileContracts: []types.FileContract{fc}, + } + + utxos, _, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) == 0 { + t.Fatal("expected at least one utxo") + } + + fcTxn.SiacoinInputs = []types.SiacoinInput{ + {ParentID: utxos[0].ID, UnlockConditions: uc}, + } + change := utxos[0].SiacoinOutput.Value.Sub(fc.Payout) + fcTxn.SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: change}, // burn the rest for easy testing + } + fcTxn.Signatures = []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + } + + sigHash := cm.TipState().WholeSigHash(fcTxn, fcTxn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + fcTxn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + fcID := fcTxn.FileContractID(0) + + // mine until the proof window + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(fc.WindowStart-cm.Tip().Height-1)) + waitForBlock(t, cm, db) + + // submit a valid proof + index := cs.StorageProofLeafIndex(fc.Filesize, cm.Tip().ID, fcID) + sectorIndex := index / proto4.LeavesPerSector + leafIndex := index % proto4.LeavesPerSector + leafProof := proto2.ConvertProofOrdering(proto2.BuildProof(§or, leafIndex, leafIndex+1, nil), leafIndex) + sectorProof := proto2.ConvertProofOrdering(proto2.BuildSectorRangeProof(roots, sectorIndex, sectorIndex+1), sectorIndex) + proofTxn := types.Transaction{ + StorageProofs: []types.StorageProof{{ + ParentID: fcID, + Leaf: [64]byte(sector[leafIndex*proto4.LeafSize:][:proto4.LeafSize]), + Proof: append(leafProof, sectorProof...), + }}, + } + if _, err := cm.AddPoolTransactions([]types.Transaction{proofTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // mine until the payout matures + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + testOrigin(t, tn, pk, uc, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceContract, + ID: types.Hash256(fcID), + Index: 1, + }) + }) + + t.Run("missed contract", func(t *testing.T) { + pk := types.GeneratePrivateKey() + uc := types.StandardUnlockConditions(pk.PublicKey()) + addr := uc.UnlockHash() + + network, genesisBlock := testutil.V1Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: types.VoidAddress, Value: types.Siacoins(2)}, + {Address: addr, Value: types.Siacoins(150)}, // gift output is index 2 + } + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + var sector [proto4.SectorSize]byte + frand.Read(sector[:]) + roots := []types.Hash256{proto4.SectorRoot(§or)} + + fc := types.FileContract{ + UnlockHash: addr, + Filesize: proto4.SectorSize, + FileMerkleRoot: proto2.MetaRoot(roots), + Payout: taxAdjustedPayout(types.Siacoins(3)), + WindowStart: cm.Tip().Height + 10, + WindowEnd: cm.Tip().Height + 20, + ValidProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(3)}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1)}, + {Address: addr, Value: types.Siacoins(2)}, // origin index 1 + }, + } + + fcTxn := types.Transaction{ + FileContracts: []types.FileContract{fc}, + } + + utxos, _, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) == 0 { + t.Fatal("expected at least one utxo") + } + + fcTxn.SiacoinInputs = []types.SiacoinInput{ + {ParentID: utxos[0].ID, UnlockConditions: uc}, + } + change := utxos[0].SiacoinOutput.Value.Sub(fc.Payout) + fcTxn.SiacoinOutputs = []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: change}, // burn the rest for easy testing + } + fcTxn.Signatures = []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + } + + sigHash := cm.TipState().WholeSigHash(fcTxn, fcTxn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + fcTxn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + fcID := fcTxn.FileContractID(0) + + // mine until contract expires and output matures + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(fc.WindowEnd-cm.Tip().Height+network.MaturityDelay+1)) + waitForBlock(t, cm, db) + + testOrigin(t, tn, pk, uc, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceContract, + ID: types.Hash256(fcID), + Index: 1, + }) + }) + + t.Run("v2 contract", func(t *testing.T) { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesis := ctestutil.V2Network() + genesis.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1000)}, + } + genesis.Transactions[0].SiafundOutputs[0].Address = addr + tn := newTestNode(t, network, genesis, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + renterKey, hostKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + cs := cm.TipState() + + // generate tax revenue by creating and funding a file contract + fc := types.V2FileContract{ + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.Siacoins(250), + }, + RenterOutput: types.SiacoinOutput{ + Address: addr, // renter output is created regardless + Value: types.Siacoins(250), + }, + RenterPublicKey: renterKey.PublicKey(), + HostPublicKey: hostKey.PublicKey(), + ProofHeight: cs.Index.Height + 10, + ExpirationHeight: cs.Index.Height + 20, + } + fc.RenterSignature = renterKey.SignHash(cs.ContractSigHash(fc)) + fc.HostSignature = hostKey.SignHash(cs.ContractSigHash(fc)) + + utxos, basis, err := tn.manager.AddressSiacoinOutputs(addr, false, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) == 0 { + t.Fatal("expected at least one utxo") + } + + fundAmount := types.Siacoins(500).Add(cs.V2FileContractTax(fc)) + fcTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0].SiacoinElement, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(fundAmount)}, // burn the rest for easy testing + }, + FileContracts: []types.V2FileContract{fc}, + } + fcTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(fcTxn))} + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + // mine until the contract expires + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(fc.ExpirationHeight-cm.Tip().Height+1)) + waitForBlock(t, cm, db) + + // keep track of the file contract element + var fce *types.V2FileContractElement + _, applied, err := cm.UpdatesSince(types.ChainIndex{}, 100) + if err != nil { + t.Fatal(err) + } + for _, cau := range applied { + for _, diff := range cau.V2FileContractElementDiffs() { + if diff.Created { + fce = &diff.V2FileContractElement + } + } + if fce != nil { + cau.UpdateElementProof(&fce.StateElement) + } + } + if fce == nil { + t.Fatal("could not find file contract element") + } + + // resolve the contract to get the payout utxo + resolveTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: *fce, + Resolution: &types.V2FileContractExpiration{}, + }, + }, + } + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolveTxn}); err != nil { + t.Fatal(err) + } + // mine until the payout matures + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)+1) + waitForBlock(t, cm, db) + + testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceContract, + ID: types.Hash256(fce.ID), + Index: 1, + }) + }) + + t.Run("foundation", func(t *testing.T) { + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesisBlock := testutil.V2Network() + network.HardforkFoundation.PrimaryAddress = addr + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + cm, db := tn.Chain, tn.Store + + // mine until the first subsidy + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.HardforkFoundation.Height)) + waitForBlock(t, cm, db) + + foundatonSubsidyID := cm.Tip().ID + + // mine until the first foundation subsidy matures + ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)+1) + waitForBlock(t, cm, db) + + testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ + Source: wallet.ElementSourceFoundationSubsidy, + ID: types.Hash256(foundatonSubsidyID), + Index: 0, + }) + }) +} From af66ac0819b6cee06bbb41c1037e8eb949d3132f Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 17:58:41 -0800 Subject: [PATCH 569/630] update openapi.yml --- openapi.yml | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 174 insertions(+), 4 deletions(-) diff --git a/openapi.yml b/openapi.yml index c7d4916..001e0eb 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1476,8 +1476,10 @@ components: Block: type: object - description: Block as returned by the walletd consensus endpoints. + description: Block as returned by the walletd consensus endpoints with origin information for inputs. properties: + id: + $ref: "#/components/schemas/BlockID" parentID: $ref: "#/components/schemas/BlockID" nonce: @@ -1493,10 +1495,10 @@ components: transactions: type: array items: - $ref: "#/components/schemas/Transaction" + $ref: "#/components/schemas/ConsensusTransaction" v2: - $ref: "#/components/schemas/V2BlockData" - required: [parentID, nonce, timestamp, minerPayouts, transactions] + $ref: "#/components/schemas/ConsensusV2BlockData" + required: [id, parentID, nonce, timestamp, minerPayouts, transactions] ConsensusCheckpointResponse: type: object @@ -1853,6 +1855,174 @@ components: $ref: "#/components/schemas/Address" required: [parentID, unlockConditions, address] + SiacoinOrigin: + type: object + description: Origin information for a siacoin output, analogous to txnid:vout in Bitcoin. + properties: + source: + type: string + description: The source type of the siacoin output. + enum: + - transaction + - minerPayout + - contractPayout + - siafundClaim + - foundationSubsidy + id: + $ref: "#/components/schemas/Hash256" + description: The ID of the source (transaction ID, block ID, or contract ID). + index: + type: integer + format: uint64 + description: The index of the output within the source. + required: [source, id, index] + + ConsensusSiacoinInput: + type: object + description: Siacoin input with origin information as returned by consensus block endpoints. + properties: + parentID: + $ref: "#/components/schemas/SiacoinOutputID" + unlockConditions: + $ref: "#/components/schemas/UnlockConditions" + origin: + $ref: "#/components/schemas/SiacoinOrigin" + required: [parentID, unlockConditions, origin] + + ConsensusV2SiacoinInput: + type: object + description: V2 siacoin input with origin information as returned by consensus block endpoints. + properties: + parent: + $ref: "#/components/schemas/SiacoinElement" + satisfiedPolicy: + $ref: "#/components/schemas/SatisfiedPolicy" + origin: + $ref: "#/components/schemas/SiacoinOrigin" + required: [parent, satisfiedPolicy, origin] + + ConsensusSiacoinOutput: + allOf: + - $ref: "#/components/schemas/SiacoinOutput" + - type: object + properties: + id: + $ref: "#/components/schemas/SiacoinOutputID" + required: [id] + + ConsensusTransaction: + type: object + description: V1 transaction with origin information as returned by consensus block endpoints. + properties: + id: + $ref: "#/components/schemas/TransactionID" + siacoinInputs: + type: array + items: + $ref: "#/components/schemas/ConsensusSiacoinInput" + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/ConsensusSiacoinOutput" + fileContracts: + type: array + items: + $ref: "#/components/schemas/FileContract" + fileContractRevisions: + type: array + items: + $ref: "#/components/schemas/FileContractRevision" + storageProofs: + type: array + items: + $ref: "#/components/schemas/StorageProof" + siafundInputs: + type: array + items: + $ref: "#/components/schemas/SiafundInput" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/SiafundOutput" + minerFees: + type: array + items: + $ref: "#/components/schemas/Currency" + arbitraryData: + type: array + description: Arbitrary data entries encoded as base64 strings. + items: + type: string + format: byte + signatures: + type: array + items: + $ref: "#/components/schemas/TransactionSignature" + required: [id] + + ConsensusV2Transaction: + type: object + description: V2 transaction with origin information as returned by consensus block endpoints. + properties: + id: + $ref: "#/components/schemas/TransactionID" + siacoinInputs: + type: array + items: + $ref: "#/components/schemas/ConsensusV2SiacoinInput" + siacoinOutputs: + type: array + items: + $ref: "#/components/schemas/ConsensusSiacoinOutput" + siafundInputs: + type: array + items: + $ref: "#/components/schemas/V2SiafundInput" + siafundOutputs: + type: array + items: + $ref: "#/components/schemas/SiafundOutput" + fileContracts: + type: array + items: + $ref: "#/components/schemas/V2FileContract" + fileContractRevisions: + type: array + items: + $ref: "#/components/schemas/V2FileContractRevision" + fileContractResolutions: + type: array + items: + $ref: "#/components/schemas/V2FileContractResolution" + attestations: + type: array + items: + $ref: "#/components/schemas/Attestation" + arbitraryData: + type: string + format: byte + description: Arbitrary data encoded as a base64 string. + newFoundationAddress: + $ref: "#/components/schemas/Address" + minerFee: + $ref: "#/components/schemas/Currency" + required: [id] + + ConsensusV2BlockData: + type: object + description: V2-specific block data with consensus transaction information. + properties: + height: + type: integer + format: uint64 + commitment: + $ref: "#/components/schemas/Hash256" + transactions: + type: array + items: + $ref: "#/components/schemas/ConsensusV2Transaction" + required: [height, commitment, transactions] + SiafundInput: type: object properties: From 8a52d6815183cb67396cefe5a2c3623a63356da8 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 18:10:28 -0800 Subject: [PATCH 570/630] handle unkown --- persist/sqlite/consensus.go | 6 ++- wallet/update.go | 2 + wallet/update_test.go | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index d8a5409..242d32c 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1475,11 +1475,13 @@ func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, e var originID nullDecodable[types.Hash256] var index sql.NullInt64 err := stmt.QueryRow(encode(id)).Scan(&source, &originID, &index) - if err != nil { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return wallet.SiacoinOrigin{}, fmt.Errorf("failed to query siacoin input source for %q: %w", id, err) } else if !source.Valid || !originID.Valid || !index.Valid { // don't allow partially null origins - return wallet.SiacoinOrigin{}, nil + return wallet.SiacoinOrigin{ + Source: wallet.ElementSourceUnknown, + }, nil } return wallet.SiacoinOrigin{ Source: source.String, diff --git a/wallet/update.go b/wallet/update.go index 5c1912e..98d3f82 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -24,6 +24,8 @@ const ( // ElementSourceFoundationSubsidy indicates that a siacoin element originated // from a foundation subsidy payout. ElementSourceFoundationSubsidy SiacoinElementSource = "foundationSubsidy" + + ElementSourceUnknown SiacoinElementSource = "unknown" ) type ( diff --git a/wallet/update_test.go b/wallet/update_test.go index a95eab9..acedf26 100644 --- a/wallet/update_test.go +++ b/wallet/update_test.go @@ -746,4 +746,80 @@ func TestDecorateBlock(t *testing.T) { Index: 0, }) }) + + t.Run("unknown", func(t *testing.T) { + // this test is different because it needs to spend an + // element without the wallet manager tracking it. + pk := types.GeneratePrivateKey() + sp := types.PolicyPublicKey(pk.PublicKey()) + addr := sp.Address() + + network, genesisBlock := testutil.V2Network() + genesisBlock.Transactions[0].SiacoinOutputs = []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1000)}, + } + tn := newTestNode(t, network, genesisBlock) + cm, db := tn.Chain, tn.Store + + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + var sce *types.SiacoinElement + _, applied, err := cm.UpdatesSince(types.ChainIndex{}, 100) + if err != nil { + t.Fatal(err) + } + for _, cau := range applied { + for _, diff := range cau.SiacoinElementDiffs() { + if diff.Created && diff.SiacoinElement.SiacoinOutput.Address == addr { + sce = &diff.SiacoinElement + } + } + if sce != nil { + cau.UpdateElementProof(&sce.StateElement) + } + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: *sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + ctestutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + // check the last block contains the decorated input + block, ok := cm.Block(cm.Tip().ID) + if !ok { + t.Fatal("could not retrieve block") + } + + expected := wallet.SiacoinOrigin{ + Source: wallet.ElementSourceUnknown, + } + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatal(err) + } else if len(decorated.V2.Transactions) != 2 { + // one "coinbase" txn + the test txn + t.Fatalf("expected 2 transactions, got %d", len(decorated.V2.Transactions)) + } else if len(decorated.V2.Transactions[1].SiacoinInputs) != 1 { + t.Fatalf("expected 1 siacoin input, got %d", len(decorated.V2.Transactions[0].SiacoinInputs)) + } else if decorated.V2.Transactions[1].SiacoinInputs[0].Origin != expected { + t.Fatalf("expected origin %v, got %v", expected, decorated.V2.Transactions[1].SiacoinInputs[0].Origin) + } + }) } From 31c6112811ab868f73789cdd2f6dc4e95c8040aa Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 18:13:40 -0800 Subject: [PATCH 571/630] fix lint --- wallet/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/update.go b/wallet/update.go index 98d3f82..d2f1eb9 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -24,7 +24,7 @@ const ( // ElementSourceFoundationSubsidy indicates that a siacoin element originated // from a foundation subsidy payout. ElementSourceFoundationSubsidy SiacoinElementSource = "foundationSubsidy" - + // ElementSourceUnknown indicates that the source of a siacoin element is unknown. ElementSourceUnknown SiacoinElementSource = "unknown" ) From 716575cf7b890c04f057437cf07c1ce2bbeee863 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 18:20:51 -0800 Subject: [PATCH 572/630] fix test --- persist/sqlite/consensus_test.go | 58 ++++++++++++++++---------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index fdad7d2..e39c22d 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -2,7 +2,6 @@ package sqlite import ( "path/filepath" - "strings" "testing" "go.sia.tech/core/consensus" @@ -359,17 +358,21 @@ func TestDecorateConsensusBlock(t *testing.T) { t.Fatalf("expected 1 siacoin input, got %d", len(decorated.Transactions[0].SiacoinInputs)) } + expected := wallet.SiacoinOrigin{Source: wallet.ElementSourceUnknown} origin := decorated.Transactions[0].SiacoinInputs[0].Origin - if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { - t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) + if origin != expected { + t.Fatalf("expected origin %v, got %v", expected, origin) } }) t.Run("CompleteOriginFields", func(t *testing.T) { outputID := types.SiacoinOutputID{4, 5, 6} value := types.Siacoins(200) - originTxnID := types.TransactionID{7, 8, 9} - originIndex := uint64(2) + expected := wallet.SiacoinOrigin{ + Source: wallet.ElementSourceTransaction, + ID: types.Hash256{7, 8, 9}, + Index: 2, + } err := db.transaction(func(tx *txn) error { var indexID int64 @@ -389,7 +392,7 @@ func TestDecorateConsensusBlock(t *testing.T) { _, err = tx.Exec(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - encode(outputID), encode(value), encode([]types.Hash256{}), 1, 0, addressID, true, indexID, "transaction", encode(originTxnID), originIndex) + encode(outputID), encode(value), encode([]types.Hash256{}), 1, 0, addressID, true, indexID, "transaction", encode(expected.ID), expected.Index) return err }) if err != nil { @@ -416,12 +419,8 @@ func TestDecorateConsensusBlock(t *testing.T) { } origin := decorated.Transactions[0].SiacoinInputs[0].Origin - if origin.Source != "transaction" { - t.Errorf("expected origin source 'transaction', got %q", origin.Source) - } else if origin.ID != types.Hash256(originTxnID) { - t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) - } else if origin.Index != originIndex { - t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) + if origin != expected { + t.Fatalf("expected origin %v, got %v", expected, origin) } }) @@ -484,16 +483,20 @@ func TestDecorateConsensusBlock(t *testing.T) { } origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin - if origin.Source != "" || origin.ID != (types.Hash256{}) || origin.Index != 0 { - t.Errorf("expected empty origin, got Source=%q, ID=%v, Index=%d", origin.Source, origin.ID, origin.Index) + expected := wallet.SiacoinOrigin{Source: wallet.ElementSourceUnknown} + if origin != expected { + t.Fatalf("expected origin %v, got %v", expected, origin) } }) t.Run("V2CompleteOriginFields", func(t *testing.T) { outputID := types.SiacoinOutputID{13, 14, 15} value := types.Siacoins(400) - originTxnID := types.TransactionID{16, 17, 18} - originIndex := uint64(3) + expected := wallet.SiacoinOrigin{ + Source: wallet.ElementSourceTransaction, + ID: types.Hash256{16, 17, 18}, + Index: 3, + } err := db.transaction(func(tx *txn) error { var indexID int64 @@ -512,7 +515,7 @@ func TestDecorateConsensusBlock(t *testing.T) { _, err = tx.Exec(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id, origin_source, origin_transaction_id, origin_transaction_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - encode(outputID), encode(value), encode([]types.Hash256{}), 3, 0, addressID, true, indexID, "transaction", encode(originTxnID), originIndex) + encode(outputID), encode(value), encode([]types.Hash256{}), 3, 0, addressID, true, indexID, "transaction", encode(expected.ID), expected.Index) return err }) if err != nil { @@ -550,16 +553,13 @@ func TestDecorateConsensusBlock(t *testing.T) { } origin := decorated.V2.Transactions[0].SiacoinInputs[0].Origin - if origin.Source != "transaction" { - t.Errorf("expected origin source 'transaction', got %q", origin.Source) - } else if origin.ID != types.Hash256(originTxnID) { - t.Errorf("expected origin ID %v, got %v", types.Hash256(originTxnID), origin.ID) - } else if origin.Index != originIndex { - t.Errorf("expected origin index %d, got %d", originIndex, origin.Index) + if origin != expected { + t.Fatalf("expected origin %v, got %v", expected, origin) } }) t.Run("MissingElement", func(t *testing.T) { + // in "personal" mode elements can be missing nonExistentID := types.SiacoinOutputID{99, 99, 99} pk := types.GeneratePrivateKey() @@ -572,14 +572,14 @@ func TestDecorateConsensusBlock(t *testing.T) { }}, } - _, err := db.DecorateConsensusBlock(block) - if err == nil { - t.Fatal("expected error when decorating block with missing element, got nil") + decorated, err := db.DecorateConsensusBlock(block) + if err != nil { + t.Fatal(err) } - expectedErrMsg := "failed to query siacoin input source" - if !strings.Contains(err.Error(), expectedErrMsg) { - t.Errorf("expected error containing %q, got %q", expectedErrMsg, err.Error()) + expected := wallet.SiacoinOrigin{Source: wallet.ElementSourceUnknown} + if decorated.Transactions[0].SiacoinInputs[0].Origin != expected { + t.Fatalf("expected origin %v, got %v", expected, decorated.Transactions[0].SiacoinInputs[0].Origin) } }) } From f985cb924e307d6a5438935cbd42aff1c3efa2b8 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 2 Feb 2026 18:26:31 -0800 Subject: [PATCH 573/630] address comments --- openapi.yml | 1 + persist/sqlite/consensus.go | 10 ++++++---- wallet/update_test.go | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/openapi.yml b/openapi.yml index 001e0eb..4fca9f1 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1868,6 +1868,7 @@ components: - contractPayout - siafundClaim - foundationSubsidy + - unknown id: $ref: "#/components/schemas/Hash256" description: The ID of the source (transaction ID, block ID, or contract ID). diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 242d32c..7534b28 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1507,10 +1507,12 @@ func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, e } return outputs }(), - SiafundInputs: txn.SiafundInputs, - SiafundOutputs: txn.SiafundOutputs, - FileContracts: txn.FileContracts, - Signatures: txn.Signatures, + SiafundInputs: txn.SiafundInputs, + SiafundOutputs: txn.SiafundOutputs, + FileContracts: txn.FileContracts, + FileContractRevisions: txn.FileContractRevisions, + StorageProofs: txn.StorageProofs, + Signatures: txn.Signatures, } for _, sci := range txn.SiacoinInputs { diff --git a/wallet/update_test.go b/wallet/update_test.go index acedf26..1ab3063 100644 --- a/wallet/update_test.go +++ b/wallet/update_test.go @@ -734,7 +734,7 @@ func TestDecorateBlock(t *testing.T) { ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.HardforkFoundation.Height)) waitForBlock(t, cm, db) - foundatonSubsidyID := cm.Tip().ID + foundationSubsidyID := cm.Tip().ID // mine until the first foundation subsidy matures ctestutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)+1) @@ -742,7 +742,7 @@ func TestDecorateBlock(t *testing.T) { testV2Origin(t, tn, pk, sp, wallet.SiacoinOrigin{ Source: wallet.ElementSourceFoundationSubsidy, - ID: types.Hash256(foundatonSubsidyID), + ID: types.Hash256(foundationSubsidyID), Index: 0, }) }) From d92b97cb2e2b919a4f47192dc9048cdf2dd8fbd6 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Feb 2026 11:03:54 -0800 Subject: [PATCH 574/630] remove element pruning --- cmd/walletd/node.go | 6 ++++-- persist/sqlite/consensus.go | 37 ------------------------------------- 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 3873d6d..9abd40f 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -177,6 +177,8 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { _, existsErr := os.Open(consensusDBPath) consensusExists := !errors.Is(existsErr, os.ErrNotExist) + chainOpts := []chain.ManagerOption{chain.WithLog(log.Named("chain"))} + var cm *chain.Manager if cfg.Checkpoint != (types.ChainIndex{}) && !consensusExists { log.Info("beginning instant sync", zap.Stringer("checkpoint", cfg.Checkpoint)) @@ -196,7 +198,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } - cm = chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) + cm = chain.NewManager(dbstore, tipState, chainOpts...) if err := store.SetCheckpoint(cfg.Checkpoint); err != nil { return fmt.Errorf("failed to set wallet db checkpoint: %w", err) } @@ -217,7 +219,7 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } - cm = chain.NewManager(dbstore, tipState, chain.WithLog(log.Named("chain"))) + cm = chain.NewManager(dbstore, tipState, chainOpts...) } syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 7534b28..37c68bc 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -196,25 +196,6 @@ func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain. return fmt.Errorf("failed to set last committed index: %w", err) } - // skip pruning if there are no applied updates - if len(applied) == 0 { - return nil - } - - if state.Index.Height > s.spentElementRetentionBlocks { - pruneHeight := state.Index.Height - s.spentElementRetentionBlocks - - siacoins, err := pruneSpentSiacoinElements(tx, pruneHeight) - if err != nil { - return fmt.Errorf("failed to cleanup siacoin elements: %w", err) - } - - siafunds, err := pruneSpentSiafundElements(tx, pruneHeight) - if err != nil { - return fmt.Errorf("failed to cleanup siafund elements: %w", err) - } - log.Debug("pruned elements", zap.Int64("siacoins", siacoins), zap.Int64("siafunds", siafunds), zap.Uint64("pruneHeight", pruneHeight)) - } return nil }) } @@ -1404,24 +1385,6 @@ func revertOrphans(tx *txn, index types.ChainIndex, log *zap.Logger) error { return err } -func pruneSpentSiacoinElements(tx *txn, height uint64) (removed int64, err error) { - const query = `DELETE FROM siacoin_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` - res, err := tx.Exec(query, height) - if err != nil { - return 0, fmt.Errorf("failed to query siacoin elements: %w", err) - } - return res.RowsAffected() -} - -func pruneSpentSiafundElements(tx *txn, height uint64) (removed int64, err error) { - const query = `DELETE FROM siafund_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` - res, err := tx.Exec(query, height) - if err != nil { - return 0, fmt.Errorf("failed to query siacoin elements: %w", err) - } - return res.RowsAffected() -} - func setGlobalState(tx *txn, index types.ChainIndex, numLeaves uint64) error { _, err := tx.Exec(`UPDATE global_settings SET last_indexed_height=$1, last_indexed_id=$2, element_num_leaves=$3`, index.Height, encode(index.ID), numLeaves) return err From 931919d97eb650538fa9b142ccce8146c5b6c3d8 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Feb 2026 11:06:24 -0800 Subject: [PATCH 575/630] fix tests --- api/api_test.go | 16 -------------- persist/sqlite/consensus_test.go | 36 ++------------------------------ 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index c5a7fad..c43e57b 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1200,14 +1200,6 @@ func TestSpentElement(t *testing.T) { t.Fatalf("expected siacoin element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) } - // mine until the utxo is pruned - tn.MineBlocks(t, types.VoidAddress, 144) - - _, err = c.SpentSiacoinElement(sce[0].ID) - if !strings.Contains(err.Error(), "not found") { - t.Fatalf("expected error to contain %q, got %q", "not found", err) - } - sfe, basis, err := c.AddressSiafundOutputs(senderAddr, false, 0, 100) if err != nil { t.Fatal(err) @@ -1267,14 +1259,6 @@ func TestSpentElement(t *testing.T) { } else if spent.Event.Type != wallet.EventTypeV2Transaction { t.Fatalf("expected siafund element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) } - - // mine until the utxo is pruned - tn.MineBlocks(t, types.VoidAddress, 144) - - _, err = c.SpentSiafundElement(sfe[0].ID) - if !strings.Contains(err.Error(), "not found") { - t.Fatalf("expected error to contain %q, got %q", "not found", err) - } } func TestDebugMine(t *testing.T) { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index e39c22d..442a210 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -47,7 +47,7 @@ func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { } } -func TestPruneSiacoins(t *testing.T) { +func TestSpendSiacoins(t *testing.T) { db := newTestStore(t, WithRetainSpentElements(20)) bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) @@ -164,25 +164,9 @@ func TestPruneSiacoins(t *testing.T) { // the utxo should now have 0 balance and 1 spent element assertBalance(types.ZeroCurrency, types.ZeroCurrency) assertUTXOs(1, 0) - - // mine until the element is pruned - for i := uint64(0); i < db.spentElementRetentionBlocks-1; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - assertUTXOs(1, 0) // check that the element is not pruned early - } - - // trigger the pruning - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - assertUTXOs(0, 0) } -func TestPruneSiafunds(t *testing.T) { +func TestSpendSiafunds(t *testing.T) { db := newTestStore(t) bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) @@ -283,22 +267,6 @@ func TestPruneSiafunds(t *testing.T) { // the utxo should now have 0 balance and 1 spent element assertBalance(0) assertUTXOs(1, 0) - - // mine until the element is pruned - for i := uint64(0); i < db.spentElementRetentionBlocks-1; i++ { - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) // check that the element is not pruned early - assertUTXOs(1, 0) - } - - // the spent element should now be pruned - if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { - t.Fatal(err) - } - syncDB(t, db, cm) - assertUTXOs(0, 0) } func TestDecorateConsensusBlock(t *testing.T) { From cbf498b4cacf33c753b02e475d37e3d5b28c2565 Mon Sep 17 00:00:00 2001 From: Nate Date: Tue, 3 Feb 2026 12:48:03 -0800 Subject: [PATCH 576/630] index blocks that are created and spent in the same block --- persist/sqlite/consensus.go | 13 ++++---- wallet/update.go | 60 +++++++++++++++++++++---------------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 37c68bc..39f17ce 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -127,18 +127,17 @@ func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState return fmt.Errorf("failed to add events: %w", err) } - if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { - return fmt.Errorf("failed to spend siacoin elements: %w", err) - } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, ut.indexMode, log.Named("addSiacoinElements")); err != nil { + if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, ut.indexMode, log.Named("addSiacoinElements")); err != nil { return fmt.Errorf("failed to add siacoin elements: %w", err) + } else if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { + return fmt.Errorf("failed to spend siacoin elements: %w", err) } - if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { - return fmt.Errorf("failed to spend siafund elements: %w", err) - } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, ut.indexMode, log.Named("addSiafundElements")); err != nil { + if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, ut.indexMode, log.Named("addSiafundElements")); err != nil { return fmt.Errorf("failed to add siafund elements: %w", err) + } else if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { + return fmt.Errorf("failed to spend siafund elements: %w", err) } - return nil } diff --git a/wallet/update.go b/wallet/update.go index d2f1eb9..0850766 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -261,13 +261,17 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e // add new siafund elements to the store for _, sfed := range cau.SiafundElementDiffs() { sfe := sfed.SiafundElement - if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { - continue - } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { + if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { panic(err) } else if !relevant { continue } + + // handle outputs that were created and spent in the same block + if sfed.Created { + applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, sfe) + } + if sfed.Spent { spentTxnID, ok := spentEventIDs[types.Hash256(sfe.ID)] if !ok { @@ -277,21 +281,30 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e SiafundElement: sfe, EventID: spentTxnID, }) - } else { - applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, sfe) } } // add new siacoin elements to the store for _, sced := range cau.SiacoinElementDiffs() { sce := sced.SiacoinElement - if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { - continue - } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { + if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { panic(err) } else if !relevant { continue } + + // handle outputs that were created and spent in the same block + if sced.Created { + origin, ok := scoOrigins[sce.ID] + if !ok { + panic("missing origin for created siacoin element " + sce.ID.String()) + } + applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, CreatedSiacoinElement{ + SiacoinElement: sce, + Origin: origin, + }) + } + if sced.Spent { spentTxnID, ok := spentEventIDs[types.Hash256(sce.ID)] if !ok { @@ -301,15 +314,6 @@ func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) e SiacoinElement: sce, EventID: spentTxnID, }) - } else { - origin, ok := scoOrigins[sce.ID] - if !ok { - panic("missing origin for created siacoin element " + sce.ID.String()) - } - applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, CreatedSiacoinElement{ - SiacoinElement: sce, - Origin: origin, - }) } } @@ -360,33 +364,37 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. for _, sced := range cru.SiacoinElementDiffs() { sce := sced.SiacoinElement - if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { - continue - } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { + if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { panic(err) } else if !relevant { continue } + if sced.Spent { - // re-add any spent siacoin elements + // unspend any spent siacoin elements reverted.UnspentSiacoinElements = append(reverted.UnspentSiacoinElements, sce) - } else { + } + + if sced.Created { // delete any created siacoin elements reverted.DeletedSiacoinElements = append(reverted.DeletedSiacoinElements, sce) } } for _, sfed := range cru.SiafundElementDiffs() { sfe := sfed.SiafundElement - if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { - continue - } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { + if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { panic(err) } else if !relevant { continue } + if sfed.Spent { + // unspend any spent siafund elements reverted.UnspentSiafundElements = append(reverted.UnspentSiafundElements, sfe) - } else { + } + + if sfed.Created { + // delete any created siafund elements reverted.DeletedSiafundElements = append(reverted.DeletedSiafundElements, sfe) } } From e4a5b3c70b0fba4c7af57385b1d777282b3c68ca Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Feb 2026 15:23:40 +0000 Subject: [PATCH 577/630] chore: prepare release 2.12.0 --- ...lag_for_instant_syncing_to_a_given_chain_index.md | 5 ----- ...sponse_body_of_walletsidconstructv2transaction.md | 5 ----- .../added_siacoin_input_origin_to_consensusblock.md | 5 ----- .../update_core_dependency_from_0175_to_0180.md | 5 ----- CHANGELOG.md | 12 ++++++++++++ go.mod | 2 +- 6 files changed, 13 insertions(+), 21 deletions(-) delete mode 100644 .changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md delete mode 100644 .changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md delete mode 100644 .changeset/added_siacoin_input_origin_to_consensusblock.md delete mode 100644 .changeset/update_core_dependency_from_0175_to_0180.md diff --git a/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md b/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md deleted file mode 100644 index ea0e8d8..0000000 --- a/.changeset/add_checkpoint_cli_flag_for_instant_syncing_to_a_given_chain_index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Add -checkpoint CLI flag for instant-syncing to a given chain index. diff --git a/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md b/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md deleted file mode 100644 index c2ea477..0000000 --- a/.changeset/add_inputsighash_to_response_body_of_walletsidconstructv2transaction.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Add `inputSigHash` to response body of `/wallets/:id/construct/v2/transaction` diff --git a/.changeset/added_siacoin_input_origin_to_consensusblock.md b/.changeset/added_siacoin_input_origin_to_consensusblock.md deleted file mode 100644 index e9981df..0000000 --- a/.changeset/added_siacoin_input_origin_to_consensusblock.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Added Siacoin input origin to consensus/block diff --git a/.changeset/update_core_dependency_from_0175_to_0180.md b/.changeset/update_core_dependency_from_0175_to_0180.md deleted file mode 100644 index 03cbcef..0000000 --- a/.changeset/update_core_dependency_from_0175_to_0180.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core dependency to v0.19.0 and coreutils dependency to v0.20.0. diff --git a/CHANGELOG.md b/CHANGELOG.md index 418d7ca..19658ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## 2.12.0 (2026-02-05) + +### Features + +- Add -checkpoint CLI flag for instant-syncing to a given chain index. +- Add `inputSigHash` to response body of `/wallets/:id/construct/v2/transaction` +- Added Siacoin input origin to consensus/block + +### Fixes + +- Update core dependency to v0.19.0 and coreutils dependency to v0.20.0. + ## 2.11.0 (2025-10-01) ### Features diff --git a/go.mod b/go.mod index 9f0653f..5e4fcfe 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.11.0 +module go.sia.tech/walletd/v2 // v2.12.0 go 1.24.3 From 0533cb150a0087b5528f857592953b88839d0808 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 18:50:51 +0000 Subject: [PATCH 578/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/coreutils` from 0.20.1 to 0.21.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.20.1...v0.21.0) Updates `golang.org/x/term` from 0.39.0 to 0.40.0 - [Commits](https://github.com/golang/term/compare/v0.39.0...v0.40.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.21.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.40.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 19 ++++++++++--------- go.sum | 42 ++++++++++++++++++++++-------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 9f0653f..da052ed 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ go 1.24.3 require ( github.com/mattn/go-sqlite3 v1.14.33 go.sia.tech/core v0.19.0 - go.sia.tech/coreutils v0.20.1 + go.sia.tech/coreutils v0.21.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.35.0 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.39.0 + golang.org/x/term v0.40.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -18,17 +18,18 @@ require ( ) require ( + github.com/dunglas/httpsfv v1.1.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.58.0 // indirect - github.com/quic-go/webtransport-go v0.9.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/quic-go/webtransport-go v0.10.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.46.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/text v0.32.0 // indirect - golang.org/x/tools v0.39.0 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/tools v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index 0f6745e..a8eb160 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= +github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -14,10 +16,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.58.0 h1:ggY2pvZaVdB9EyojxL1p+5mptkuHyX5MOSv4dgWF4Ug= -github.com/quic-go/quic-go v0.58.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/quic-go/webtransport-go v0.9.0 h1:jgys+7/wm6JarGDrW+lD/r9BGqBAmqY/ssklE09bA70= -github.com/quic-go/webtransport-go v0.9.0/go.mod h1:4FUYIiUc75XSsF6HShcLeXXYZJ9AGwo/xh3L8M/P1ao= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= +github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -26,8 +28,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= -go.sia.tech/coreutils v0.20.1 h1:KrvR4BJohqgP3C+HPk/lIx6joTuatmif5q3+6lkJQgQ= -go.sia.tech/coreutils v0.20.1/go.mod h1:1UglfaKEcW3lwPkMOD6HbM9xQ+Qt4r+9vBdlok7kT/U= +go.sia.tech/coreutils v0.21.0 h1:bYFFycd8xYEXv3z7NgZyjU3s2SJbc9DYCpgkzxtVXHM= +go.sia.tech/coreutils v0.21.0/go.mod h1:jm48w2gp6X5PJbWrMvU8YGBGf2VikoQz4hQv57gWwAQ= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -44,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 61619d913ddaf415a28e193682fa40641980f43f Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:38:13 +0100 Subject: [PATCH 579/630] changeset --- ...5_to_0180.md => update_core_dependency_from_0175_to_0210.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .changeset/{update_core_dependency_from_0175_to_0180.md => update_core_dependency_from_0175_to_0210.md} (90%) diff --git a/.changeset/update_core_dependency_from_0175_to_0180.md b/.changeset/update_core_dependency_from_0175_to_0210.md similarity index 90% rename from .changeset/update_core_dependency_from_0175_to_0180.md rename to .changeset/update_core_dependency_from_0175_to_0210.md index 03cbcef..7be2e02 100644 --- a/.changeset/update_core_dependency_from_0175_to_0180.md +++ b/.changeset/update_core_dependency_from_0175_to_0210.md @@ -2,4 +2,4 @@ default: patch --- -# Update core dependency to v0.19.0 and coreutils dependency to v0.20.0. +# Update core dependency to v0.19.0 and coreutils dependency to v0.21.0. From 3173aeb11d9626ce47c73eaef4e8a51c71e87149 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 12 Feb 2026 12:11:12 -0800 Subject: [PATCH 580/630] remove unused maps --- ...hemeral_and_created_maps_in_revert_path.md | 5 +++++ wallet/update.go | 21 ------------------- 2 files changed, 5 insertions(+), 21 deletions(-) create mode 100644 .changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md diff --git a/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md b/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md new file mode 100644 index 0000000..d59258f --- /dev/null +++ b/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Removed unused ephemeral and created maps in revert path diff --git a/wallet/update.go b/wallet/update.go index 0850766..b601d6c 100644 --- a/wallet/update.go +++ b/wallet/update.go @@ -341,27 +341,6 @@ func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types. NumLeaves: cru.State.Elements.NumLeaves, } - // determine which siacoin and siafund elements are ephemeral - // - // note: I thought we could use LeafIndex == EphemeralLeafIndex, but - // it seems to be set before the subscriber is called. - created := make(map[types.Hash256]bool) - ephemeral := make(map[types.Hash256]bool) - for _, txn := range cru.Block.Transactions { - for i := range txn.SiacoinOutputs { - created[types.Hash256(txn.SiacoinOutputID(i))] = true - } - for _, input := range txn.SiacoinInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - for i := range txn.SiafundOutputs { - created[types.Hash256(txn.SiafundOutputID(i))] = true - } - for _, input := range txn.SiafundInputs { - ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] - } - } - for _, sced := range cru.SiacoinElementDiffs() { sce := sced.SiacoinElement if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { From 8ecd41ae74cc5547b747bac28f1eecd5e26cacdc Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 12 Feb 2026 12:12:01 -0800 Subject: [PATCH 581/630] remove unused --- persist/sqlite/options.go | 8 -------- persist/sqlite/store.go | 6 ++---- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/persist/sqlite/options.go b/persist/sqlite/options.go index 2497731..15f9b7e 100644 --- a/persist/sqlite/options.go +++ b/persist/sqlite/options.go @@ -11,11 +11,3 @@ func WithLog(log *zap.Logger) Option { s.log = log } } - -// WithRetainSpentElements sets the number of blocks to retain -// spent elements. -func WithRetainSpentElements(blocks uint64) Option { - return func(s *Store) { - s.spentElementRetentionBlocks = blocks - } -} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 557beb1..82b68ea 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -15,8 +15,7 @@ import ( type ( // A Store is a persistent store that uses a SQL database as its backend. Store struct { - indexMode wallet.IndexMode - spentElementRetentionBlocks uint64 // number of blocks to retain spent elements + indexMode wallet.IndexMode db *sql.DB log *zap.Logger @@ -86,8 +85,7 @@ func OpenDatabase(fp string, opts ...Option) (*Store, error) { store := &Store{ db: db, - log: zap.NewNop(), - spentElementRetentionBlocks: 144, // default to 144 blocks (1 day) + log: zap.NewNop(), } for _, opt := range opts { opt(store) From b8a7c1dde9b4935014eecd516d3450e01e396bd8 Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 12 Feb 2026 12:18:58 -0800 Subject: [PATCH 582/630] fix test --- persist/sqlite/consensus_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index 442a210..fdb858b 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -48,7 +48,7 @@ func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { } func TestSpendSiacoins(t *testing.T) { - db := newTestStore(t, WithRetainSpentElements(20)) + db := newTestStore(t) bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) if err != nil { From 267cfe877fbf1790da3cbafbe93c8b61b23c411a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Fri, 13 Feb 2026 10:20:24 +0100 Subject: [PATCH 583/630] fix knope workflow --- .github/workflows/prepare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 7b6ca01..efcaeff 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -19,7 +19,7 @@ jobs: run: | git config --global user.name github-actions[bot] git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - uses: knope-dev/action@v2.1.1 - run: knope prepare-release --verbose env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 74d01a98b504d43c5b14891b1dc4f8a84f9c0271 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Fri, 13 Feb 2026 10:54:18 +0100 Subject: [PATCH 584/630] use specific commit hash --- .github/workflows/prepare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index efcaeff..6911da2 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -19,7 +19,7 @@ jobs: run: | git config --global user.name github-actions[bot] git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: knope-dev/action@v2.1.1 + - uses: knope-dev/action@1ba8f6acf146130c3f5b196465018aa9f553381a - run: knope prepare-release --verbose env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4c51f05840cf52e896d8e659ca05c65e0d7530aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 17:37:18 +0000 Subject: [PATCH 585/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `github.com/mattn/go-sqlite3` from 1.14.33 to 1.14.34 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.33...v1.14.34) Updates `go.sia.tech/web/walletd` from 0.35.0 to 0.36.0 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.35.0...hostd@0.36.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.34 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.36.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index dd79a18..e629530 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.24.3 require ( - github.com/mattn/go-sqlite3 v1.14.33 + github.com/mattn/go-sqlite3 v1.14.34 go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.21.0 go.sia.tech/jape v0.14.1 - go.sia.tech/web/walletd v0.35.0 + go.sia.tech/web/walletd v0.36.0 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.40.0 diff --git a/go.sum b/go.sum index a8eb160..99930f0 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0= -github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= +github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -36,8 +36,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.35.0 h1:M1qrEIKPRhvrmojXPg1UTqi0MYr6FPd3VtO+6vgYOwc= -go.sia.tech/web/walletd v0.35.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.36.0 h1:/3rVFoQ55zxJikvrb6hz6N4zTl1FP5qL3j/8E8wf2a4= +go.sia.tech/web/walletd v0.36.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From 62abfbcad02c881a1992d6a831781b86ce467d09 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 25 Feb 2026 10:18:12 -0800 Subject: [PATCH 586/630] build: update to Go 1.26 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e629530..2f4f4c7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.sia.tech/walletd/v2 // v2.12.0 -go 1.24.3 +go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.34 From 0e3051831b53e12fa6dd7b3797f5804a3da21183 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 25 Feb 2026 10:29:50 -0800 Subject: [PATCH 587/630] add changeset for Go 1.26 update --- .changeset/update_go_to_1260.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_go_to_1260.md diff --git a/.changeset/update_go_to_1260.md b/.changeset/update_go_to_1260.md new file mode 100644 index 0000000..b1f573c --- /dev/null +++ b/.changeset/update_go_to_1260.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Update Go to 1.26.0. From 5f027b38d9de03b7fd936645a27be0671f62ce44 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 25 Feb 2026 10:38:39 -0800 Subject: [PATCH 588/630] build: update Dockerfile to Go 1.26 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a28e642..f5d677d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/golang:1.24 AS builder +FROM docker.io/library/golang:1.26 AS builder WORKDIR /walletd From ca19f6226a5a147f714bf9d0bc65e949a024d0e3 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 25 Feb 2026 11:43:17 -0800 Subject: [PATCH 589/630] deps: update coreutils to v0.21.1 Co-Authored-By: Claude Opus 4.6 --- .changeset/update_coreutils_to_v0211.md | 5 +++++ go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 17 deletions(-) create mode 100644 .changeset/update_coreutils_to_v0211.md diff --git a/.changeset/update_coreutils_to_v0211.md b/.changeset/update_coreutils_to_v0211.md new file mode 100644 index 0000000..8ea93cd --- /dev/null +++ b/.changeset/update_coreutils_to_v0211.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils from v0.21.0 to v0.21.1 diff --git a/go.mod b/go.mod index 2f4f4c7..fd6dc19 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.34 go.sia.tech/core v0.19.0 - go.sia.tech/coreutils v0.21.0 + go.sia.tech/coreutils v0.21.1 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.0 go.uber.org/zap v1.27.1 @@ -27,9 +27,9 @@ require ( go.sia.tech/mux v1.4.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.49.0 // indirect golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.33.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/tools v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 99930f0..de9c281 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= -go.sia.tech/coreutils v0.21.0 h1:bYFFycd8xYEXv3z7NgZyjU3s2SJbc9DYCpgkzxtVXHM= -go.sia.tech/coreutils v0.21.0/go.mod h1:jm48w2gp6X5PJbWrMvU8YGBGf2VikoQz4hQv57gWwAQ= +go.sia.tech/coreutils v0.21.1 h1:63uW8ohS280wsyg5zcYmNw006NUo2XnyVtpyN7GW8Q8= +go.sia.tech/coreutils v0.21.1/go.mod h1:nQyjMvBsi57G29w5zw/jWgAy3XL5PvvDi5RWxJBBEto= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From e422566d1d9da4db634a36c0ecdb3035f8ea59d4 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Thu, 26 Feb 2026 10:38:33 -0800 Subject: [PATCH 590/630] ci: remove openapi-publish workflow --- .github/workflows/openapi-publish.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/openapi-publish.yml diff --git a/.github/workflows/openapi-publish.yml b/.github/workflows/openapi-publish.yml deleted file mode 100644 index 8d073a6..0000000 --- a/.github/workflows/openapi-publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Publish OpenAPI to Scalar Registry - -permissions: - contents: read - -on: - push: - branches: [ master ] - paths: - - "openapi.yml" - workflow_dispatch: - -jobs: - publish: - uses: SiaFoundation/workflows/.github/workflows/publish-openapi.yml@master - with: - slug: walletd - spec_path: openapi.yml - docs_slug: walletd - secrets: - SCALAR_API_KEY: ${{ secrets.SCALAR_API_KEY }} - SCALAR_ACCESS_TOKEN: ${{ secrets.SCALAR_ACCESS_TOKEN }} From 6a82a353d7782fe601e43de48329bb327b100a7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:25:29 +0000 Subject: [PATCH 591/630] build(deps): bump go.sia.tech/web/walletd in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `go.sia.tech/web/walletd` from 0.36.0 to 0.36.1 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/hostd@0.36.0...walletd@0.36.1) --- updated-dependencies: - dependency-name: go.sia.tech/web/walletd dependency-version: 0.36.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fd6dc19..e8e864f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.21.1 go.sia.tech/jape v0.14.1 - go.sia.tech/web/walletd v0.36.0 + go.sia.tech/web/walletd v0.36.1 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.40.0 diff --git a/go.sum b/go.sum index de9c281..4d7b525 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.36.0 h1:/3rVFoQ55zxJikvrb6hz6N4zTl1FP5qL3j/8E8wf2a4= -go.sia.tech/web/walletd v0.36.0/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.36.1 h1:ytdK4nF7z8uq5Emcz/1wQB5o2oiT5uZDTgQK1drJgN0= +go.sia.tech/web/walletd v0.36.1/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From de939b1962078ea2918e6656274f331bec02b03b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:21:18 +0000 Subject: [PATCH 592/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.34 to 1.14.37 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.34...v1.14.37) Updates `golang.org/x/term` from 0.40.0 to 0.41.0 - [Commits](https://github.com/golang/term/compare/v0.40.0...v0.41.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.37 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.41.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index e8e864f..e82e13f 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.34 + github.com/mattn/go-sqlite3 v1.14.37 go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.21.1 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.1 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.40.0 + golang.org/x/term v0.41.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -29,7 +29,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 4d7b525..98c9ca7 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= -github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg= +github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -56,10 +56,10 @@ golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= From 99fd7718b660660455941bdb9c6ba630b50ebb9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 18:03:36 +0000 Subject: [PATCH 593/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.37 to 1.14.38 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.37...v1.14.38) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.38 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e82e13f..a459df9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.37 + github.com/mattn/go-sqlite3 v1.14.38 go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.21.1 go.sia.tech/jape v0.14.1 diff --git a/go.sum b/go.sum index 98c9ca7..417737b 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg= -github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.38 h1:tDUzL85kMvOrvpCt8P64SbGgVFtJB11GPi2AdmITgb4= +github.com/mattn/go-sqlite3 v1.14.38/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= From 66a8d22e7938332546e50c9382e9e86adee0aac0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 16:34:21 +0000 Subject: [PATCH 594/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [go.sia.tech/web/walletd](https://github.com/SiaFoundation/web). Updates `github.com/mattn/go-sqlite3` from 1.14.38 to 1.14.41 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.38...v1.14.41) Updates `go.sia.tech/web/walletd` from 0.36.1 to 0.36.2 - [Release notes](https://github.com/SiaFoundation/web/releases) - [Commits](https://github.com/SiaFoundation/web/compare/walletd@0.36.1...walletd@0.36.2) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.41 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/web/walletd dependency-version: 0.36.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a459df9..3ef3404 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.38 + github.com/mattn/go-sqlite3 v1.14.41 go.sia.tech/core v0.19.0 go.sia.tech/coreutils v0.21.1 go.sia.tech/jape v0.14.1 - go.sia.tech/web/walletd v0.36.1 + go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.41.0 diff --git a/go.sum b/go.sum index 417737b..0a9fe76 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.38 h1:tDUzL85kMvOrvpCt8P64SbGgVFtJB11GPi2AdmITgb4= -github.com/mattn/go-sqlite3 v1.14.38/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA= +github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -36,8 +36,8 @@ go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= -go.sia.tech/web/walletd v0.36.1 h1:ytdK4nF7z8uq5Emcz/1wQB5o2oiT5uZDTgQK1drJgN0= -go.sia.tech/web/walletd v0.36.1/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= +go.sia.tech/web/walletd v0.36.2 h1:yzQt9CYZw9A/t1/sQGW6va0v1iLd586S2z6+e6Hr6Vs= +go.sia.tech/web/walletd v0.36.2/go.mod h1:44AtA5QpfeeGBpephvPPiQTB2VWOUm/4Rv2sL9xbfYc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= From d58fcb4663beaa9dcaaee621f5e4e3cfa118ac52 Mon Sep 17 00:00:00 2001 From: Christopher Tarry Date: Thu, 2 Apr 2026 15:18:51 -0400 Subject: [PATCH 595/630] update action to use newer Node --- .github/workflows/prepare-release.yml | 2 +- .github/workflows/publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 6911da2..8fa7cfb 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -12,7 +12,7 @@ jobs: if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.2.2 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Configure Git diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4304162..e8f4951 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -32,7 +32,7 @@ jobs: needs: - publish steps: - - uses: actions/checkout@v4.2.2 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Download artifacts From 990214983639c84ef23b2baa5c1c48f43f2b443f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 17:50:27 +0000 Subject: [PATCH 596/630] build(deps): bump the all-dependencies group with 4 updates Bumps the all-dependencies group with 4 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), [go.sia.tech/core](https://github.com/SiaFoundation/core), [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.41 to 1.14.42 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.41...v1.14.42) Updates `go.sia.tech/core` from 0.19.0 to 0.19.1 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.19.0...v0.19.1) Updates `go.sia.tech/coreutils` from 0.21.1 to 0.21.2 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.21.1...v0.21.2) Updates `golang.org/x/term` from 0.41.0 to 0.42.0 - [Commits](https://github.com/golang/term/compare/v0.41.0...v0.42.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.42 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/core dependency-version: 0.19.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.21.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.42.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 20 ++++++++++---------- go.sum | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 3ef3404..2059979 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.41 - go.sia.tech/core v0.19.0 - go.sia.tech/coreutils v0.21.1 + github.com/mattn/go-sqlite3 v1.14.42 + go.sia.tech/core v0.19.1 + go.sia.tech/coreutils v0.21.2 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.27.1 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.41.0 + golang.org/x/term v0.42.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -24,12 +24,12 @@ require ( github.com/quic-go/quic-go v0.59.0 // indirect github.com/quic-go/webtransport-go v0.10.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect - go.sia.tech/mux v1.4.0 // indirect + go.sia.tech/mux v1.5.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.48.0 // indirect - golang.org/x/net v0.49.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.34.0 // indirect - golang.org/x/tools v0.41.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/tools v0.43.0 // indirect ) diff --git a/go.sum b/go.sum index 0a9fe76..1303ff5 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA= -github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= +github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -26,14 +26,14 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.19.0 h1:mj/lsixiI25hNTq1FzLHs94BCewTABulkqq2pHSHmdo= -go.sia.tech/core v0.19.0/go.mod h1:Gge/hpiE9m1ugPLz8RR1ZMoYZTPWLEdRWviHr/4rVeA= -go.sia.tech/coreutils v0.21.1 h1:63uW8ohS280wsyg5zcYmNw006NUo2XnyVtpyN7GW8Q8= -go.sia.tech/coreutils v0.21.1/go.mod h1:nQyjMvBsi57G29w5zw/jWgAy3XL5PvvDi5RWxJBBEto= +go.sia.tech/core v0.19.1 h1:rinLh0vuB4eBMVIVZ+Fmaym7B1nxw43I/v/SxYYPVeI= +go.sia.tech/core v0.19.1/go.mod h1:UYT4fwCjNx2wsY1cvGTjXVQPjCEa2LNEgFF3lbHjUVs= +go.sia.tech/coreutils v0.21.2 h1:Qxk6oarnS3LeSo3tdT2sIuRlMwvDXBfYavZECH9fUp0= +go.sia.tech/coreutils v0.21.2/go.mod h1:Pznmht8HgBHyutm0wtWcvUrqeRfs5iMzbJ7KC9Zx+nw= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= -go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= -go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= +go.sia.tech/mux v1.5.0 h1:6bQeO5y4AQPcf+UYHjhxpaNX+2V2wI5LIl0BbAg2YDA= +go.sia.tech/mux v1.5.0/go.mod h1:1/SlgVsLOUsca5tXxAEEwl+Ohm4B8te2YdRhpRGaUZw= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.36.2 h1:yzQt9CYZw9A/t1/sQGW6va0v1iLd586S2z6+e6Hr6Vs= @@ -46,24 +46,24 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 039cb685e383a3f860a3c880b22d93e5d205d543 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:32:52 +0000 Subject: [PATCH 597/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.19.1 to 0.20.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.19.1...v0.20.0) Updates `go.sia.tech/coreutils` from 0.21.2 to 0.21.3 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.21.2...v0.21.3) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.20.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.21.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 2059979..e4150d8 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.42 - go.sia.tech/core v0.19.1 - go.sia.tech/coreutils v0.21.2 + go.sia.tech/core v0.20.0 + go.sia.tech/coreutils v0.21.3 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.27.1 @@ -22,14 +22,14 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect - github.com/quic-go/webtransport-go v0.10.0 // indirect + github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.5.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.50.0 // indirect - golang.org/x/net v0.52.0 // indirect + golang.org/x/net v0.53.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect - golang.org/x/tools v0.43.0 // indirect + golang.org/x/tools v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index 1303ff5..1785728 100644 --- a/go.sum +++ b/go.sum @@ -18,18 +18,18 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/quic-go/webtransport-go v0.10.0 h1:LqXXPOXuETY5Xe8ITdGisBzTYmUOy5eSj+9n4hLTjHI= -github.com/quic-go/webtransport-go v0.10.0/go.mod h1:LeGIXr5BQKE3UsynwVBeQrU1TPrbh73MGoC6jd+V7ow= +github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c h1:qnILxGINaIzEFPrZVtfexAvKw5unmQV9PAvEuKYgp94= +github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c/go.mod h1:ocpwcCqYQbWRGNaCYlToTUVgjsbh0yEjLAyXl8yAIdA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.19.1 h1:rinLh0vuB4eBMVIVZ+Fmaym7B1nxw43I/v/SxYYPVeI= -go.sia.tech/core v0.19.1/go.mod h1:UYT4fwCjNx2wsY1cvGTjXVQPjCEa2LNEgFF3lbHjUVs= -go.sia.tech/coreutils v0.21.2 h1:Qxk6oarnS3LeSo3tdT2sIuRlMwvDXBfYavZECH9fUp0= -go.sia.tech/coreutils v0.21.2/go.mod h1:Pznmht8HgBHyutm0wtWcvUrqeRfs5iMzbJ7KC9Zx+nw= +go.sia.tech/core v0.20.0 h1:/KegmrjDgSdnJsbMavGfxShuGA8CfeRQQhulHpB6iys= +go.sia.tech/core v0.20.0/go.mod h1:nZsd0YjU6slPLkpz0rxLcMJJQTKyp0hxjNIKReF7wBQ= +go.sia.tech/coreutils v0.21.3 h1:VXLvRPHQOocxGR21HJP6s8B35MwOI9V0FP908VX/aEk= +go.sia.tech/coreutils v0.21.3/go.mod h1:jPFwxdUjURWSan9UgQkCFO0Zlt1bErLSu7v/l1tuhfg= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.0 h1:6bQeO5y4AQPcf+UYHjhxpaNX+2V2wI5LIl0BbAg2YDA= @@ -50,10 +50,10 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= -golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= @@ -62,8 +62,8 @@ golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 273d3c483f80c55b25d48a2683dd1b65b2de33e0 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 28 Apr 2026 10:17:54 +0200 Subject: [PATCH 598/630] changeset --- .changeset/update_core_to_0200.md | 5 +++++ .changeset/update_coreutils_to_0213.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/update_core_to_0200.md create mode 100644 .changeset/update_coreutils_to_0213.md diff --git a/.changeset/update_core_to_0200.md b/.changeset/update_core_to_0200.md new file mode 100644 index 0000000..b2ba2e0 --- /dev/null +++ b/.changeset/update_core_to_0200.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update core to 0.20.0. diff --git a/.changeset/update_coreutils_to_0213.md b/.changeset/update_coreutils_to_0213.md new file mode 100644 index 0000000..515cbbc --- /dev/null +++ b/.changeset/update_coreutils_to_0213.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils to 0.21.3. From 5d5d6fde2b8fd365e784e4d2928c8a968fadaa94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 21:05:00 +0000 Subject: [PATCH 599/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [go.uber.org/zap](https://github.com/uber-go/zap). Updates `github.com/mattn/go-sqlite3` from 1.14.42 to 1.14.44 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.42...v1.14.44) Updates `go.uber.org/zap` from 1.27.1 to 1.28.0 - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.27.1...v1.28.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.44 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.uber.org/zap dependency-version: 1.28.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e4150d8..35876bb 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.42 + github.com/mattn/go-sqlite3 v1.14.44 go.sia.tech/core v0.20.0 go.sia.tech/coreutils v0.21.3 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 - go.uber.org/zap v1.27.1 + go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/term v0.42.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 1785728..0a933b8 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= -github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= +github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -44,8 +44,10 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= From d5d4a5ee05da43bee6a07c45949b290937247c85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 00:02:19 +0000 Subject: [PATCH 600/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [golang.org/x/term](https://github.com/golang/term). Updates `go.sia.tech/core` from 0.20.0 to 0.21.0 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.20.0...v0.21.0) Updates `golang.org/x/term` from 0.42.0 to 0.43.0 - [Commits](https://github.com/golang/term/compare/v0.42.0...v0.43.0) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.21.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.43.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 35876bb..3cfc29f 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.44 - go.sia.tech/core v0.20.0 + go.sia.tech/core v0.21.0 go.sia.tech/coreutils v0.21.3 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.42.0 + golang.org/x/term v0.43.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -29,7 +29,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.50.0 // indirect golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/sys v0.44.0 // indirect golang.org/x/text v0.36.0 // indirect golang.org/x/tools v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index 0a933b8..c45e899 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.20.0 h1:/KegmrjDgSdnJsbMavGfxShuGA8CfeRQQhulHpB6iys= -go.sia.tech/core v0.20.0/go.mod h1:nZsd0YjU6slPLkpz0rxLcMJJQTKyp0hxjNIKReF7wBQ= +go.sia.tech/core v0.21.0 h1:BtZPT/UGPj5YjDmIA31fmZIm+fmqw/X4yshskr3cOwQ= +go.sia.tech/core v0.21.0/go.mod h1:b39iLfen0vRV3dkq1eSeBHDcaLFf89YPh2/TyBDY31Y= go.sia.tech/coreutils v0.21.3 h1:VXLvRPHQOocxGR21HJP6s8B35MwOI9V0FP908VX/aEk= go.sia.tech/coreutils v0.21.3/go.mod h1:jPFwxdUjURWSan9UgQkCFO0Zlt1bErLSu7v/l1tuhfg= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= @@ -58,10 +58,10 @@ golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= From 022fe377a0043ad6f03e23d4e97d73b9fe63614e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Fri, 15 May 2026 09:24:52 -0700 Subject: [PATCH 601/630] ci: pin GitHub Actions to SHA-256 commits Pin all action references to specific commit SHAs to prevent supply-chain attacks. Each pin is annotated with a human-readable version tag for clarity. Versions stay within the existing major version to avoid breaking changes. --- .github/workflows/main.yml | 2 +- .github/workflows/openapi-sync.yml | 2 +- .github/workflows/prepare-release.yml | 4 ++-- .github/workflows/project-add.yml | 2 +- .github/workflows/publish.yml | 6 +++--- .github/workflows/ui.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3e416e..e206c37 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,4 +10,4 @@ env: jobs: test: - uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master + uses: SiaFoundation/workflows/.github/workflows/go-test.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master diff --git a/.github/workflows/openapi-sync.yml b/.github/workflows/openapi-sync.yml index 2f2aade..84c4c73 100644 --- a/.github/workflows/openapi-sync.yml +++ b/.github/workflows/openapi-sync.yml @@ -11,6 +11,6 @@ on: jobs: sync: - uses: SiaFoundation/workflows/.github/workflows/sync-openapi-version.yml@master + uses: SiaFoundation/workflows/.github/workflows/sync-openapi-version.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master with: spec_path: openapi.yml diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 8fa7cfb..c7f146e 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -12,14 +12,14 @@ jobs: if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Configure Git run: | git config --global user.name github-actions[bot] git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com - - uses: knope-dev/action@1ba8f6acf146130c3f5b196465018aa9f553381a + - uses: knope-dev/action@19617851f9f13ab2f27a05989c55efb18aca3675 # v2.1.2 - run: knope prepare-release --verbose env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index 8b93101..42298ef 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -10,5 +10,5 @@ on: jobs: add-to-project: - uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master + uses: SiaFoundation/workflows/.github/workflows/project-add.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e8f4951..31f6e85 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,7 +16,7 @@ concurrency: jobs: publish: - uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master + uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master secrets: inherit with: linux-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' @@ -32,11 +32,11 @@ jobs: needs: - publish steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: fetch-depth: 0 - name: Download artifacts - uses: actions/download-artifact@v4.3.0 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts - name: Upload artifacts to release diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index a91153b..30eafba 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update UI and open PR - uses: SiaFoundation/workflows/.github/actions/ui-update@master + uses: SiaFoundation/workflows/.github/actions/ui-update@22e56c0750c0febb09784caa01c60021e0b5f111 # master with: moduleName: 'walletd' goVersion: '1.21' From 05814befb590d66b61e9270cb8f9db360bdb6389 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 02:12:06 +0000 Subject: [PATCH 602/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.21.3 to 0.22.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.21.3...v0.22.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.22.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 3cfc29f..9f6bcee 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.44 go.sia.tech/core v0.21.0 - go.sia.tech/coreutils v0.21.3 + go.sia.tech/coreutils v0.22.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 @@ -21,15 +21,15 @@ require ( github.com/dunglas/httpsfv v1.1.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.59.0 // indirect + github.com/quic-go/quic-go v0.59.1 // indirect github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c // indirect go.etcd.io/bbolt v1.4.3 // indirect go.sia.tech/mux v1.5.0 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.50.0 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/sys v0.44.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/tools v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index c45e899..12bd60f 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= -github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c h1:qnILxGINaIzEFPrZVtfexAvKw5unmQV9PAvEuKYgp94= github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c/go.mod h1:ocpwcCqYQbWRGNaCYlToTUVgjsbh0yEjLAyXl8yAIdA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -28,8 +28,8 @@ go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.sia.tech/core v0.21.0 h1:BtZPT/UGPj5YjDmIA31fmZIm+fmqw/X4yshskr3cOwQ= go.sia.tech/core v0.21.0/go.mod h1:b39iLfen0vRV3dkq1eSeBHDcaLFf89YPh2/TyBDY31Y= -go.sia.tech/coreutils v0.21.3 h1:VXLvRPHQOocxGR21HJP6s8B35MwOI9V0FP908VX/aEk= -go.sia.tech/coreutils v0.21.3/go.mod h1:jPFwxdUjURWSan9UgQkCFO0Zlt1bErLSu7v/l1tuhfg= +go.sia.tech/coreutils v0.22.0 h1:JNohN27L8fLNQDLeLyQtsmVv7Sm3CmBPUxKUtQkJhWI= +go.sia.tech/coreutils v0.22.0/go.mod h1:lpHJn8sS+4JkWxG8ZC2fPCdg11+u0pK8h8ae+WEovnQ= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.0 h1:6bQeO5y4AQPcf+UYHjhxpaNX+2V2wI5LIl0BbAg2YDA= @@ -48,8 +48,8 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= @@ -62,8 +62,8 @@ golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 3369036288ccde892327d4e8b527ad15ce97c220 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Tue, 19 May 2026 10:01:28 +0200 Subject: [PATCH 603/630] update workflow to latest master --- .github/workflows/main.yml | 2 +- .github/workflows/openapi-sync.yml | 2 +- .github/workflows/project-add.yml | 2 +- .github/workflows/publish.yml | 6 +++--- .github/workflows/ui.yml | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e206c37..0a44758 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,4 +10,4 @@ env: jobs: test: - uses: SiaFoundation/workflows/.github/workflows/go-test.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master + uses: SiaFoundation/workflows/.github/workflows/go-test.yml@dddded471aae1c7b6f0fbc388d603b8b16336f6f # master diff --git a/.github/workflows/openapi-sync.yml b/.github/workflows/openapi-sync.yml index 84c4c73..38f0606 100644 --- a/.github/workflows/openapi-sync.yml +++ b/.github/workflows/openapi-sync.yml @@ -11,6 +11,6 @@ on: jobs: sync: - uses: SiaFoundation/workflows/.github/workflows/sync-openapi-version.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master + uses: SiaFoundation/workflows/.github/workflows/sync-openapi-version.yml@dddded471aae1c7b6f0fbc388d603b8b16336f6f # master with: spec_path: openapi.yml diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml index 42298ef..9244a16 100644 --- a/.github/workflows/project-add.yml +++ b/.github/workflows/project-add.yml @@ -10,5 +10,5 @@ on: jobs: add-to-project: - uses: SiaFoundation/workflows/.github/workflows/project-add.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master + uses: SiaFoundation/workflows/.github/workflows/project-add.yml@dddded471aae1c7b6f0fbc388d603b8b16336f6f # master secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 31f6e85..a488798 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,7 +16,7 @@ concurrency: jobs: publish: - uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@22e56c0750c0febb09784caa01c60021e0b5f111 # master + uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@dddded471aae1c7b6f0fbc388d603b8b16336f6f # master secrets: inherit with: linux-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' @@ -29,7 +29,7 @@ jobs: upload: if: github.event_name == 'push' && startsWith(github.ref_name, 'v') runs-on: ubuntu-latest - needs: + needs: - publish steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 @@ -45,4 +45,4 @@ jobs: gh release upload ${{ github.ref_name }} * env: GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} - continue-on-error: true \ No newline at end of file + continue-on-error: true diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 30eafba..d330e88 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -11,8 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Update UI and open PR - uses: SiaFoundation/workflows/.github/actions/ui-update@22e56c0750c0febb09784caa01c60021e0b5f111 # master + uses: SiaFoundation/workflows/.github/actions/ui-update@dddded471aae1c7b6f0fbc388d603b8b16336f6f # master with: - moduleName: 'walletd' - goVersion: '1.21' - token: ${{ secrets.GITHUB_TOKEN }} + moduleName: "walletd" + goVersion: "1.21" + token: ${{ secrets.GITHUB_TOKEN }} From c52d0441f82d07ee307c2751c49374e48e3ee6c6 Mon Sep 17 00:00:00 2001 From: Alright Date: Fri, 5 Jun 2026 14:20:34 -0400 Subject: [PATCH 604/630] Fix pprof named profiles --- api/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/server.go b/api/server.go index ad84195..4a8945f 100644 --- a/api/server.go +++ b/api/server.go @@ -1631,9 +1631,8 @@ func (s *server) pprofHandler(jc jape.Context) { case "trace": pprof.Trace(jc.ResponseWriter, jc.Request) default: - pprof.Index(jc.ResponseWriter, jc.Request) + pprof.Handler(handler).ServeHTTP(jc.ResponseWriter, jc.Request) } - pprof.Index(jc.ResponseWriter, jc.Request) } // NewServer returns an HTTP handler that serves the walletd API. From 45df97488cdf6dcb9f9c55059300e6bb803568e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:33:22 +0000 Subject: [PATCH 605/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.44 to 1.14.45 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.44...v1.14.45) Updates `golang.org/x/term` from 0.43.0 to 0.44.0 - [Commits](https://github.com/golang/term/compare/v0.43.0...v0.44.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.45 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.44.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9f6bcee..2ccbf3e 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.44 + github.com/mattn/go-sqlite3 v1.14.45 go.sia.tech/core v0.21.0 go.sia.tech/coreutils v0.22.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.43.0 + golang.org/x/term v0.44.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -29,7 +29,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.51.0 // indirect golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.44.0 // indirect + golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/tools v0.44.0 // indirect ) diff --git a/go.sum b/go.sum index 12bd60f..2f94dba 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= -github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.45 h1:6KA/spDguL3KV8rnybG7ezSaE4SeMR3KC9VbUoAQaIk= +github.com/mattn/go-sqlite3 v1.14.45/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= @@ -58,10 +58,10 @@ golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= -golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= From eb45f7a135b0d5c8ef128165d88ec1a9685e2204 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:33:23 +0000 Subject: [PATCH 606/630] build(deps): bump go.sia.tech/core in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/core](https://github.com/SiaFoundation/core). Updates `go.sia.tech/core` from 0.21.0 to 0.21.1 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.21.0...v0.21.1) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.21.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 2ccbf3e..c6521db 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.45 - go.sia.tech/core v0.21.0 + go.sia.tech/core v0.21.1 go.sia.tech/coreutils v0.22.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 @@ -24,12 +24,12 @@ require ( github.com/quic-go/quic-go v0.59.1 // indirect github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c // indirect go.etcd.io/bbolt v1.4.3 // indirect - go.sia.tech/mux v1.5.0 // indirect + go.sia.tech/mux v1.5.2 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/tools v0.45.0 // indirect ) diff --git a/go.sum b/go.sum index 2f94dba..64fc3ce 100644 --- a/go.sum +++ b/go.sum @@ -26,14 +26,14 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= -go.sia.tech/core v0.21.0 h1:BtZPT/UGPj5YjDmIA31fmZIm+fmqw/X4yshskr3cOwQ= -go.sia.tech/core v0.21.0/go.mod h1:b39iLfen0vRV3dkq1eSeBHDcaLFf89YPh2/TyBDY31Y= +go.sia.tech/core v0.21.1 h1:IZY7KvX52IMP6SBrlMp7i38f18Q4k9IJYkuVlZLxvQQ= +go.sia.tech/core v0.21.1/go.mod h1:HUIelqenk1TTkDpYnsN6vgAFzNLxW/ueNYxvZCvoIBs= go.sia.tech/coreutils v0.22.0 h1:JNohN27L8fLNQDLeLyQtsmVv7Sm3CmBPUxKUtQkJhWI= go.sia.tech/coreutils v0.22.0/go.mod h1:lpHJn8sS+4JkWxG8ZC2fPCdg11+u0pK8h8ae+WEovnQ= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= -go.sia.tech/mux v1.5.0 h1:6bQeO5y4AQPcf+UYHjhxpaNX+2V2wI5LIl0BbAg2YDA= -go.sia.tech/mux v1.5.0/go.mod h1:1/SlgVsLOUsca5tXxAEEwl+Ohm4B8te2YdRhpRGaUZw= +go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= +go.sia.tech/mux v1.5.2/go.mod h1:MW00TmBIJY4CrdOwKohBaGalbBf27/Zcf2S5YVIEMn8= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.36.2 h1:yzQt9CYZw9A/t1/sQGW6va0v1iLd586S2z6+e6Hr6Vs= @@ -48,24 +48,24 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 4d84253dc606d1d4b6002902f8302e0cc15e7808 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:33:16 +0000 Subject: [PATCH 607/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.45 to 1.14.47 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.45...v1.14.47) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.47 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c6521db..3e27d18 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.12.0 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.45 + github.com/mattn/go-sqlite3 v1.14.47 go.sia.tech/core v0.21.1 go.sia.tech/coreutils v0.22.0 go.sia.tech/jape v0.14.1 diff --git a/go.sum b/go.sum index 64fc3ce..4701359 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.45 h1:6KA/spDguL3KV8rnybG7ezSaE4SeMR3KC9VbUoAQaIk= -github.com/mattn/go-sqlite3 v1.14.45/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.47 h1:jOBI62gS7nKeZv+as1oGEy0+1qISgXwH/QBlR6KbfIo= +github.com/mattn/go-sqlite3 v1.14.47/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= From c1a5bf6ca7f629c86667d72e24a6f74b8a1a8fdb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:33:04 +0000 Subject: [PATCH 608/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.22.0 to 0.22.1 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.22.0...v0.22.1) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.22.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 3e27d18..cefc56d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 go.sia.tech/core v0.21.1 - go.sia.tech/coreutils v0.22.0 + go.sia.tech/coreutils v0.22.1 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 @@ -21,9 +21,9 @@ require ( github.com/dunglas/httpsfv v1.1.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.59.1 // indirect - github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c // indirect - go.etcd.io/bbolt v1.4.3 // indirect + github.com/quic-go/quic-go v0.60.0 // indirect + github.com/quic-go/webtransport-go v0.11.0 // indirect + go.etcd.io/bbolt v1.5.0 // indirect go.sia.tech/mux v1.5.2 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index 4701359..7afd453 100644 --- a/go.sum +++ b/go.sum @@ -14,22 +14,24 @@ github.com/mattn/go-sqlite3 v1.14.47 h1:jOBI62gS7nKeZv+as1oGEy0+1qISgXwH/QBlR6Kb github.com/mattn/go-sqlite3 v1.14.47/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= +github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= -github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c h1:qnILxGINaIzEFPrZVtfexAvKw5unmQV9PAvEuKYgp94= -github.com/quic-go/webtransport-go v0.10.1-0.20260312060737-05fe5253a73c/go.mod h1:ocpwcCqYQbWRGNaCYlToTUVgjsbh0yEjLAyXl8yAIdA= +github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= +github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= +github.com/quic-go/webtransport-go v0.11.0 h1:3afiZq7MHv3gmKCbMwZ8D5M1u0y/1RdONN9KlWp32J0= +github.com/quic-go/webtransport-go v0.11.0/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= -go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.1 h1:IZY7KvX52IMP6SBrlMp7i38f18Q4k9IJYkuVlZLxvQQ= go.sia.tech/core v0.21.1/go.mod h1:HUIelqenk1TTkDpYnsN6vgAFzNLxW/ueNYxvZCvoIBs= -go.sia.tech/coreutils v0.22.0 h1:JNohN27L8fLNQDLeLyQtsmVv7Sm3CmBPUxKUtQkJhWI= -go.sia.tech/coreutils v0.22.0/go.mod h1:lpHJn8sS+4JkWxG8ZC2fPCdg11+u0pK8h8ae+WEovnQ= +go.sia.tech/coreutils v0.22.1 h1:6sAfpixFy4p6P/BvLcDZXmwv9ROE1DwOLJ4wsdAiLqQ= +go.sia.tech/coreutils v0.22.1/go.mod h1:cdzG9reaePsSAIdi9fI+noOvzJSerHtsj9Xuu8HZdA0= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From 0c2f87350c133fc54bc41897ae08bba63bcd9fb2 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jul 2026 08:43:01 -0700 Subject: [PATCH 609/630] update core and coreutils --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index cefc56d..129aaef 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 - go.sia.tech/core v0.21.1 - go.sia.tech/coreutils v0.22.1 + go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 + go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 @@ -28,8 +28,8 @@ require ( go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.53.0 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/tools v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 7afd453..956c054 100644 --- a/go.sum +++ b/go.sum @@ -28,10 +28,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= -go.sia.tech/core v0.21.1 h1:IZY7KvX52IMP6SBrlMp7i38f18Q4k9IJYkuVlZLxvQQ= -go.sia.tech/core v0.21.1/go.mod h1:HUIelqenk1TTkDpYnsN6vgAFzNLxW/ueNYxvZCvoIBs= -go.sia.tech/coreutils v0.22.1 h1:6sAfpixFy4p6P/BvLcDZXmwv9ROE1DwOLJ4wsdAiLqQ= -go.sia.tech/coreutils v0.22.1/go.mod h1:cdzG9reaePsSAIdi9fI+noOvzJSerHtsj9Xuu8HZdA0= +go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 h1:yI8R1BtDz+1WHB4L3Ju4Sg0jPvgaskJ1f6V+VEoqbX8= +go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318/go.mod h1:v0NyAMbZbol7O4OeI8X7QJOPyICIRg72eRCHQUfg9XI= +go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671 h1:gvcsyKlslAmXqa5NFpDJ1diwrfKKomZfa6pQccgPZEI= +go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= @@ -54,10 +54,10 @@ golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -66,8 +66,8 @@ golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 0e8bb545901785bab7ffbbb0edad94ab10cb623d Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jul 2026 09:09:11 -0700 Subject: [PATCH 610/630] update core and coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 129aaef..37edba0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 - go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671 + go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 diff --git a/go.sum b/go.sum index 956c054..444e4b9 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 h1:yI8R1BtDz+1WHB4L3Ju4Sg0jPvgaskJ1f6V+VEoqbX8= go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318/go.mod h1:v0NyAMbZbol7O4OeI8X7QJOPyICIRg72eRCHQUfg9XI= -go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671 h1:gvcsyKlslAmXqa5NFpDJ1diwrfKKomZfa6pQccgPZEI= -go.sia.tech/coreutils v0.22.2-0.20260703153931-884a313f9671/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= +go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201 h1:kvVCB+ukR2GwC2HQeamIu+C03pGsC6bbLYDq/RXcA88= +go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From 12a6b9d0f51afa5170d42b37ff63a23677be91b4 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 3 Jul 2026 09:22:28 -0700 Subject: [PATCH 611/630] update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 37edba0..6220a85 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 - go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201 + go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 diff --git a/go.sum b/go.sum index 444e4b9..e2a3497 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 h1:yI8R1BtDz+1WHB4L3Ju4Sg0jPvgaskJ1f6V+VEoqbX8= go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318/go.mod h1:v0NyAMbZbol7O4OeI8X7QJOPyICIRg72eRCHQUfg9XI= -go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201 h1:kvVCB+ukR2GwC2HQeamIu+C03pGsC6bbLYDq/RXcA88= -go.sia.tech/coreutils v0.22.2-0.20260703160732-3f486fb04201/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= +go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6 h1:hf0hmM+jLFGv4Twzl1Ze259BTjsZT4wJOyJ21TvGj4w= +go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From 91ccb67277596367bffa174400dcffceeab00e97 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 5 Jul 2026 15:04:29 -0700 Subject: [PATCH 612/630] update coreutils --- go.mod | 6 +++--- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6220a85..41d5268 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ -module go.sia.tech/walletd/v2 // v2.12.0 +module go.sia.tech/walletd/v2 // v2.14.1 go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 - go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 - go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6 + go.sia.tech/core v0.21.4 + go.sia.tech/coreutils v0.23.2 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 diff --git a/go.sum b/go.sum index e2a3497..6e7dbe9 100644 --- a/go.sum +++ b/go.sum @@ -28,10 +28,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= -go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318 h1:yI8R1BtDz+1WHB4L3Ju4Sg0jPvgaskJ1f6V+VEoqbX8= -go.sia.tech/core v0.21.3-0.20260703153603-327be4a8d318/go.mod h1:v0NyAMbZbol7O4OeI8X7QJOPyICIRg72eRCHQUfg9XI= -go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6 h1:hf0hmM+jLFGv4Twzl1Ze259BTjsZT4wJOyJ21TvGj4w= -go.sia.tech/coreutils v0.22.2-0.20260703161806-1cbfb0499be6/go.mod h1:uaTEHKV9lLRmj3fCwgCAFxeTzFZkY22J5tTdWGV9Km4= +go.sia.tech/core v0.21.4 h1:EFbw8tJ2Jo4cqU1e6VNTUUg5WPGaUlpGD5t1RYje59c= +go.sia.tech/core v0.21.4/go.mod h1:AluTCM6Q+1YAw9nXNz2un4NEh4J8GPySEuYxBdCvlCM= +go.sia.tech/coreutils v0.23.2 h1:bRGzYZA7qv/kDu0SQNQSpM+dwRpbJzlHLoW9fUjjeo4= +go.sia.tech/coreutils v0.23.2/go.mod h1:UOE/4k7hBGZC3teXU8Nc951DIEojRGi5PeRUM8optJw= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From e26da9ac9d2107be40ec91034f6ac6f60facca4e Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 5 Jul 2026 15:10:02 -0700 Subject: [PATCH 613/630] chore: fix changesets --- .changeset/update_core_dependency_from_0175_to_0210.md | 5 ----- .changeset/update_core_to_0200.md | 5 ----- .changeset/update_coreutils_to_0213.md | 5 ----- .changeset/update_coreutils_to_v0211.md | 2 +- 4 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 .changeset/update_core_dependency_from_0175_to_0210.md delete mode 100644 .changeset/update_core_to_0200.md delete mode 100644 .changeset/update_coreutils_to_0213.md diff --git a/.changeset/update_core_dependency_from_0175_to_0210.md b/.changeset/update_core_dependency_from_0175_to_0210.md deleted file mode 100644 index 7be2e02..0000000 --- a/.changeset/update_core_dependency_from_0175_to_0210.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core dependency to v0.19.0 and coreutils dependency to v0.21.0. diff --git a/.changeset/update_core_to_0200.md b/.changeset/update_core_to_0200.md deleted file mode 100644 index b2ba2e0..0000000 --- a/.changeset/update_core_to_0200.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update core to 0.20.0. diff --git a/.changeset/update_coreutils_to_0213.md b/.changeset/update_coreutils_to_0213.md deleted file mode 100644 index 515cbbc..0000000 --- a/.changeset/update_coreutils_to_0213.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to 0.21.3. diff --git a/.changeset/update_coreutils_to_v0211.md b/.changeset/update_coreutils_to_v0211.md index 8ea93cd..3bc83a0 100644 --- a/.changeset/update_coreutils_to_v0211.md +++ b/.changeset/update_coreutils_to_v0211.md @@ -2,4 +2,4 @@ default: patch --- -# Update coreutils from v0.21.0 to v0.21.1 +# Update coreutils to v0.23.2 and core to v0.21.4 From fc062bb9ab1ebedadb9ecf413d2e04ee2b8d73c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:10:14 +0000 Subject: [PATCH 614/630] chore: prepare release 2.15.0 --- ...nused_ephemeral_and_created_maps_in_revert_path.md | 5 ----- .changeset/update_coreutils_to_v0211.md | 5 ----- .changeset/update_go_to_1260.md | 5 ----- CHANGELOG.md | 11 +++++++++++ go.mod | 2 +- 5 files changed, 12 insertions(+), 16 deletions(-) delete mode 100644 .changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md delete mode 100644 .changeset/update_coreutils_to_v0211.md delete mode 100644 .changeset/update_go_to_1260.md diff --git a/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md b/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md deleted file mode 100644 index d59258f..0000000 --- a/.changeset/removed_unused_ephemeral_and_created_maps_in_revert_path.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Removed unused ephemeral and created maps in revert path diff --git a/.changeset/update_coreutils_to_v0211.md b/.changeset/update_coreutils_to_v0211.md deleted file mode 100644 index 3bc83a0..0000000 --- a/.changeset/update_coreutils_to_v0211.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to v0.23.2 and core to v0.21.4 diff --git a/.changeset/update_go_to_1260.md b/.changeset/update_go_to_1260.md deleted file mode 100644 index b1f573c..0000000 --- a/.changeset/update_go_to_1260.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Update Go to 1.26.0. diff --git a/CHANGELOG.md b/CHANGELOG.md index 19658ac..d4035c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## 2.15.0 (2026-07-05) + +### Features + +- Update Go to 1.26.0. + +### Fixes + +- Removed unused ephemeral and created maps in revert path +- Update coreutils to v0.23.2 and core to v0.21.4 + ## 2.12.0 (2026-02-05) ### Features diff --git a/go.mod b/go.mod index 41d5268..d56220e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.14.1 +module go.sia.tech/walletd/v2 // v2.15.0 go 1.26.0 From 90f85d4983b108fed1274c8631de9aea9860c8f2 Mon Sep 17 00:00:00 2001 From: Nate Date: Sun, 5 Jul 2026 22:05:18 -0700 Subject: [PATCH 615/630] update coreutils --- .changeset/update_coreutils_to_v0233.md | 5 +++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changeset/update_coreutils_to_v0233.md diff --git a/.changeset/update_coreutils_to_v0233.md b/.changeset/update_coreutils_to_v0233.md new file mode 100644 index 0000000..6e5d4d3 --- /dev/null +++ b/.changeset/update_coreutils_to_v0233.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils to v0.23.3 diff --git a/go.mod b/go.mod index d56220e..47e25cc 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 go.sia.tech/core v0.21.4 - go.sia.tech/coreutils v0.23.2 + go.sia.tech/coreutils v0.23.3 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 diff --git a/go.sum b/go.sum index 6e7dbe9..6bacf9b 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.4 h1:EFbw8tJ2Jo4cqU1e6VNTUUg5WPGaUlpGD5t1RYje59c= go.sia.tech/core v0.21.4/go.mod h1:AluTCM6Q+1YAw9nXNz2un4NEh4J8GPySEuYxBdCvlCM= -go.sia.tech/coreutils v0.23.2 h1:bRGzYZA7qv/kDu0SQNQSpM+dwRpbJzlHLoW9fUjjeo4= -go.sia.tech/coreutils v0.23.2/go.mod h1:UOE/4k7hBGZC3teXU8Nc951DIEojRGi5PeRUM8optJw= +go.sia.tech/coreutils v0.23.3 h1:+207YOihCb/xhCyMKYtAsDjAXasiduhURyntmb798XI= +go.sia.tech/coreutils v0.23.3/go.mod h1:WHMcopWu5kqIeGIf3ipPRvT8YQCqlwdEgnWxydRZdz8= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From 48ddbc5c13467a37e5cdda3149ed1c2e3df8422d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 05:05:30 +0000 Subject: [PATCH 616/630] chore: prepare release 2.15.1 --- .changeset/update_coreutils_to_v0233.md | 5 ----- CHANGELOG.md | 6 ++++++ go.mod | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/update_coreutils_to_v0233.md diff --git a/.changeset/update_coreutils_to_v0233.md b/.changeset/update_coreutils_to_v0233.md deleted file mode 100644 index 6e5d4d3..0000000 --- a/.changeset/update_coreutils_to_v0233.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to v0.23.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index d4035c9..1b0c4aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.15.1 (2026-07-06) + +### Fixes + +- Update coreutils to v0.23.3 + ## 2.15.0 (2026-07-05) ### Features diff --git a/go.mod b/go.mod index 47e25cc..abda9e6 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.15.0 +module go.sia.tech/walletd/v2 // v2.15.1 go 1.26.0 From ca5c2406d03240d9d7771f4cf3ee7f8819e9412b Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 6 Jul 2026 13:30:00 -0700 Subject: [PATCH 617/630] update coreutils to v0.23.4 --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index abda9e6..11ee198 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 - go.sia.tech/core v0.21.4 - go.sia.tech/coreutils v0.23.3 + go.sia.tech/core v0.21.5 + go.sia.tech/coreutils v0.23.4 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 @@ -22,7 +22,7 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.60.0 // indirect - github.com/quic-go/webtransport-go v0.11.0 // indirect + github.com/quic-go/webtransport-go v0.11.1 // indirect go.etcd.io/bbolt v1.5.0 // indirect go.sia.tech/mux v1.5.2 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect diff --git a/go.sum b/go.sum index 6bacf9b..c3f69f0 100644 --- a/go.sum +++ b/go.sum @@ -20,18 +20,18 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= -github.com/quic-go/webtransport-go v0.11.0 h1:3afiZq7MHv3gmKCbMwZ8D5M1u0y/1RdONN9KlWp32J0= -github.com/quic-go/webtransport-go v0.11.0/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= +github.com/quic-go/webtransport-go v0.11.1 h1:rrFQMO+7/52ZDJ04fsrjIaWqn6q1z1MYo9iVFq6JtbA= +github.com/quic-go/webtransport-go v0.11.1/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= -go.sia.tech/core v0.21.4 h1:EFbw8tJ2Jo4cqU1e6VNTUUg5WPGaUlpGD5t1RYje59c= -go.sia.tech/core v0.21.4/go.mod h1:AluTCM6Q+1YAw9nXNz2un4NEh4J8GPySEuYxBdCvlCM= -go.sia.tech/coreutils v0.23.3 h1:+207YOihCb/xhCyMKYtAsDjAXasiduhURyntmb798XI= -go.sia.tech/coreutils v0.23.3/go.mod h1:WHMcopWu5kqIeGIf3ipPRvT8YQCqlwdEgnWxydRZdz8= +go.sia.tech/core v0.21.5 h1:tIGizUDbezS2VhsX9U+aJGaU0iKlCMdDjP8FRGPNRls= +go.sia.tech/core v0.21.5/go.mod h1:AluTCM6Q+1YAw9nXNz2un4NEh4J8GPySEuYxBdCvlCM= +go.sia.tech/coreutils v0.23.4 h1:y0HZJcJU5CWMMPOZVFnbWhGdH5VVL9yN30oM6ZtT5Pc= +go.sia.tech/coreutils v0.23.4/go.mod h1:NIjd59x6PBPqcJkEUm1LDfqdJPlAma9dr4dfCnjNj0o= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From d7a0ab1355e110bbfb54833404ad07862543d682 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:00:06 +0000 Subject: [PATCH 618/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [go.sia.tech/core](https://github.com/SiaFoundation/core) and [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/core` from 0.21.5 to 0.21.6 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.21.5...v0.21.6) Updates `go.sia.tech/coreutils` from 0.23.4 to 0.23.5 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.23.4...v0.23.5) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.21.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: go.sia.tech/coreutils dependency-version: 0.23.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 11ee198..2148749 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.47 - go.sia.tech/core v0.21.5 - go.sia.tech/coreutils v0.23.4 + go.sia.tech/core v0.21.6 + go.sia.tech/coreutils v0.23.5 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 diff --git a/go.sum b/go.sum index c3f69f0..6b101b1 100644 --- a/go.sum +++ b/go.sum @@ -28,10 +28,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= -go.sia.tech/core v0.21.5 h1:tIGizUDbezS2VhsX9U+aJGaU0iKlCMdDjP8FRGPNRls= -go.sia.tech/core v0.21.5/go.mod h1:AluTCM6Q+1YAw9nXNz2un4NEh4J8GPySEuYxBdCvlCM= -go.sia.tech/coreutils v0.23.4 h1:y0HZJcJU5CWMMPOZVFnbWhGdH5VVL9yN30oM6ZtT5Pc= -go.sia.tech/coreutils v0.23.4/go.mod h1:NIjd59x6PBPqcJkEUm1LDfqdJPlAma9dr4dfCnjNj0o= +go.sia.tech/core v0.21.6 h1:fge57Itz8R9jXegVd63y/jIQHAOLS+X2mEqYnbw1d/k= +go.sia.tech/core v0.21.6/go.mod h1:UAc637UMThRXG13B2SEc9MCmJSsh7yuF78VfvweXyhE= +go.sia.tech/coreutils v0.23.5 h1:KrkaV5MgFcsx3Aqma4qymWStTZKRz8JTuRQUl/PBfx0= +go.sia.tech/coreutils v0.23.5/go.mod h1:DN3dIRWJaS8ITrIuDx/Z6AlPneD2KjxVjIjnMYtl7D4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= From 723d985d747efdd1658765763942f9c4913f5507 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:01:54 +0200 Subject: [PATCH 619/630] changeset --- .changeset/update_gosiatechcore_to_0216.md | 5 +++++ .changeset/update_gosiatechcoreutils_to_0235.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/update_gosiatechcore_to_0216.md create mode 100644 .changeset/update_gosiatechcoreutils_to_0235.md diff --git a/.changeset/update_gosiatechcore_to_0216.md b/.changeset/update_gosiatechcore_to_0216.md new file mode 100644 index 0000000..52e5910 --- /dev/null +++ b/.changeset/update_gosiatechcore_to_0216.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update go.sia.tech/core to 0.21.6. diff --git a/.changeset/update_gosiatechcoreutils_to_0235.md b/.changeset/update_gosiatechcoreutils_to_0235.md new file mode 100644 index 0000000..f61879b --- /dev/null +++ b/.changeset/update_gosiatechcoreutils_to_0235.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update go.sia.tech/coreutils to 0.23.5. From 08c276ec4a267b98dee8e55efe50b90137c78625 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:08:27 +0000 Subject: [PATCH 620/630] chore: prepare release 2.15.2 --- .changeset/update_gosiatechcore_to_0216.md | 5 ----- .changeset/update_gosiatechcoreutils_to_0235.md | 5 ----- CHANGELOG.md | 7 +++++++ go.mod | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .changeset/update_gosiatechcore_to_0216.md delete mode 100644 .changeset/update_gosiatechcoreutils_to_0235.md diff --git a/.changeset/update_gosiatechcore_to_0216.md b/.changeset/update_gosiatechcore_to_0216.md deleted file mode 100644 index 52e5910..0000000 --- a/.changeset/update_gosiatechcore_to_0216.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update go.sia.tech/core to 0.21.6. diff --git a/.changeset/update_gosiatechcoreutils_to_0235.md b/.changeset/update_gosiatechcoreutils_to_0235.md deleted file mode 100644 index f61879b..0000000 --- a/.changeset/update_gosiatechcoreutils_to_0235.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update go.sia.tech/coreutils to 0.23.5. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0c4aa..b2c8ad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2.15.2 (2026-07-08) + +### Fixes + +- Update go.sia.tech/core to 0.21.6. +- Update go.sia.tech/coreutils to 0.23.5. + ## 2.15.1 (2026-07-06) ### Fixes diff --git a/go.mod b/go.mod index 2148749..91be75a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.15.1 +module go.sia.tech/walletd/v2 // v2.15.2 go 1.26.0 From 689131a2b4ccb6381d32fb68d7489ebc5753fc5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:33:19 +0000 Subject: [PATCH 621/630] build(deps): bump the all-dependencies group with 2 updates Bumps the all-dependencies group with 2 updates: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and [golang.org/x/term](https://github.com/golang/term). Updates `github.com/mattn/go-sqlite3` from 1.14.47 to 1.14.48 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.47...v1.14.48) Updates `golang.org/x/term` from 0.44.0 to 0.45.0 - [Commits](https://github.com/golang/term/compare/v0.44.0...v0.45.0) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.48 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: golang.org/x/term dependency-version: 0.45.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 91be75a..f38eaf6 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module go.sia.tech/walletd/v2 // v2.15.2 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.47 + github.com/mattn/go-sqlite3 v1.14.48 go.sia.tech/core v0.21.6 go.sia.tech/coreutils v0.23.5 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.44.0 + golang.org/x/term v0.45.0 gopkg.in/yaml.v3 v3.0.1 lukechampine.com/flagg v1.1.1 lukechampine.com/frand v1.5.1 @@ -29,7 +29,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sys v0.46.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 6b101b1..864eadb 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.47 h1:jOBI62gS7nKeZv+as1oGEy0+1qISgXwH/QBlR6KbfIo= -github.com/mattn/go-sqlite3 v1.14.47/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= +github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= @@ -60,10 +60,10 @@ golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= From 8889d8944dc87a8b33228c96bfead29d20b3d5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:33:17 +0000 Subject: [PATCH 622/630] build(deps): bump go.sia.tech/core in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/core](https://github.com/SiaFoundation/core). Updates `go.sia.tech/core` from 0.21.6 to 0.21.7 - [Release notes](https://github.com/SiaFoundation/core/releases) - [Changelog](https://github.com/SiaFoundation/core/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/core/compare/v0.21.6...v0.21.7) --- updated-dependencies: - dependency-name: go.sia.tech/core dependency-version: 0.21.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index f38eaf6..71b63fc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.48 - go.sia.tech/core v0.21.6 + go.sia.tech/core v0.21.7 go.sia.tech/coreutils v0.23.5 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 @@ -24,12 +24,12 @@ require ( github.com/quic-go/quic-go v0.60.0 // indirect github.com/quic-go/webtransport-go v0.11.1 // indirect go.etcd.io/bbolt v1.5.0 // indirect - go.sia.tech/mux v1.5.2 // indirect + go.sia.tech/mux v1.5.3 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.53.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.38.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.48.0 // indirect ) diff --git a/go.sum b/go.sum index 864eadb..00735d1 100644 --- a/go.sum +++ b/go.sum @@ -28,14 +28,14 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= -go.sia.tech/core v0.21.6 h1:fge57Itz8R9jXegVd63y/jIQHAOLS+X2mEqYnbw1d/k= -go.sia.tech/core v0.21.6/go.mod h1:UAc637UMThRXG13B2SEc9MCmJSsh7yuF78VfvweXyhE= +go.sia.tech/core v0.21.7 h1:Qgi2293i/d+UfpuVGlAcXfDY0Vzkj/GTjpkuEBXmIks= +go.sia.tech/core v0.21.7/go.mod h1:80xXoUUnfIFVazv7i4qZH4e/+kbxSadd4B3EK1+MOtw= go.sia.tech/coreutils v0.23.5 h1:KrkaV5MgFcsx3Aqma4qymWStTZKRz8JTuRQUl/PBfx0= go.sia.tech/coreutils v0.23.5/go.mod h1:DN3dIRWJaS8ITrIuDx/Z6AlPneD2KjxVjIjnMYtl7D4= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= -go.sia.tech/mux v1.5.2 h1:MU07hUSZJJNo+ulN/7Gb5/3KBLaxXgunb63fIhxFy+o= -go.sia.tech/mux v1.5.2/go.mod h1:MW00TmBIJY4CrdOwKohBaGalbBf27/Zcf2S5YVIEMn8= +go.sia.tech/mux v1.5.3 h1:0LSoSUMUThKYYHPha3i3YADdfBVaGHcI8UKXhALndHg= +go.sia.tech/mux v1.5.3/go.mod h1:cYRXgCdhC5kH+8f6knyQJ2Wzk6kB7Ndam1KuEt4gI9M= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= go.sia.tech/web/walletd v0.36.2 h1:yzQt9CYZw9A/t1/sQGW6va0v1iLd586S2z6+e6Hr6Vs= @@ -50,24 +50,24 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From cffda971683dbf456eab9960a83beab36d65b101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:33:03 +0000 Subject: [PATCH 623/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.48 to 1.14.49 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.48...v1.14.49) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.49 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 71b63fc..cc6358c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.15.2 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.48 + github.com/mattn/go-sqlite3 v1.14.49 go.sia.tech/core v0.21.7 go.sia.tech/coreutils v0.23.5 go.sia.tech/jape v0.14.1 diff --git a/go.sum b/go.sum index 00735d1..967c9d5 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= -github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= +github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= From 24298ba5d800ab8177551ad4b477fdbd826757ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:49:53 +0000 Subject: [PATCH 624/630] build(deps): bump go.sia.tech/coreutils in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/coreutils](https://github.com/SiaFoundation/coreutils). Updates `go.sia.tech/coreutils` from 0.23.5 to 0.24.0 - [Release notes](https://github.com/SiaFoundation/coreutils/releases) - [Changelog](https://github.com/SiaFoundation/coreutils/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/coreutils/compare/v0.23.5...v0.24.0) --- updated-dependencies: - dependency-name: go.sia.tech/coreutils dependency-version: 0.24.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index cc6358c..906bc77 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.0 require ( github.com/mattn/go-sqlite3 v1.14.49 go.sia.tech/core v0.21.7 - go.sia.tech/coreutils v0.23.5 + go.sia.tech/coreutils v0.24.0 go.sia.tech/jape v0.14.1 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 @@ -21,8 +21,8 @@ require ( github.com/dunglas/httpsfv v1.1.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.60.0 // indirect - github.com/quic-go/webtransport-go v0.11.1 // indirect + github.com/quic-go/quic-go v0.61.0 // indirect + github.com/quic-go/webtransport-go v0.12.0 // indirect go.etcd.io/bbolt v1.5.0 // indirect go.sia.tech/mux v1.5.3 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect diff --git a/go.sum b/go.sum index 967c9d5..50febff 100644 --- a/go.sum +++ b/go.sum @@ -18,10 +18,10 @@ github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4 github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.60.0 h1:xcQioE8OM66UQLeUMHltK1CCcOu3JbVB4JAQdDQSB+0= -github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2vDyK1TuJk= -github.com/quic-go/webtransport-go v0.11.1 h1:rrFQMO+7/52ZDJ04fsrjIaWqn6q1z1MYo9iVFq6JtbA= -github.com/quic-go/webtransport-go v0.11.1/go.mod h1:SHgEzUFVyj+9WUSuGB1P6Zd351Pww2leWV3SwlTovkA= +github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA= +github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= +github.com/quic-go/webtransport-go v0.12.0 h1:CpnKNwZvdV0LD73xoHO8QaR0NI3llqpWRwnazdZS0sE= +github.com/quic-go/webtransport-go v0.12.0/go.mod h1:GHne8aRFJ24h73pAMrcywXtuaz/ShBXCLXLvG/NPFdU= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -30,8 +30,8 @@ go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.7 h1:Qgi2293i/d+UfpuVGlAcXfDY0Vzkj/GTjpkuEBXmIks= go.sia.tech/core v0.21.7/go.mod h1:80xXoUUnfIFVazv7i4qZH4e/+kbxSadd4B3EK1+MOtw= -go.sia.tech/coreutils v0.23.5 h1:KrkaV5MgFcsx3Aqma4qymWStTZKRz8JTuRQUl/PBfx0= -go.sia.tech/coreutils v0.23.5/go.mod h1:DN3dIRWJaS8ITrIuDx/Z6AlPneD2KjxVjIjnMYtl7D4= +go.sia.tech/coreutils v0.24.0 h1:xz3CJ3SS38cGTF6WVxmZ4dR6t+2LostNscyZ6n2hbOI= +go.sia.tech/coreutils v0.24.0/go.mod h1:xNzCC31sJkKXVnEjv12aHXutRfc4nPDl53sVJQhw6+k= go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= go.sia.tech/mux v1.5.3 h1:0LSoSUMUThKYYHPha3i3YADdfBVaGHcI8UKXhALndHg= From c29a595461f843984e02db27acf4ed96597e1f1d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:53:17 +0200 Subject: [PATCH 625/630] fix build --- cmd/walletd/node.go | 8 ++++---- internal/testutil/testutil.go | 4 ++-- persist/sqlite/consensus_test.go | 8 ++++---- wallet/wallet_test.go | 14 ++++++++------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index 9abd40f..bebb45b 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -194,11 +194,11 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } defer bdb.Close() - dbstore, tipState, err := chain.NewDBStoreAtCheckpoint(bdb, cs, b, chain.NewZapMigrationLogger(log.Named("chaindb"))) + dbstore, err := chain.NewDBStoreAtCheckpoint(bdb, cs, b, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } - cm = chain.NewManager(dbstore, tipState, chainOpts...) + cm = chain.NewManager(dbstore, chainOpts...) if err := store.SetCheckpoint(cfg.Checkpoint); err != nil { return fmt.Errorf("failed to set wallet db checkpoint: %w", err) } @@ -215,11 +215,11 @@ func runNode(ctx context.Context, cfg config.Config, log *zap.Logger) error { } defer bdb.Close() - dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) + dbstore, err := chain.NewDBStore(bdb, network, genesisBlock, chain.NewZapMigrationLogger(log.Named("chaindb"))) if err != nil { return fmt.Errorf("failed to create chain store: %w", err) } - cm = chain.NewManager(dbstore, tipState, chainOpts...) + cm = chain.NewManager(dbstore, chainOpts...) } syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 7f01f2f..b767bee 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -73,11 +73,11 @@ func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, } tb.Cleanup(func() { l.Close() }) - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis, nil) + dbstore, err := chain.NewDBStore(chain.NewMemDB(), n, genesis, nil) if err != nil { tb.Fatal(err) } - cm := chain.NewManager(dbstore, tipState) + cm := chain.NewManager(dbstore) store, err := sqlite.OpenDatabase(filepath.Join(tb.TempDir(), "walletd.sqlite"), sqlite.WithLog(log.Named("sqlite3"))) if err != nil { diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go index fdb858b..846cad2 100644 --- a/persist/sqlite/consensus_test.go +++ b/persist/sqlite/consensus_test.go @@ -61,12 +61,12 @@ func TestSpendSiacoins(t *testing.T) { addr := types.StandardUnlockHash(pk.PublicKey()) network, genesisBlock := testutil.Network() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) + store, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } - cm := chain.NewManager(store, genesisState) + cm := chain.NewManager(store) // create a wallet w, err := db.AddWallet(wallet.Wallet{Name: "test"}) @@ -182,12 +182,12 @@ func TestSpendSiafunds(t *testing.T) { network, genesisBlock := testutil.Network() // send the siafund airdrop to the wallet genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) + store, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } - cm := chain.NewManager(store, genesisState) + cm := chain.NewManager(store) // create a wallet w, err := db.AddWallet(wallet.Wallet{Name: "test"}) diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index bc52d3e..25235d6 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -2864,11 +2864,12 @@ func TestEventTypes(t *testing.T) { network, genesisBlock := testV2Network(addr) // raise the require height to test v1 events network.HardforkV2.RequireHeight = 250 - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) + store, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } - cm := chain.NewManager(store, genesisState) + cm := chain.NewManager(store) + genesisState := cm.TipState() // helper to mine blocks mineBlock := func(n int, addr types.Address) { @@ -3881,22 +3882,23 @@ func TestReset(t *testing.T) { } defer bdb.Close() - store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock, nil) + store, err := chain.NewDBStore(bdb, network, genesisBlock, nil) if err != nil { t.Fatal(err) } - cm1 := chain.NewManager(store, genesisState) + cm1 := chain.NewManager(store) + genesisState := cm1.TipState() bdb2, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus2.db")) if err != nil { t.Fatal(err) } defer bdb2.Close() - store2, genesisState2, err := chain.NewDBStore(bdb2, network, genesisBlock, nil) + store2, err := chain.NewDBStore(bdb2, network, genesisBlock, nil) if err != nil { t.Fatal(err) } - cm2 := chain.NewManager(store2, genesisState2) + cm2 := chain.NewManager(store2) // mine blocks before starting the wallet manager for i := 0; i < 25; i++ { From bc2fbbdda23c9377b56a3a78bf5f96d2109e9790 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl <3903476+ChrisSchinnerl@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:53:45 +0200 Subject: [PATCH 626/630] changeset --- .changeset/update_coreutils_to_v0240.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/update_coreutils_to_v0240.md diff --git a/.changeset/update_coreutils_to_v0240.md b/.changeset/update_coreutils_to_v0240.md new file mode 100644 index 0000000..86cb8b4 --- /dev/null +++ b/.changeset/update_coreutils_to_v0240.md @@ -0,0 +1,5 @@ +--- +default: patch +--- + +# Update coreutils to v0.24.0 From 65ea081dd4aa147101990f4122a30b87022a25f3 Mon Sep 17 00:00:00 2001 From: Nate Date: Sat, 15 Aug 2026 12:05:00 -0700 Subject: [PATCH 627/630] concurrent reads --- ...d_transactions_for_improved_scalability.md | 5 + persist/sqlite/addresses.go | 229 ++++++++++-------- persist/sqlite/consensus.go | 41 ++-- persist/sqlite/events.go | 15 +- persist/sqlite/peers.go | 23 +- persist/sqlite/sql.go | 22 ++ persist/sqlite/store.go | 98 ++++++-- persist/sqlite/store_test.go | 61 +++++ persist/sqlite/utxo.go | 61 ++--- persist/sqlite/wallet.go | 155 ++++++------ wallet/addresses_test.go | 45 ++++ 11 files changed, 479 insertions(+), 276 deletions(-) create mode 100644 .changeset/enabled_concurrent_read_transactions_for_improved_scalability.md diff --git a/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md b/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md new file mode 100644 index 0000000..f34a959 --- /dev/null +++ b/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# Enabled concurrent read transactions for improved scalability. diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go index 42e453b..cc8a344 100644 --- a/persist/sqlite/addresses.go +++ b/persist/sqlite/addresses.go @@ -15,76 +15,64 @@ import ( // // If the index mode is not full, this function will only return true if // an address is registered with a wallet. -func (s *Store) CheckAddresses(addresses []types.Address) (known bool, err error) { - err = s.transaction(func(tx *txn) error { - stmt, err := tx.Prepare(`SELECT true FROM sia_addresses WHERE sia_address=$1`) - if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() - - for _, addr := range addresses { - if err := stmt.QueryRow(encode(addr)).Scan(&known); err != nil { - if errors.Is(err, sql.ErrNoRows) { - continue - } - return fmt.Errorf("failed to query address: %w", err) - } - if known { - return nil - } +func (s *Store) CheckAddresses(addresses []types.Address) (bool, error) { + if len(addresses) == 0 { + return false, nil + } + return valuedTransaction(s, func(tx *txn) (known bool, _ error) { + query := `SELECT EXISTS(SELECT 1 FROM sia_addresses WHERE sia_address IN (` + queryPlaceHolders(len(addresses)) + `))` + if err := tx.QueryRow(query, encodeSlice(addresses)...).Scan(&known); err != nil { + return false, fmt.Errorf("failed to query addresses: %w", err) } - return nil + return known, nil }) - return } // AddressBalance returns the aggregate balance of the addresses. -func (s *Store) AddressBalance(address ...types.Address) (balance wallet.Balance, err error) { +func (s *Store) AddressBalance(address ...types.Address) (wallet.Balance, error) { if len(address) == 0 { return wallet.Balance{}, nil // no addresses, no balance } - err = s.transaction(func(tx *txn) error { - const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` - stmt, err := tx.Prepare(query) + return valuedTransaction(s, func(tx *txn) (balance wallet.Balance, _ error) { + query := `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address IN (` + queryPlaceHolders(len(address)) + `)` + rows, err := tx.Query(query, encodeSlice(address)...) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return wallet.Balance{}, fmt.Errorf("failed to query addresses: %w", err) } - defer stmt.Close() + defer rows.Close() - for _, addr := range address { + for rows.Next() { var siacoins, immatureSiacoins types.Currency var siafunds uint64 - if err := stmt.QueryRow(encode(addr)).Scan(decode(&siacoins), decode(&immatureSiacoins), &siafunds); err != nil && !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("failed to query address %q: %w", addr, err) + if err := rows.Scan(decode(&siacoins), decode(&immatureSiacoins), &siafunds); err != nil { + return wallet.Balance{}, fmt.Errorf("failed to scan address balance: %w", err) } balance.Siacoins = balance.Siacoins.Add(siacoins) balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(immatureSiacoins) balance.Siafunds += siafunds } - return nil + return balance, rows.Err() }) - return } // BatchAddressEvents returns the events for a batch of addresses. -func (s *Store) BatchAddressEvents(addresses []types.Address, offset, limit int) (events []wallet.Event, err error) { +func (s *Store) BatchAddressEvents(addresses []types.Address, offset, limit int) ([]wallet.Event, error) { if len(addresses) == 0 { return nil, nil // no addresses, no events } - err = s.transaction(func(tx *txn) error { + return valuedTransaction(s, func(tx *txn) ([]wallet.Event, error) { dbIDs, err := s.getAddressesEvents(tx, addresses, offset, limit) if err != nil { - return fmt.Errorf("failed to get events for addresses: %w", err) + return nil, fmt.Errorf("failed to get events for addresses: %w", err) } if len(dbIDs) == 0 { - return nil // no events found + return nil, nil // no events found } - events, err = getEventsByID(tx, dbIDs) + events, err := getEventsByID(tx, dbIDs) if err != nil { - return fmt.Errorf("failed to get events by ID: %w", err) + return nil, fmt.Errorf("failed to get events by ID: %w", err) } addressMap := make(map[types.Address]bool) @@ -150,42 +138,53 @@ func (s *Store) BatchAddressEvents(addresses []types.Address, offset, limit int) events[i].Relevant = append(events[i].Relevant, ev.SiacoinElement.SiacoinOutput.Address) } } - return nil + return events, nil }) - return } // BatchAddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (s *Store) BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { - basis, err = getScanBasis(tx) +func (s *Store) BatchAddressSiacoinOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + if len(addresses) == 0 { + return nil, types.ChainIndex{}, nil + } + return valuedTransaction2(s, func(tx *txn) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, _ error) { + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } - query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height + addressIDs, err := getAddressDBIDs(tx, addresses) + if err != nil { + return nil, types.ChainIndex{}, fmt.Errorf("failed to get address IDs: %w", err) + } else if len(addressIDs) == 0 { + return nil, basis, nil + } + + // filtering on sa.sia_address instead makes the planner drive the query + // off spent_index_id and scan every unspent element in the database + query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height FROM siacoin_elements se INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address IN (` + queryPlaceHolders(len(addresses)) + `) AND se.maturity_height <= ? AND se.spent_index_id IS NULL + WHERE se.address_id IN (` + queryPlaceHolders(len(addressIDs)) + `) AND se.maturity_height <= ? AND se.spent_index_id IS NULL LIMIT ? OFFSET ?` - rows, err := tx.Query(query, append(encodeSlice(addresses), basis.Height, limit, offset)...) + rows, err := tx.Query(query, append(anySlice(addressIDs), basis.Height, limit, offset)...) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siacoin element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siacoin element: %w", err) } siacoins = append(siacoins, siacoin) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siacoin elements @@ -196,48 +195,59 @@ func (s *Store) BatchAddressSiacoinOutputs(addresses []types.Address, offset, li } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siacoins[i].StateElement.MerkleProof = proof } } - return nil + return siacoins, basis, nil }) - return } // BatchAddressSiafundOutputs returns the unspent siafund outputs for an address. -func (s *Store) BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { - basis, err = getScanBasis(tx) +func (s *Store) BatchAddressSiafundOutputs(addresses []types.Address, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + if len(addresses) == 0 { + return nil, types.ChainIndex{}, nil + } + return valuedTransaction2(s, func(tx *txn) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, _ error) { + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } + addressIDs, err := getAddressDBIDs(tx, addresses) + if err != nil { + return nil, types.ChainIndex{}, fmt.Errorf("failed to get address IDs: %w", err) + } else if len(addressIDs) == 0 { + return nil, basis, nil + } + + // filtering on sa.sia_address instead makes the planner drive the query + // off spent_index_id and scan every unspent element in the database query := `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height FROM siafund_elements se INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) - WHERE sa.sia_address IN(` + queryPlaceHolders(len(addresses)) + `) AND se.spent_index_id IS NULL + WHERE se.address_id IN (` + queryPlaceHolders(len(addressIDs)) + `) AND se.spent_index_id IS NULL ORDER BY se.id DESC LIMIT ? OFFSET ?` - rows, err := tx.Query(query, append(encodeSlice(addresses), limit, offset)...) + rows, err := tx.Query(query, append(anySlice(addressIDs), limit, offset)...) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siafund, err := scanUnspentSiafundElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siafund element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siafund element: %w", err) } siafunds = append(siafunds, siafund) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siafund elements @@ -248,47 +258,45 @@ func (s *Store) BatchAddressSiafundOutputs(addresses []types.Address, offset, li } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siafunds[i].StateElement.MerkleProof = proof } } - return nil + return siafunds, basis, nil }) - return } // AddressEvents returns the events of a single address. -func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) { + return valuedTransaction(s, func(tx *txn) ([]wallet.Event, error) { dbIDs, err := getAddressEvents(tx, address, offset, limit) if err != nil { - return err + return nil, err } - events, err = getEventsByID(tx, dbIDs) + events, err := getEventsByID(tx, dbIDs) if err != nil { - return fmt.Errorf("failed to get events by ID: %w", err) + return nil, fmt.Errorf("failed to get events by ID: %w", err) } for i := range events { events[i].Relevant = []types.Address{address} } - return nil + return events, nil }) - return } // AddressSiacoinOutputs returns the unspent siacoin outputs for an address. -func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { - basis, err = getScanBasis(tx) +func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types.SiacoinOutputID, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + return valuedTransaction2(s, func(tx *txn) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, _ error) { + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } - query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height + query := `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height FROM siacoin_elements se INNER JOIN chain_indices ci ON (se.chain_index_id = ci.id) INNER JOIN sia_addresses sa ON (se.address_id = sa.id) @@ -307,20 +315,20 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types. rows, err := tx.Query(query, params...) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siacoin element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siacoin element: %w", err) } siacoins = append(siacoins, siacoin) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siacoin elements @@ -331,23 +339,22 @@ func (s *Store) AddressSiacoinOutputs(address types.Address, tpoolSpent []types. } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siacoins[i].StateElement.MerkleProof = proof } } - return nil + return siacoins, basis, nil }) - return } // AddressSiafundOutputs returns the unspent siafund outputs for an address. -func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { - basis, err = getScanBasis(tx) +func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types.SiafundOutputID, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + return valuedTransaction2(s, func(tx *txn) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, _ error) { + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } query := `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height @@ -370,19 +377,19 @@ func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types. rows, err := tx.Query(query, params...) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siafund, err := scanUnspentSiafundElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siafund element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siafund element: %w", err) } siafunds = append(siafunds, siafund) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siafund elements @@ -393,27 +400,26 @@ func (s *Store) AddressSiafundOutputs(address types.Address, tpoolSpent []types. } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siafunds[i].StateElement.MerkleProof = proof } } - return nil + return siafunds, basis, nil }) - return } // AnnotateV1Events annotates a list of unconfirmed transactions with // relevant addresses and siacoin/siafund elements. -func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []wallet.Event, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) ([]wallet.Event, error) { + return valuedTransaction(s, func(tx *txn) (annotated []wallet.Event, _ error) { siacoinElementStmt, err := tx.Prepare(`SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address FROM siacoin_elements se INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) if err != nil { - return fmt.Errorf("failed to prepare siacoin statement: %w", err) + return nil, fmt.Errorf("failed to prepare siacoin statement: %w", err) } defer siacoinElementStmt.Close() @@ -436,7 +442,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) if err != nil { - return fmt.Errorf("failed to prepare siafund statement: %w", err) + return nil, fmt.Errorf("failed to prepare siafund statement: %w", err) } defer siafundElementStmt.Close() @@ -477,7 +483,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 if errors.Is(err, sql.ErrNoRows) { continue // ignore elements that are not found } else if err != nil { - return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + return nil, fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) } ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) relevant = true @@ -501,7 +507,7 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 if errors.Is(err, sql.ErrNoRows) { continue // ignore elements that are not found } else if err != nil { - return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + return nil, fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) } ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) relevant = true @@ -525,9 +531,32 @@ func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 addEvent(types.Hash256(txn.ID()), ev) } - return nil + return annotated, nil }) - return +} + +// getAddressDBIDs returns the database IDs of the addresses. Addresses that are +// not in the database are omitted. +func getAddressDBIDs(tx *txn, addresses []types.Address) (ids []int64, err error) { + if len(addresses) == 0 { + return nil, nil + } + + query := `SELECT id FROM sia_addresses WHERE sia_address IN (` + queryPlaceHolders(len(addresses)) + `)` + rows, err := tx.Query(query, encodeSlice(addresses)...) + if err != nil { + return nil, fmt.Errorf("failed to query address IDs: %w", err) + } + defer rows.Close() + + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("failed to scan address ID: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() } func getAddressEvents(tx *txn, address types.Address, offset, limit int) (eventIDs []int64, err error) { diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go index 39f17ce..35b822b 100644 --- a/persist/sqlite/consensus.go +++ b/persist/sqlite/consensus.go @@ -1408,27 +1408,27 @@ func addressRefStmt(tx *txn) (func(types.Address) (addressRef, error), func() er // decorating its transactions with additional information such as siacoin input // origins. func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, error) { - cb := api.ConsensusBlock{ - ID: block.ID(), - ParentID: block.ParentID, - Nonce: block.Nonce, - Timestamp: block.Timestamp, - MinerPayouts: block.MinerPayouts, - Transactions: make([]api.ConsensusTransaction, 0, len(block.Transactions)), - } - - if block.V2 != nil { - cb.V2 = &api.ConsensusV2BlockData{ - Height: block.V2.Height, - Commitment: block.V2.Commitment, - Transactions: make([]api.ConsensusV2Transaction, 0, len(block.V2Transactions())), + return valuedTransaction(s, func(tx *txn) (api.ConsensusBlock, error) { + cb := api.ConsensusBlock{ + ID: block.ID(), + ParentID: block.ParentID, + Nonce: block.Nonce, + Timestamp: block.Timestamp, + MinerPayouts: block.MinerPayouts, + Transactions: make([]api.ConsensusTransaction, 0, len(block.Transactions)), + } + + if block.V2 != nil { + cb.V2 = &api.ConsensusV2BlockData{ + Height: block.V2.Height, + Commitment: block.V2.Commitment, + Transactions: make([]api.ConsensusV2Transaction, 0, len(block.V2Transactions())), + } } - } - err := s.transaction(func(tx *txn) error { stmt, err := tx.Prepare(`SELECT origin_source, origin_transaction_id, origin_transaction_index FROM siacoin_elements WHERE id=$1`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return api.ConsensusBlock{}, fmt.Errorf("failed to prepare statement: %w", err) } defer stmt.Close() @@ -1480,7 +1480,7 @@ func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, e for _, sci := range txn.SiacoinInputs { origin, err := getUTXOOrigin(sci.ParentID) if err != nil { - return fmt.Errorf("failed to get siacoin input source for %q: %w", sci.ParentID, err) + return api.ConsensusBlock{}, fmt.Errorf("failed to get siacoin input source for %q: %w", sci.ParentID, err) } apiTx.SiacoinInputs = append(apiTx.SiacoinInputs, api.ConsensusSiacoinInput{ @@ -1523,7 +1523,7 @@ func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, e for _, sci := range txn.SiacoinInputs { origin, err := getUTXOOrigin(sci.Parent.ID) if err != nil { - return fmt.Errorf("failed to get siacoin input source for %q: %w", sci.Parent.ID, err) + return api.ConsensusBlock{}, fmt.Errorf("failed to get siacoin input source for %q: %w", sci.Parent.ID, err) } apiTx.SiacoinInputs = append(apiTx.SiacoinInputs, api.ConsensusV2SiacoinInput{ @@ -1535,7 +1535,6 @@ func (s *Store) DecorateConsensusBlock(block types.Block) (api.ConsensusBlock, e cb.V2.Transactions = append(cb.V2.Transactions, apiTx) } - return nil + return cb, nil }) - return cb, err } diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go index c357c82..166ff7a 100644 --- a/persist/sqlite/events.go +++ b/persist/sqlite/events.go @@ -11,12 +11,12 @@ import ( // Events returns the events with the given event IDs. If an event is not found, // it is skipped. -func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) Events(eventIDs []types.Hash256) ([]wallet.Event, error) { + return valuedTransaction(s, func(tx *txn) ([]wallet.Event, error) { var scanHeight uint64 err := tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) if err != nil { - return fmt.Errorf("failed to get last indexed height: %w", err) + return nil, fmt.Errorf("failed to get last indexed height: %w", err) } // sqlite doesn't have easy support for IN clauses, use a statement since @@ -39,23 +39,22 @@ WHERE ev.event_id = $1` stmt, err := tx.Prepare(query) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return nil, fmt.Errorf("failed to prepare statement: %w", err) } defer stmt.Close() - events = make([]wallet.Event, 0, len(eventIDs)) + events := make([]wallet.Event, 0, len(eventIDs)) for _, id := range eventIDs { event, _, err := scanEvent(stmt.QueryRow(encode(id)), scanHeight) if errors.Is(err, sql.ErrNoRows) { continue } else if err != nil { - return fmt.Errorf("failed to query transaction %q: %w", id, err) + return nil, fmt.Errorf("failed to query transaction %q: %w", id, err) } events = append(events, event) } - return nil + return events, nil }) - return } func decodeEventData[T wallet.EventPayout | diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go index daca41f..57307ec 100644 --- a/persist/sqlite/peers.go +++ b/persist/sqlite/peers.go @@ -109,24 +109,23 @@ func (s *Store) AddPeer(peer string) error { } // Peers returns the addresses of all known peers. -func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { - err := s.transaction(func(tx *txn) error { +func (s *Store) Peers() ([]syncer.PeerInfo, error) { + return valuedTransaction(s, func(tx *txn) (peers []syncer.PeerInfo, _ error) { const query = `SELECT peer_address, first_seen FROM syncer_peers` rows, err := tx.Query(query) if err != nil { - return err + return nil, err } defer rows.Close() for rows.Next() { peer, err := scanPeerInfo(rows) if err != nil { - return fmt.Errorf("failed to scan peer info: %w", err) + return nil, fmt.Errorf("failed to scan peer info: %w", err) } peers = append(peers, peer) } - return rows.Err() + return peers, rows.Err() }) - return peers, err } // normalizePeer normalizes a peer address to a CIDR subnet. @@ -177,7 +176,7 @@ func (s *Store) Ban(peer string, duration time.Duration, reason string) error { } // Banned returns true if the peer is banned. -func (s *Store) Banned(peer string) (banned bool, _ error) { +func (s *Store) Banned(peer string) (bool, error) { // normalize the peer into a CIDR subnet peer, err := normalizePeer(peer) if err != nil { @@ -206,10 +205,10 @@ func (s *Store) Banned(peer string) (banned bool, _ error) { checkSubnets = append(checkSubnets, subnet.String()) } - err = s.transaction(func(tx *txn) error { + banned, err := valuedTransaction(s, func(tx *txn) (banned bool, _ error) { checkSubnetStmt, err := tx.Prepare(`SELECT expiration FROM syncer_bans WHERE net_cidr = $1 ORDER BY expiration DESC LIMIT 1`) if err != nil { - return fmt.Errorf("failed to prepare statement: %w", err) + return false, fmt.Errorf("failed to prepare statement: %w", err) } defer checkSubnetStmt.Close() @@ -219,13 +218,13 @@ func (s *Store) Banned(peer string) (banned bool, _ error) { err := checkSubnetStmt.QueryRow(subnet).Scan(decode(&expiration)) banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows if err != nil && !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("failed to check ban status: %w", err) + return false, fmt.Errorf("failed to check ban status: %w", err) } else if banned { s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) - return nil + return true, nil } } - return nil + return false, nil }) if err != nil && !errors.Is(err, sql.ErrNoRows) { return false, fmt.Errorf("failed to check ban status: %w", err) diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go index a0d6541..4a2ca32 100644 --- a/persist/sqlite/sql.go +++ b/persist/sqlite/sql.go @@ -3,6 +3,7 @@ package sqlite import ( "context" "database/sql" + "math/rand" "strings" "time" @@ -13,6 +14,11 @@ import ( const ( longQueryDuration = 10 * time.Millisecond longTxnDuration = time.Second // reduce syncing spam + + busyTimeout = 10 * time.Second + maxRetryAttempts = 30 // 30 attempts + factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds + maxBackoff = 15 * time.Second ) type ( @@ -184,6 +190,11 @@ func setDBVersion(tx *txn, version int64) error { return tx.QueryRow(query, version).Scan(&dbID) } +// jitterSleep sleeps for a random duration between t and t*1.5. +func jitterSleep(t time.Duration) { + time.Sleep(t + time.Duration(rand.Int63n(int64(t/2)))) +} + func queryPlaceHolders(n int) string { if n == 0 { return "" @@ -191,6 +202,17 @@ func queryPlaceHolders(n int) string { return strings.Repeat("?,", n-1) + "?" } +func anySlice[T any](args []T) []any { + if len(args) == 0 { + return nil + } + out := make([]any, len(args)) + for i, arg := range args { + out[i] = arg + } + return out +} + func encodeSlice[T any](args []T) []any { if len(args) == 0 { return nil diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go index 82b68ea..96e4dec 100644 --- a/persist/sqlite/store.go +++ b/persist/sqlite/store.go @@ -2,14 +2,17 @@ package sqlite import ( "database/sql" + "encoding/hex" "errors" "fmt" + "math" "strings" "time" "github.com/mattn/go-sqlite3" "go.sia.tech/walletd/v2/wallet" "go.uber.org/zap" + "lukechampine.com/frand" ) type ( @@ -29,40 +32,100 @@ func (s *Store) Close() error { // transaction executes a function within a database transaction. If the // function returns an error, the transaction is rolled back. Otherwise, the -// transaction is committed. +// transaction is committed. If the transaction fails due to a busy error, it is +// retried up to maxRetryAttempts times before returning. func (s *Store) transaction(fn func(*txn) error) error { - log := s.log.Named("transaction") - + var err error + txnID := hex.EncodeToString(frand.Bytes(4)) + log := s.log.Named("transaction").With(zap.String("id", txnID)) start := time.Now() - tx, err := s.db.Begin() + attempt := 1 + for ; attempt <= maxRetryAttempts; attempt++ { + attemptStart := time.Now() + log := log.With(zap.Int("attempt", attempt)) + err = doTransaction(s.db, log, fn) + if err == nil { + return nil + } + + // return immediately if the error is not a busy error + if !strings.Contains(err.Error(), "database is locked") { + break + } + // exponential backoff + sleep := min(time.Duration(math.Pow(factor, float64(attempt)))*time.Millisecond, maxBackoff) + log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) + jitterSleep(sleep) + } + return fmt.Errorf("transaction failed (attempt %d): %w", attempt, err) +} + +// doTransaction executes fn within a database transaction. If fn returns an +// error, the transaction is rolled back. Otherwise, the transaction is +// committed. +func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) (err error) { + dbtx, err := db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction: %w", err) } + start := time.Now() defer func() { - if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + if err := dbtx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { log.Error("failed to rollback transaction", zap.Error(err)) } + // log the transaction if it took longer than txn duration + if time.Since(start) > longTxnDuration { + log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) + } }() - if err := fn(&txn{ - Tx: tx, + + tx := &txn{ + Tx: dbtx, log: log, - }); err != nil { - return err } - // log the transaction if it took longer than txn duration - if time.Since(start) > longTxnDuration { - log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) - } - // commit the transaction - if err := tx.Commit(); err != nil { + if err := fn(tx); err != nil { + return err + } else if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } return nil } +// valuedTransaction executes fn within a database transaction and returns the +// value it produces. fn must build its result in local variables; nothing it +// produced is returned unless the transaction commits. +func valuedTransaction[T any](s *Store, fn func(*txn) (T, error)) (T, error) { + var v T + if err := s.transaction(func(tx *txn) error { + var err error + v, err = fn(tx) + return err + }); err != nil { + var zero T + return zero, err + } + return v, nil +} + +// valuedTransaction2 is [valuedTransaction] for functions returning two values. +func valuedTransaction2[T1, T2 any](s *Store, fn func(*txn) (T1, T2, error)) (T1, T2, error) { + var v1 T1 + var v2 T2 + if err := s.transaction(func(tx *txn) error { + var err error + v1, v2, err = fn(tx) + return err + }); err != nil { + var zero1 T1 + var zero2 T2 + return zero1, zero2, err + } + return v1, v2, nil +} + func sqliteFilepath(fp string) string { params := []string{ - fmt.Sprintf("_busy_timeout=%d", time.Minute.Milliseconds()), + fmt.Sprintf("_busy_timeout=%d", busyTimeout.Milliseconds()), "_foreign_keys=true", "_journal_mode=WAL", "_secure_delete=false", @@ -78,9 +141,6 @@ func OpenDatabase(fp string, opts ...Option) (*Store, error) { if err != nil { return nil, err } - // set the number of open connections to 1 to prevent "database is locked" - // errors - db.SetMaxOpenConns(1) store := &Store{ db: db, diff --git a/persist/sqlite/store_test.go b/persist/sqlite/store_test.go index 23aa14a..d55be8b 100644 --- a/persist/sqlite/store_test.go +++ b/persist/sqlite/store_test.go @@ -1,6 +1,7 @@ package sqlite import ( + "errors" "path/filepath" "testing" @@ -23,3 +24,63 @@ func newTestStore(t testing.TB, opts ...Option) *Store { }) return db } + +func TestTransactionRetry(t *testing.T) { + db := newTestStore(t) + + t.Run("retries busy errors", func(t *testing.T) { + var attempts int + err := db.transaction(func(tx *txn) error { + attempts++ + if attempts < 3 { + return errors.New("database is locked") + } + return nil + }) + if err != nil { + t.Fatal(err) + } else if attempts != 3 { + t.Fatalf("expected 3 attempts, got %d", attempts) + } + }) + + t.Run("does not retry other errors", func(t *testing.T) { + expected := errors.New("constraint violation") + var attempts int + err := db.transaction(func(tx *txn) error { + attempts++ + return expected + }) + if !errors.Is(err, expected) { + t.Fatalf("expected %v, got %v", expected, err) + } else if attempts != 1 { + t.Fatalf("expected 1 attempt, got %d", attempts) + } + }) + + t.Run("rolls back a failed attempt", func(t *testing.T) { + var attempts int + err := db.transaction(func(tx *txn) error { + attempts++ + if _, err := tx.Exec(`INSERT INTO syncer_peers (peer_address, first_seen) VALUES (?, ?)`, "1.2.3.4:9981", 0); err != nil { + return err + } + if attempts < 2 { + return errors.New("database is locked") + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + var count int + if err := db.transaction(func(tx *txn) error { + return tx.QueryRow(`SELECT COUNT(*) FROM syncer_peers WHERE peer_address=?`, "1.2.3.4:9981").Scan(&count) + }); err != nil { + t.Fatal(err) + } else if count != 1 { + t.Fatalf("expected 1 peer, got %d", count) + } + }) +} diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go index 55f428e..bb32646 100644 --- a/persist/sqlite/utxo.go +++ b/persist/sqlite/utxo.go @@ -58,82 +58,73 @@ WHERE se.id=$1 AND spent_index_id IS NULL` } // SiacoinElement returns an unspent Siacoin UTXO by its ID. -func (s *Store) SiacoinElement(id types.SiacoinOutputID) (ele types.SiacoinElement, err error) { - err = s.transaction(func(tx *txn) error { - ele, err = getSiacoinElement(tx, id, s.indexMode) +func (s *Store) SiacoinElement(id types.SiacoinOutputID) (types.SiacoinElement, error) { + return valuedTransaction(s, func(tx *txn) (types.SiacoinElement, error) { + ele, err := getSiacoinElement(tx, id, s.indexMode) if errors.Is(err, sql.ErrNoRows) { - return wallet.ErrNotFound + return types.SiacoinElement{}, wallet.ErrNotFound } - return err + return ele, err }) - return } // SiafundElement returns an unspent Siafund UTXO by its ID. -func (s *Store) SiafundElement(id types.SiafundOutputID) (ele types.SiafundElement, err error) { - err = s.transaction(func(tx *txn) error { - ele, err = getSiafundElement(tx, id, s.indexMode) +func (s *Store) SiafundElement(id types.SiafundOutputID) (types.SiafundElement, error) { + return valuedTransaction(s, func(tx *txn) (types.SiafundElement, error) { + ele, err := getSiafundElement(tx, id, s.indexMode) if errors.Is(err, sql.ErrNoRows) { - return wallet.ErrNotFound + return types.SiafundElement{}, wallet.ErrNotFound } - return err + return ele, err }) - return } // SiacoinElementSpentEvent returns the event that spent a Siacoin UTXO. -func (s *Store) SiacoinElementSpentEvent(id types.SiacoinOutputID) (ev wallet.Event, spent bool, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) SiacoinElementSpentEvent(id types.SiacoinOutputID) (wallet.Event, bool, error) { + return valuedTransaction2(s, func(tx *txn) (wallet.Event, bool, error) { const query = `SELECT spent_event_id FROM siacoin_elements WHERE id=$1` var spentEventID sql.NullInt64 - err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + err := tx.QueryRow(query, encode(id)).Scan(&spentEventID) if errors.Is(err, sql.ErrNoRows) { - return wallet.ErrNotFound + return wallet.Event{}, false, wallet.ErrNotFound } else if err != nil { - return fmt.Errorf("failed to query spent event ID: %w", err) + return wallet.Event{}, false, fmt.Errorf("failed to query spent event ID: %w", err) } else if !spentEventID.Valid { - return nil + return wallet.Event{}, false, nil } - spent = true events, err := getEventsByID(tx, []int64{spentEventID.Int64}) if err != nil { - return fmt.Errorf("failed to get events by ID: %w", err) + return wallet.Event{}, false, fmt.Errorf("failed to get events by ID: %w", err) } else if len(events) != 1 { panic("expected exactly one event") // should never happen } - ev = events[0] - return nil + return events[0], true, nil }) - return } // SiafundElementSpentEvent returns the event that spent a Siafund UTXO. -func (s *Store) SiafundElementSpentEvent(id types.SiafundOutputID) (ev wallet.Event, spent bool, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) SiafundElementSpentEvent(id types.SiafundOutputID) (wallet.Event, bool, error) { + return valuedTransaction2(s, func(tx *txn) (wallet.Event, bool, error) { const query = `SELECT spent_event_id FROM siafund_elements WHERE id=$1` var spentEventID sql.NullInt64 - err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + err := tx.QueryRow(query, encode(id)).Scan(&spentEventID) if errors.Is(err, sql.ErrNoRows) { - return wallet.ErrNotFound + return wallet.Event{}, false, wallet.ErrNotFound } else if err != nil { - return fmt.Errorf("failed to query spent event ID: %w", err) + return wallet.Event{}, false, fmt.Errorf("failed to query spent event ID: %w", err) } else if !spentEventID.Valid { - return nil + return wallet.Event{}, false, nil } - spent = true events, err := getEventsByID(tx, []int64{spentEventID.Int64}) if err != nil { - return fmt.Errorf("failed to get events by ID: %w", err) + return wallet.Event{}, false, fmt.Errorf("failed to get events by ID: %w", err) } else if len(events) != 1 { panic("expected exactly one event") // should never happen } - ev = events[0] - return nil + return events[0], true, nil }) - - return } diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go index 0bcc48a..1152655 100644 --- a/persist/sqlite/wallet.go +++ b/persist/sqlite/wallet.go @@ -51,29 +51,28 @@ WHERE wa.wallet_id=? AND ea.event_id=?`) } // WalletEvents returns the events relevant to a wallet, sorted by height descending. -func (s *Store) WalletEvents(id wallet.ID, offset, limit int) (events []wallet.Event, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) { + return valuedTransaction(s, func(tx *txn) ([]wallet.Event, error) { dbIDs, err := getWalletEvents(tx, id, offset, limit) if err != nil { - return fmt.Errorf("failed to get wallet events: %w", err) + return nil, fmt.Errorf("failed to get wallet events: %w", err) } - events, err = getEventsByID(tx, dbIDs) + events, err := getEventsByID(tx, dbIDs) if err != nil { - return fmt.Errorf("failed to get events by ID: %w", err) + return nil, fmt.Errorf("failed to get events by ID: %w", err) } eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, id, dbIDs) if err != nil { - return fmt.Errorf("failed to get relevant addresses: %w", err) + return nil, fmt.Errorf("failed to get relevant addresses: %w", err) } for i := range events { events[i].Relevant = eventRelevantAddresses[dbIDs[i]] } - return nil + return events, nil }) - return } // AddWallet adds a wallet to the database. @@ -81,26 +80,29 @@ func (s *Store) AddWallet(w wallet.Wallet) (wallet.Wallet, error) { w.DateCreated = time.Now().Truncate(time.Second) w.LastUpdated = time.Now().Truncate(time.Second) - err := s.transaction(func(tx *txn) error { + return valuedTransaction(s, func(tx *txn) (wallet.Wallet, error) { const query = `INSERT INTO wallets (friendly_name, description, date_created, last_updated, extra_data) VALUES ($1, $2, $3, $4, $5) RETURNING id` - return tx.QueryRow(query, w.Name, w.Description, encode(w.DateCreated), encode(w.LastUpdated), w.Metadata).Scan(&w.ID) + if err := tx.QueryRow(query, w.Name, w.Description, encode(w.DateCreated), encode(w.LastUpdated), w.Metadata).Scan(&w.ID); err != nil { + return wallet.Wallet{}, err + } + return w, nil }) - return w, err } // UpdateWallet updates a wallet in the database. func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { w.LastUpdated = time.Now() - err := s.transaction(func(tx *txn) error { + return valuedTransaction(s, func(tx *txn) (wallet.Wallet, error) { var dummyID int64 const query = `UPDATE wallets SET friendly_name=$1, description=$2, last_updated=$3, extra_data=$4 WHERE id=$5 RETURNING id, date_created, last_updated` err := tx.QueryRow(query, w.Name, w.Description, encode(w.LastUpdated), w.Metadata, w.ID).Scan(&dummyID, decode(&w.DateCreated), decode(&w.LastUpdated)) if errors.Is(err, sql.ErrNoRows) { - return wallet.ErrNotFound + return wallet.Wallet{}, wallet.ErrNotFound + } else if err != nil { + return wallet.Wallet{}, err } - return err + return w, nil }) - return w, err } // DeleteWallet deletes a wallet from the database. This does not stop tracking @@ -122,26 +124,25 @@ func (s *Store) DeleteWallet(id wallet.ID) error { } // Wallets returns a map of wallet names to wallet extra data. -func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) Wallets() ([]wallet.Wallet, error) { + return valuedTransaction(s, func(tx *txn) (wallets []wallet.Wallet, _ error) { const query = `SELECT id, friendly_name, description, date_created, last_updated, extra_data FROM wallets` rows, err := tx.Query(query) if err != nil { - return err + return nil, err } defer rows.Close() for rows.Next() { var w wallet.Wallet if err := rows.Scan(&w.ID, &w.Name, &w.Description, decode(&w.DateCreated), decode(&w.LastUpdated), (*[]byte)(&w.Metadata)); err != nil { - return fmt.Errorf("failed to scan wallet: %w", err) + return nil, fmt.Errorf("failed to scan wallet: %w", err) } wallets = append(wallets, w) } - return rows.Err() + return wallets, rows.Err() }) - return } // AddWalletAddresses adds the given addresses to a wallet. @@ -201,10 +202,10 @@ func (s *Store) RemoveWalletAddress(id wallet.ID, address types.Address) error { } // WalletAddress returns an address registered to the wallet. -func (s *Store) WalletAddress(id wallet.ID, address types.Address) (addr wallet.Address, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletAddress(id wallet.ID, address types.Address) (wallet.Address, error) { + return valuedTransaction(s, func(tx *txn) (wallet.Address, error) { if err := walletExists(tx, id); err != nil { - return err + return wallet.Address{}, err } const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data @@ -212,17 +213,15 @@ FROM wallet_addresses wa INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1 AND sa.sia_address=$2` - addr, err = scanWalletAddress(tx.QueryRow(query, id, encode(address))) - return err + return scanWalletAddress(tx.QueryRow(query, id, encode(address))) }) - return } // WalletAddresses returns a slice of addresses registered to the wallet. -func (s *Store) WalletAddresses(id wallet.ID) (addresses []wallet.Address, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletAddresses(id wallet.ID) ([]wallet.Address, error) { + return valuedTransaction(s, func(tx *txn) (addresses []wallet.Address, _ error) { if err := walletExists(tx, id); err != nil { - return err + return nil, err } const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data @@ -232,32 +231,31 @@ WHERE wa.wallet_id=$1` rows, err := tx.Query(query, id) if err != nil { - return err + return nil, err } defer rows.Close() for rows.Next() { addr, err := scanWalletAddress(rows) if err != nil { - return fmt.Errorf("failed to scan address: %w", err) + return nil, fmt.Errorf("failed to scan address: %w", err) } addresses = append(addresses, addr) } - return rows.Err() + return addresses, rows.Err() }) - return } // WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. -func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) ([]wallet.UnspentSiacoinElement, types.ChainIndex, error) { + return valuedTransaction2(s, func(tx *txn) (siacoins []wallet.UnspentSiacoinElement, basis types.ChainIndex, _ error) { if err := walletExists(tx, id); err != nil { - return err + return nil, types.ChainIndex{}, err } - basis, err = getScanBasis(tx) + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address, ci.height @@ -269,21 +267,21 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins rows, err := tx.Query(query, basis.Height, id, limit, offset) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siacoin, err := scanUnspentSiacoinElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siacoin element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siacoin element: %w", err) } siacoins = append(siacoins, siacoin) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siacoin elements @@ -294,27 +292,26 @@ func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siacoins[i].StateElement.MerkleProof = proof } } - return nil + return siacoins, basis, nil }) - return } // WalletSiafundOutputs returns the unspent siafund outputs for a wallet. -func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) ([]wallet.UnspentSiafundElement, types.ChainIndex, error) { + return valuedTransaction2(s, func(tx *txn) (siafunds []wallet.UnspentSiafundElement, basis types.ChainIndex, _ error) { if err := walletExists(tx, id); err != nil { - return err + return nil, types.ChainIndex{}, err } - basis, err = getScanBasis(tx) + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to get basis: %w", err) } const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address, ci.height @@ -326,19 +323,19 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds rows, err := tx.Query(query, id, limit, offset) if err != nil { - return err + return nil, types.ChainIndex{}, err } defer rows.Close() for rows.Next() { siafund, err := scanUnspentSiafundElement(rows, basis.Height) if err != nil { - return fmt.Errorf("failed to scan siafund element: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to scan siafund element: %w", err) } siafunds = append(siafunds, siafund) } if err := rows.Err(); err != nil { - return err + return nil, types.ChainIndex{}, err } // retrieve the merkle proofs for the siacoin elements @@ -349,22 +346,21 @@ func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds } proofs, err := fillElementProofs(tx, indices) if err != nil { - return fmt.Errorf("failed to fill element proofs: %w", err) + return nil, types.ChainIndex{}, fmt.Errorf("failed to fill element proofs: %w", err) } for i, proof := range proofs { siafunds[i].StateElement.MerkleProof = proof } } - return nil + return siafunds, basis, nil }) - return } // WalletBalance returns the total balance of a wallet. -func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletBalance(id wallet.ID) (wallet.Balance, error) { + return valuedTransaction(s, func(tx *txn) (balance wallet.Balance, _ error) { if err := walletExists(tx, id); err != nil { - return err + return wallet.Balance{}, err } const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses sa @@ -373,7 +369,7 @@ func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) rows, err := tx.Query(query, id) if err != nil { - return err + return wallet.Balance{}, err } defer rows.Close() @@ -383,30 +379,29 @@ func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) var addressSF uint64 if err := rows.Scan(decode(&addressSC), decode(&addressISC), &addressSF); err != nil { - return fmt.Errorf("failed to scan address balance: %w", err) + return wallet.Balance{}, fmt.Errorf("failed to scan address balance: %w", err) } balance.Siacoins = balance.Siacoins.Add(addressSC) balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(addressISC) balance.Siafunds += addressSF } - return rows.Err() + return balance, rows.Err() }) - return } // WalletUnconfirmedEvents annotates a list of unconfirmed transactions with // relevant addresses and siacoin/siafund elements. -func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []wallet.Event, err error) { - err = s.transaction(func(tx *txn) error { +func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) ([]wallet.Event, error) { + return valuedTransaction(s, func(tx *txn) (annotated []wallet.Event, _ error) { if err := walletExists(tx, id); err != nil { - return err + return nil, err } addrStmt, err := tx.Prepare(`SELECT sa.id FROM sia_addresses sa INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1`) if err != nil { - return fmt.Errorf("failed to prepare address statement: %w", err) + return nil, fmt.Errorf("failed to prepare address statement: %w", err) } defer addrStmt.Close() @@ -437,7 +432,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) if err != nil { - return fmt.Errorf("failed to prepare siacoin statement: %w", err) + return nil, fmt.Errorf("failed to prepare siacoin statement: %w", err) } defer siacoinElementStmt.Close() @@ -460,7 +455,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti INNER JOIN sia_addresses sa ON (se.address_id = sa.id) WHERE se.id=$1`) if err != nil { - return fmt.Errorf("failed to prepare siafund statement: %w", err) + return nil, fmt.Errorf("failed to prepare siafund statement: %w", err) } defer siafundElementStmt.Close() @@ -511,7 +506,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti // fetch the siacoin element sce, err := fetchSiacoinElement(input.ParentID) if err != nil { - return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + return nil, fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) } ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) } @@ -550,7 +545,7 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti // fetch the siafund element sfe, err := fetchSiafundElement(input.ParentID) if err != nil { - return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + return nil, fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) } ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) } @@ -625,9 +620,8 @@ func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, ti addEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, wallet.EventV2Transaction(txn), relevant) } - return nil + return annotated, nil }) - return } func scanUnspentSiacoinElement(s scanner, basisHeight uint64) (se wallet.UnspentSiacoinElement, err error) { @@ -796,11 +790,11 @@ func walletExists(tx *txn, id wallet.ID) error { } // OverwriteElementProofs overwrites the element proofs for the given transactions. -func (s *Store) OverwriteElementProofs(txns []types.V2Transaction) (basis types.ChainIndex, updated []types.V2Transaction, err error) { - err = s.transaction(func(tx *txn) error { - basis, err = getScanBasis(tx) +func (s *Store) OverwriteElementProofs(txns []types.V2Transaction) (types.ChainIndex, []types.V2Transaction, error) { + return valuedTransaction2(s, func(tx *txn) (basis types.ChainIndex, updated []types.V2Transaction, _ error) { + basis, err := getScanBasis(tx) if err != nil { - return fmt.Errorf("failed to get basis: %w", err) + return types.ChainIndex{}, nil, fmt.Errorf("failed to get basis: %w", err) } for _, txn := range txns { @@ -810,7 +804,7 @@ func (s *Store) OverwriteElementProofs(txns []types.V2Transaction) (basis types. if errors.Is(err, sql.ErrNoRows) { continue } else if err != nil { - return fmt.Errorf("failed to get siacoin element: %w", err) + return types.ChainIndex{}, nil, fmt.Errorf("failed to get siacoin element: %w", err) } txn.SiacoinInputs[i].Parent = ele } @@ -819,13 +813,12 @@ func (s *Store) OverwriteElementProofs(txns []types.V2Transaction) (basis types. if errors.Is(err, sql.ErrNoRows) { continue } else if err != nil { - return fmt.Errorf("failed to get siafund element: %w", err) + return types.ChainIndex{}, nil, fmt.Errorf("failed to get siafund element: %w", err) } txn.SiafundInputs[i].Parent = ele } updated = append(updated, txn) } - return nil + return basis, updated, nil }) - return } diff --git a/wallet/addresses_test.go b/wallet/addresses_test.go index 0513c6f..6143e6e 100644 --- a/wallet/addresses_test.go +++ b/wallet/addresses_test.go @@ -109,6 +109,51 @@ func TestBatchAddresses(t *testing.T) { } } +func TestBatchAddressBalance(t *testing.T) { + network, genesisBlock := testutil.V2Network() + tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) + wm := tn.manager + + addresses := make([]types.Address, 10) + for i := range addresses { + addresses[i] = types.StandardAddress(types.GeneratePrivateKey().PublicKey()) + tn.MineBlocks(t, addresses[i], i+1) + } + tn.MineBlocks(t, types.VoidAddress, int(network.MaturityDelay)) + + var expected wallet.Balance + for _, addr := range addresses { + b, err := wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } + expected.Siacoins = expected.Siacoins.Add(b.Siacoins) + expected.ImmatureSiacoins = expected.ImmatureSiacoins.Add(b.ImmatureSiacoins) + expected.Siafunds += b.Siafunds + } + if expected.Siacoins.IsZero() { + t.Fatal("expected a non-zero balance") + } + + balance, err := wm.AddressBalance(addresses...) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expected.Siacoins) { + t.Fatalf("expected %v siacoins, got %v", expected.Siacoins, balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(expected.ImmatureSiacoins) { + t.Fatalf("expected %v immature siacoins, got %v", expected.ImmatureSiacoins, balance.ImmatureSiacoins) + } else if balance.Siafunds != expected.Siafunds { + t.Fatalf("expected %v siafunds, got %v", expected.Siafunds, balance.Siafunds) + } + + withUnknown := append(addresses, types.StandardAddress(types.GeneratePrivateKey().PublicKey())) + if balance, err := wm.AddressBalance(withUnknown...); err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expected.Siacoins) { + t.Fatalf("expected %v siacoins, got %v", expected.Siacoins, balance.Siacoins) + } +} + func TestBatchSiacoinOutputs(t *testing.T) { network, genesisBlock := testutil.V2Network() tn := newTestNode(t, network, genesisBlock, wallet.WithIndexMode(wallet.IndexModeFull)) From ea75c54fb25a273f57ac00e967b57a5b02ffaef4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:35:58 +0000 Subject: [PATCH 628/630] build(deps): bump github.com/mattn/go-sqlite3 Bumps the all-dependencies group with 1 update: [github.com/mattn/go-sqlite3](https://github.com/mattn/go-sqlite3). Updates `github.com/mattn/go-sqlite3` from 1.14.49 to 1.14.50 - [Release notes](https://github.com/mattn/go-sqlite3/releases) - [Commits](https://github.com/mattn/go-sqlite3/compare/v1.14.49...v1.14.50) --- updated-dependencies: - dependency-name: github.com/mattn/go-sqlite3 dependency-version: 1.14.50 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 906bc77..56f08eb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.sia.tech/walletd/v2 // v2.15.2 go 1.26.0 require ( - github.com/mattn/go-sqlite3 v1.14.49 + github.com/mattn/go-sqlite3 v1.14.50 go.sia.tech/core v0.21.7 go.sia.tech/coreutils v0.24.0 go.sia.tech/jape v0.14.1 diff --git a/go.sum b/go.sum index 50febff..924faf5 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= -github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY= +github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0= From 99ff3504bce163c2795ef95c94ee03717d0d2e2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:33:10 +0000 Subject: [PATCH 629/630] build(deps): bump go.sia.tech/jape in the all-dependencies group Bumps the all-dependencies group with 1 update: [go.sia.tech/jape](https://github.com/SiaFoundation/jape). Updates `go.sia.tech/jape` from 0.14.1 to 0.14.2 - [Release notes](https://github.com/SiaFoundation/jape/releases) - [Changelog](https://github.com/SiaFoundation/jape/blob/master/CHANGELOG.md) - [Commits](https://github.com/SiaFoundation/jape/compare/v0.14.1...v0.14.2) --- updated-dependencies: - dependency-name: go.sia.tech/jape dependency-version: 0.14.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 9 ++++++++- go.sum | 28 ++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56f08eb..802cdea 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.50 go.sia.tech/core v0.21.7 go.sia.tech/coreutils v0.24.0 - go.sia.tech/jape v0.14.1 + go.sia.tech/jape v0.14.2 go.sia.tech/web/walletd v0.36.2 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 @@ -18,15 +18,22 @@ require ( ) require ( + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.2 // indirect + github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.61.0 // indirect github.com/quic-go/webtransport-go v0.12.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/bbolt v1.5.0 // indirect go.sia.tech/mux v1.5.3 // indirect go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 924faf5..94ba3e0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,12 @@ +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.2 h1:90H+rcF/FwLXwfB1cudOLq/je83n683Utf4Cbp0xHCo= +github.com/bytedance/sonic v1.15.2/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= +github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= +github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= @@ -6,6 +15,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -24,16 +35,26 @@ github.com/quic-go/webtransport-go v0.12.0 h1:CpnKNwZvdV0LD73xoHO8QaR0NI3llqpWRw github.com/quic-go/webtransport-go v0.12.0/go.mod h1:GHne8aRFJ24h73pAMrcywXtuaz/ShBXCLXLvG/NPFdU= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= go.sia.tech/core v0.21.7 h1:Qgi2293i/d+UfpuVGlAcXfDY0Vzkj/GTjpkuEBXmIks= go.sia.tech/core v0.21.7/go.mod h1:80xXoUUnfIFVazv7i4qZH4e/+kbxSadd4B3EK1+MOtw= go.sia.tech/coreutils v0.24.0 h1:xz3CJ3SS38cGTF6WVxmZ4dR6t+2LostNscyZ6n2hbOI= go.sia.tech/coreutils v0.24.0/go.mod h1:xNzCC31sJkKXVnEjv12aHXutRfc4nPDl53sVJQhw6+k= -go.sia.tech/jape v0.14.1 h1:3QWpOzAxxcaECuv2Mc6tbkFh+olLnyUxxP5SfwgI/qY= -go.sia.tech/jape v0.14.1/go.mod h1:BEygF1DcgdWBenPa75iJ1b91FuxzLmKBxpglmx+C9BY= +go.sia.tech/jape v0.14.2 h1:lf11qkRFy/r+f/h++Vr+8pvfiwG95V8o8zxtsaBn7rE= +go.sia.tech/jape v0.14.2/go.mod h1:OO8uqguBaVzyOU0bg/4YGp1Lm0fhlH+kz5FSYKcbAaU= go.sia.tech/mux v1.5.3 h1:0LSoSUMUThKYYHPha3i3YADdfBVaGHcI8UKXhALndHg= go.sia.tech/mux v1.5.3/go.mod h1:cYRXgCdhC5kH+8f6knyQJ2Wzk6kB7Ndam1KuEt4gI9M= go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= @@ -50,6 +71,8 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= @@ -71,6 +94,7 @@ golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= From 59d67b1626367a430294a604e4a6cff5056f21f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 08:27:05 +0000 Subject: [PATCH 630/630] chore: prepare release 2.16.0 --- ...rrent_read_transactions_for_improved_scalability.md | 5 ----- .changeset/update_coreutils_to_v0240.md | 5 ----- CHANGELOG.md | 10 ++++++++++ go.mod | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 .changeset/enabled_concurrent_read_transactions_for_improved_scalability.md delete mode 100644 .changeset/update_coreutils_to_v0240.md diff --git a/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md b/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md deleted file mode 100644 index f34a959..0000000 --- a/.changeset/enabled_concurrent_read_transactions_for_improved_scalability.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: minor ---- - -# Enabled concurrent read transactions for improved scalability. diff --git a/.changeset/update_coreutils_to_v0240.md b/.changeset/update_coreutils_to_v0240.md deleted file mode 100644 index 86cb8b4..0000000 --- a/.changeset/update_coreutils_to_v0240.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -default: patch ---- - -# Update coreutils to v0.24.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index b2c8ad8..5d5fc92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 2.16.0 (2026-09-01) + +### Features + +- Enabled concurrent read transactions for improved scalability. + +### Fixes + +- Update coreutils to v0.24.0 + ## 2.15.2 (2026-07-08) ### Fixes diff --git a/go.mod b/go.mod index 802cdea..a72e92c 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module go.sia.tech/walletd/v2 // v2.15.2 +module go.sia.tech/walletd/v2 // v2.16.0 go 1.26.0