diff --git a/.changeset/add_signing_key_store.md b/.changeset/add_signing_key_store.md new file mode 100644 index 0000000..217495d --- /dev/null +++ b/.changeset/add_signing_key_store.md @@ -0,0 +1,26 @@ +--- +default: minor +--- + +# Add ed25519 key store + +Adds an optional ed25519 key store for integrators to store arbitrary private keys for signing transactions. It allows for both generating private keys on the server and importing private keys. Keys are stored encrypted using a user-provided secret. + +The store is disabled by default. It can be enabled through the config file or the CLI flag `--keystore`. If the store is enabled, an encryption key must also be provided through the environment variable `WALLETD_KEYSTORE_SECRET`. + +*The endpoint will return 404 if the `--public` CLI flag is set. It is only recommended for use on localhost. It is not used by the UI.* + +```go + +client := api.NewClient(walletAddr, walletdPassword) + +pubKey, err := client.GenerateSigningKey() +if err != nil { + panic(err) +} + +sig, err := client.SignHash(pubKey, hash) +if err != nil { + panic(err) +} +``` diff --git a/.changeset/add_spent_element_endpoints.md b/.changeset/add_spent_element_endpoints.md new file mode 100644 index 0000000..a117027 --- /dev/null +++ b/.changeset/add_spent_element_endpoints.md @@ -0,0 +1,17 @@ +--- +default: minor +--- + +# Added Spent Element Endpoints + +Added two new endpoints `[GET] /outputs/siacoin/:id/spent` and `[GET] /outputs/siafund/:id/spent`. These endpoints will return a boolean, indicating whether the UTXO was spent, and the transaction it was spent in. These endpoints are designed to make verifying Atomic swaps easier. + +#### Example Usage + +```` +$ curl http://localhost:9980/api/outputs/siacoin/9b89152bb967130326702c9bfb51109e9f80274ec314ba58d9ef49b881340f2f/spent +{ + spent: true, + event: {} +} +``` diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..32da25c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,57 @@ +name: Bug Report +description: File a bug report +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + Note: Please search to see if an issue already exists for the bug you encountered. + - type: textarea + id: current-behavior + attributes: + label: Current Behavior + description: A concise description of what you're experiencing. + placeholder: Tell us what you see! + validations: + required: true + - type: textarea + id: expected-behavior + attributes: + label: Expected Behavior + description: A concise description of what you expected to happen. + placeholder: Tell us what you want to see! + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to Reproduce + description: Detailed steps to reproduce the behavior. + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + - type: input + id: version + attributes: + label: Version + description: What version of walletd are you running? If you are running from source, please provide the commit hash. + placeholder: v0.8.0 + validations: + required: true + - type: input + id: os + attributes: + label: What operating system did the problem occur on (e.g. Ubuntu 22.04, macOS 12.0, Windows 11)? + validations: + required: true + - type: textarea + attributes: + label: Anything else? + description: | + Links? References? Anything that will give us more context about the issue you are encountering! + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d8e8e7d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Sia Community Discord + url: https://discord.gg/sia + about: Join the Sia community discord for more help with Sia or walletd. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..c6b7f35 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature Request +description: Request a new feature be added to walletd. +labels: ["feature"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + Note: Please search to see if an issue already exists for the feature + you want added. + - type: textarea + id: feature-description + attributes: + label: Description + description: | + A description of the feature you want added + Tip: You can attach images by clicking this area and then dragging files in. + placeholder: Tell us what you want! Be as descriptive as possible. + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of walletd are you running? + placeholder: v0.8.0 + validations: + required: false + - type: input + id: os + attributes: + label: What operating system are you running (e.g. Ubuntu 22.04, macOS, Windows 11)? + validations: + required: false + - type: textarea + attributes: + label: Anything else? + description: | + Links? References? Anything that will give us more context about the feature! + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: false \ No newline at end of file diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml deleted file mode 100644 index c44a2c0..0000000 --- a/.github/actions/test/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Test -description: Lints and tests walletd - -runs: - using: composite - steps: - - name: Configure git # required for golangci-lint on Windows - shell: bash - run: git config --global core.autocrlf false -# - name: Lint -# uses: golangci/golangci-lint-action@v3 -# with: -# skip-cache: true - - name: Analyze - uses: SiaFoundation/action-golang-analysis@HEAD - with: - analyzers: | - go.sia.tech/jape.Analyzer - - name: Test - uses: n8maninger/action-golang-test@v1 - with: - args: "-race;-tags=testing netgo" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1996ae0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "gomod" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + groups: + all-dependencies: + patterns: + - "*" \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e59abc..f3e416e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,28 +1,13 @@ -name: Main +name: Lint & Test on: - workflow_dispatch: pull_request: push: branches: - master +env: + CGO_ENABLED: 1 + jobs: test: - runs-on: ${{ matrix.os }} - permissions: - contents: read - strategy: - matrix: - os: [ ubuntu-latest , macos-latest, windows-latest ] - go-version: [ '1.19', '1.20' ] - steps: - - name: Configure git - run: git config --global core.autocrlf false # required on Windows - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Test - uses: ./.github/actions/test - - name: Build - run: go build -o bin/ ./cmd/walletd + uses: SiaFoundation/workflows/.github/workflows/go-test.yml@master diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000..7b6ca01 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,26 @@ +on: + push: + branches: [master] + +permissions: + contents: write + pull-requests: write + +name: Create Release PR +jobs: + prepare-release: + if: "!contains(github.event.head_commit.message, 'chore: prepare release')" # Skip merges from releases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config --global user.name github-actions[bot] + git config --global user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: knope-dev/action@407e9ef7c272d2dd53a4e71e39a7839e29933c48 + - run: knope prepare-release --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true diff --git a/.github/workflows/project-add.yml b/.github/workflows/project-add.yml new file mode 100644 index 0000000..8b93101 --- /dev/null +++ b/.github/workflows/project-add.yml @@ -0,0 +1,14 @@ +name: Add issues and PRs to Sia project + +on: + issues: + types: + - opened + pull_request: + types: + - opened + +jobs: + add-to-project: + uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master + secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12c19cb..5849c36 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,205 +6,23 @@ on: push: branches: - master - - its-happening tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-**' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-**" -jobs: - test: - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Test - uses: ./.github/actions/test - docker: - runs-on: ubuntu-latest - needs: [ test ] - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v3 - - uses: docker/setup-qemu-action@v2 - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/metadata-action@v4 - name: generate tags - id: meta - with: - images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} - tags: | - type=ref,event=branch - type=sha,prefix= - type=semver,pattern={{version}} - - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - build-linux: - runs-on: ubuntu-latest - needs: [ test ] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Setup - run: | - sudo apt update - sudo apt install -y gcc-aarch64-linux-gnu - go generate ./... - - name: Build amd64 - env: - CGO_ENABLED: 1 - GOOS: linux - GOARCH: amd64 - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - zip -qj $ZIP_OUTPUT bin/* - - name: Build arm64 - env: - CGO_ENABLED: 1 - GOOS: linux - GOARCH: arm64 - CC: aarch64-linux-gnu-gcc - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - zip -qj $ZIP_OUTPUT bin/* - - uses: actions/upload-artifact@v3 - with: - name: walletd - path: release/ - build-mac: - runs-on: macos-latest - needs: [ test ] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Setup - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - run: | - # extract apple cert - APPLE_CERT_PATH=$RUNNER_TEMP/apple_cert.p12 - KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db - echo -n "$APPLE_CERT_B64" | base64 --decode --output $APPLE_CERT_PATH - - # extract apple key - mkdir -p ~/private_keys - APPLE_API_KEY_PATH=~/private_keys/AuthKey_$APPLE_API_KEY.p8 - echo -n "$APPLE_KEY_B64" | base64 --decode --output $APPLE_API_KEY_PATH - - # create temp keychain - security create-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH - security default-keychain -s $KEYCHAIN_PATH - security set-keychain-settings -lut 21600 $KEYCHAIN_PATH - security unlock-keychain -p "$APPLE_KEYCHAIN_PASSWORD" $KEYCHAIN_PATH +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false - # import keychain - security import $APPLE_CERT_PATH -P $APPLE_CERT_PASSWORD -A -t cert -f pkcs12 -k $KEYCHAIN_PATH - security find-identity -v $KEYCHAIN_PATH -p codesigning - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $APPLE_KEYCHAIN_PASSWORD $KEYCHAIN_PATH - - # generate - go generate ./... - - name: Build amd64 - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - CGO_ENABLED: 1 - GOOS: darwin - GOARCH: amd64 - run: | - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - mkdir -p release - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/walletd - ditto -ck bin $ZIP_OUTPUT - xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - name: Build arm64 - env: - APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} - APPLE_API_KEY: ${{ secrets.APPLE_API_KEY }} - APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }} - APPLE_KEY_B64: ${{ secrets.APPLE_KEY_B64 }} - APPLE_CERT_B64: ${{ secrets.APPLE_CERT_B64 }} - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }} - CGO_ENABLED: 1 - GOOS: darwin - GOARCH: arm64 - run: | - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - mkdir -p release - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/walletd - cp README.md LICENSE bin/ - /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/walletd - ditto -ck bin $ZIP_OUTPUT - xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - uses: actions/upload-artifact@v3 - with: - name: walletd - path: release/ - build-windows: - runs-on: windows-latest - needs: [ test ] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: 'stable' - - name: Setup - shell: bash - run: | - dotnet tool install --global AzureSignTool - go generate ./... - - name: Build amd64 - env: - CGO_ENABLED: 1 - GOOS: windows - GOARCH: amd64 - shell: bash - run: | - mkdir -p release - ZIP_OUTPUT=release/walletd_${GOOS}_${GOARCH}.zip - go build -tags='netgo' -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd - azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/walletd.exe - cp README.md LICENSE bin/ - 7z a $ZIP_OUTPUT ./bin/* - - uses: actions/upload-artifact@v3 - with: - name: walletd - path: release/ +jobs: + publish: + uses: SiaFoundation/workflows/.github/workflows/go-publish.yml@master + secrets: inherit + with: + linux-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + windows-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' + macos-build-args: -tags='timetzdata netgo' -trimpath -a -ldflags '-s -w' + cgo-enabled: 1 + project: walletd + project-desc: "walletd: The new Sia wallet" + version-tag: ${{ github.ref_name }} diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml new file mode 100644 index 0000000..a91153b --- /dev/null +++ b/.github/workflows/ui.yml @@ -0,0 +1,18 @@ +name: Update UI and open PR + +on: + repository_dispatch: + types: [update-ui] + # Enable manual trigger + workflow_dispatch: + +jobs: + update-ui: + runs-on: ubuntu-latest + steps: + - name: Update UI and open PR + uses: SiaFoundation/workflows/.github/actions/ui-update@master + with: + moduleName: 'walletd' + goVersion: '1.21' + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ccf8ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +bin/ +walletd.yml +.DS_Store +.vscode/ \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..050db11 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,150 @@ +# Based off of the example file at https://github.com/golangci/golangci-lint + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 600s + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: [] + +# output configuration options +output: + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +# all available settings of specific linters +linters-settings: + ## Enabled linters: + govet: + # report about shadowed variables + disable-all: false + + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + + + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + enabled-tags: + - diagnostic + - style + disabled-checks: + # diagnostic + - appendAssign + - commentedOutCode + - uncheckedInlineErr + # style + - httpNoBody + - exitAfterDefer + - ifElseChain + - importShadow + - initClause + - nestingReduce + - octalLiteral + - paramTypeCombine + - ptrToRefParam + - stringsCompare + - tooManyResultsChecker + - typeDefFirst + - typeUnparen + - unlabelStmt + - unnamedResult + - whyNoLint + revive: + ignore-generated-header: true + rules: + - name: blank-imports + disabled: false + - name: bool-literal-in-expr + disabled: false + - name: confusing-naming + disabled: false + - name: confusing-results + disabled: false + - name: constant-logical-expr + disabled: false + - name: context-as-argument + disabled: false + - name: exported + disabled: false + - name: errorf + disabled: false + - name: if-return + disabled: false + - name: indent-error-flow + disabled: true + - name: increment-decrement + disabled: false + - name: modifies-value-receiver + disabled: true + - name: optimize-operands-order + disabled: false + - name: range-val-in-closure + disabled: false + - name: struct-tag + disabled: false + - name: superfluous-else + disabled: false + - name: time-equal + disabled: false + - name: unexported-naming + disabled: false + - name: unexported-return + disabled: false + - name: unnecessary-stmt + disabled: false + - name: unreachable-code + disabled: false + - name: package-comments + disabled: true + +linters: + disable-all: true + fast: false + enable: + - tagliatelle + - gocritic + - gofmt + - revive + - govet + - misspell + - typecheck + - whitespace + +issues: + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: [] + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1281049 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,350 @@ +## 2.0.0 (2025-02-21) + +### Breaking Changes + +#### Add Merkle Proof Basis to UTXO API Responses + +Changes the response to include the Merkle proof basis for the following endpoints: +- `[GET] /addresses/:address/outputs/siacoin` +- `[GET] /addresses/:address/outputs/siafund` +- `[GET] /wallets/:id/outputs/siacoin` +- `[GET] /wallets/:id/outputs/siafund` + + +```json +{ + "basis": { + "height": 1, + "id": "f362385eea61f81627f283a31af9faf6417fbb88d53b794639a34e18515996e9" + }, + "outputs": [ + { + "id": "ed556177482e70822a5dcad9343efb51998425884788415349bef8eba7e063ae", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "01048fc792904f156844a5524671304d3a020861da144afa4acc6553db63c1fd", + "33efdfaf9bb212842292ab6f298c454e1b3d412aa7beb7efdccdfccf09f5b4ee", + "102345919e408540d240460b0d84aa2f6da9a3d8f74765fd7c6daae6e46dd7f3" + ] + }, + "siacoinOutput": { + "value": "500000000000000000000000", + "address": "fbfc3d034b1eb45f63e0087571ec1f3028a9a2f8c180381d47713e6112467d91f474059476f2" + }, + "maturityHeight": 0 + } + ] +} +``` + +#### Simplified response of consensus updates endpoint + +The response of `/api/consensus/updates/:index` has been simplified to make it easier for developers to index chain state. + +```json +{ + "applied": [ + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "35b81e41f594d7faeb88bd8eaac2eaa68ce99fe1c8fe5f0cba8fafa65ab3a70e", + "stateElement": { + "leafIndex": 0, + "merkleProof": [ + "88052fa2d1e22e4a5542fed9686cdad3fbeccbc60d15d4fd36a7691d61add1e1" + ] + }, + "siacoinOutput": { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + }, + "maturityHeight": 0 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": [ + { + "siafundElement": { + "id": "69ad26a0fbd1a6985d2053246650bb3ba5f3491d818748b6c8562db1ddb2c45b", + "stateElement": { + "leafIndex": 1, + "merkleProof": [ + "837482a39d5bf66f07bae3b89191e4375b82c9f341ce6a17e22e14e0333ab9f6" + ] + }, + "siafundOutput": { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + }, + "claimStart": "0" + }, + "created": true, + "spent": false + } + ], + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "stateElement": { + "leafIndex": 2 + }, + "chainIndex": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + } + }, + "updatedLeaves": {}, + "treeGrowth": {}, + "oldNumLeaves": 0, + "numLeaves": 3 + }, + "state": { + "index": { + "height": 0, + "id": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a" + }, + "prevTimestamps": [ + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 0, + "oakTarget": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "1", + "difficulty": "4294967295", + "oakWork": "4294967297", + "elements": { + "numLeaves": 3, + "trees": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "0000000000000000000000000000000000000000000000000000000000000000", + "nonce": 0, + "timestamp": "2023-01-13T00:53:20-08:00", + "minerPayouts": [], + "transactions": [ + { + "id": "268ef8627241b3eb505cea69b21379c4b91c21dfc4b3f3f58c66316249058cfd", + "siacoinOutputs": [ + { + "value": "1000000000000000000000000000000000000", + "address": "3d7f707d05f2e0ec7ccc9220ed7c8af3bc560fbee84d068c2cc28151d617899e1ee8bc069946" + } + ], + "siafundOutputs": [ + { + "value": 10000, + "address": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807" + } + ] + } + ] + } + }, + { + "update": { + "siacoinElements": [ + { + "siacoinElement": { + "id": "ca02d6807c92f61af94e626604615fbcdb471f38fcd8f3add6c6e6e0485ce090", + "stateElement": { + "leafIndex": 3, + "merkleProof": [ + "e1c3af98d77463b767d973f8a563947d949d06428ff145db30143a2811d10014", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ] + }, + "siacoinOutput": { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + }, + "maturityHeight": 145 + }, + "created": true, + "spent": false + } + ], + "siafundElementDiffs": null, + "fileContractElementDiffs": null, + "v2FileContractElementDiffs": null, + "attestationElements": null, + "chainIndexElement": { + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006", + "stateElement": { + "leafIndex": 4 + }, + "chainIndex": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + } + }, + "updatedLeaves": {}, + "treeGrowth": { + "0": [ + "190d98a7d8ff464e57f89dc916b155455ecf927f4c74b9edf5e80c103f052bfa", + "134b1f08aec0c7fbc50203a514277d197947e3da3ab1854749bf093b56402912" + ], + "1": [ + "2b082bec52801c1e61e5b0d0c1f5fc3925bd24e16d2f490afeb70374828586f1" + ] + }, + "oldNumLeaves": 3, + "numLeaves": 5 + }, + "state": { + "index": { + "height": 1, + "id": "0000000028e731f0bb5d48662283bec83cca9427581b948d1036deb2b42c3006" + }, + "prevTimestamps": [ + "2023-01-13T08:18:19-08:00", + "2023-01-13T00:53:20-08:00", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z", + "0001-01-01T00:00:00Z" + ], + "depth": "00000000ffffffff00000000ffffffff00000000ffffffff00000000ffffffff", + "childTarget": "0000000100000000000000000000000000000000000000000000000000000000", + "siafundTaxRevenue": "0", + "oakTime": 26699000000000, + "oakTarget": "000000008052201448053c59f99803e7a8165929036cd574d91425423191387c", + "foundationSubsidyAddress": "053b2def3cbdd078c19d62ce2b4f0b1a3c5e0ffbeeff01280efb1f8969b2f5bb4fdc680f0807", + "foundationManagementAddress": "000000000000000000000000000000000000000000000000000000000000000089eb0d6a8a69", + "totalWork": "4294967297", + "difficulty": "4294967295", + "oakWork": "8568459756", + "elements": { + "numLeaves": 5, + "trees": [ + "589fb425faa23be357492394813dc575505899d42d0b23a7162e1c68f7eeb227", + "750cc671d80aef6ee5c73344ba4e74eccda77d9f0cf51ed6237952b1d84bc336" + ] + }, + "attestations": 0 + }, + "block": { + "parentID": "e23d2ee56fc5c79618ead2f8f36c1b72c6f3ec5e0f751c05e08bd6665a6ec22a", + "nonce": 10689346, + "timestamp": "2023-01-13T08:18:19-08:00", + "minerPayouts": [ + { + "value": "300000000000000000000000000000", + "address": "c5e1ca930f193cfe4c72eaed8d3bbae627f67d6c8e32c406fe692b1c00b554f4731fddf2c752" + } + ], + "transactions": [ + { + "id": "1148417ad8fa6546646da6922618358210bc7a668ef7cb25f6a8a3605851bc7b", + "arbitraryData": [ + "Tm9uU2lhAAAAAAAAAAAAAClvJjNhfcbxtEfP2yfbBM4=" + ] + } + ] + } + } + ], + "reverted": null +} +``` + +#### Support V2 Hardfork + +The V2 hardfork is scheduled to modernize Sia's consensus protocol, which has been untouched since Sia's mainnet launch back in 2014, and improve accessibility of the storage network. To ensure a smooth transition from V1, it will be executed in two phases. Additional documentation on upgrading will be released in the near future. + +##### V2 Highlights +- Drastically reduces blockchain size on disk +- Improves UTXO spend policies - including HTLC support for Atomic Swaps +- More efficient contract renewals - reducing lock up requirements for hosts and renters +- Improved transfer speeds - enables hot storage + +##### Phase 1 - Allow Height +- **Activation Height:** `52600` (June 6th, 2025) +- **New Features:** V2 transactions, contracts, and RHP4 +- **V1 Support:** Both V1 and V2 will be supported during this phase +- **Purpose:** This period gives time for integrators to transition from V1 to V2 +- **Requirements:** Users will need to update to support the hardfork before this block height + +##### Phase 2 - Require Height +- **Activation Height:** `530000` (July 6th, 2025) +- **New Features:** The consensus database can be trimmed to only store the Merkle proofs +- **V1 Support:** V1 will be disabled, including RHP2 and RHP3. Only V2 transactions will be accepted +- **Requirements:** Developers will need to update their apps to support V2 transactions and RHP4 before this block height + +#### Use standard locations for application data + +Uses standard locations for application data instead of the current directory. This brings `walletd` in line with other system services and makes it easier to manage application data. + +##### Linux, FreeBSD, OpenBSD +- Configuration: `/etc/walletd/walletd.yml` +- Data directory: `/var/lib/walletd` + +##### macOS +- Configuration: `~/Library/Application Support/walletd.yml` +- Data directory: `~/Library/Application Support/walletd` + +##### Windows +- Configuration: `%APPDATA%\SiaFoundation\walletd.yml` +- Data directory: `%APPDATA%\SiaFoundation\walletd` + +##### Docker +- Configuration: `/data/walletd.yml` +- Data directory: `/data` + +### Features + +- Add basis to wallet fund endpoints +- Log startup errors to stderr + +#### Add transaction construction API + +Adds two new endpoints to construct transactions. This combines and simplifies the existing fund flow for simple send transactions. + +See API docs for request and response bodies + +### Fixes + +- Added a test for migrations to ensure consistency between database schemas + +## 0.8.0 + +This is the first stable release for the walletd app -- the new reference wallet for users and exchanges + +### Breaking changes + +- SiaFund support +- Ledger hardware wallet support +- Multi-wallet support +- Full index mode for exchanges and wallet integrators +- Redesigned events list +- Redesigned transaction flow diff --git a/Dockerfile b/Dockerfile index 14294c5..3ad2c4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,27 @@ -FROM docker.io/library/golang:1.20 AS builder +FROM docker.io/library/golang:1.23 AS builder WORKDIR /walletd +# Install dependencies +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source COPY . . -# build -RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w' ./cmd/walletd -FROM docker.io/library/alpine:3 -LABEL maintainer="The Sia Foundation " \ - org.opencontainers.image.description.vendor="The Sia Foundation" \ - org.opencontainers.image.description="A walletd container - send and receive Siacoins and Siafunds" \ - org.opencontainers.image.source="https://github.com/SiaFoundation/walletd" \ - org.opencontainers.image.licenses=MIT +# Enable CGO for sqlite3 support +ENV CGO_ENABLED=1 -ENV PUID=0 -ENV PGID=0 +RUN go generate ./... +RUN go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/walletd + +FROM debian:bookworm-slim +LABEL maintainer="The Sia Foundation " \ + org.opencontainers.image.description.vendor="The Sia Foundation" \ + org.opencontainers.image.description="A walletd container - send and receive Siacoins and Siafunds" \ + org.opencontainers.image.source="https://github.com/SiaFoundation/walletd" \ + org.opencontainers.image.licenses=MIT -ENV WALLETD_API_PASSWORD= # copy binary and prepare data dir. COPY --from=builder /walletd/bin/* /usr/bin/ @@ -27,6 +32,7 @@ EXPOSE 9980/tcp # RPC port EXPOSE 9981/tcp -USER ${PUID}:${PGID} +ENV WALLETD_DATA_DIR=/data +ENV WALLETD_CONFIG_FILE=/data/walletd.yml -ENTRYPOINT [ "walletd", "--dir", "/data", "--http", ":9980" ] \ No newline at end of file +ENTRYPOINT [ "walletd", "--http", ":9980" ] diff --git a/README.md b/README.md index 91e1bc5..8f3052b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![GoDoc](https://godoc.org/go.sia.tech/walletd?status.svg)](https://godoc.org/go.sia.tech/walletd) +## Overview + `walletd` is the flagship Sia wallet, suitable for miners, exchanges, and everyday hodlers. Its client-server architecture gives you the flexibility to access your funds from anywhere, on any device, without compromising the @@ -11,3 +13,174 @@ Ledger hardware wallet, or another preferred method. Like other Foundation node software, `walletd` ships with a slick embedded UI, but developers can easily build headless integrations leveraging its powerful JSON API. Whether you're using a single address or millions, `walletd` scales to your needs. + +Setup guides are available at https://docs.sia.tech + +### Index Mode +`walletd` supports three different index modes for different use cases. + +**Personal** + +In "personal" index mode, `walletd` will only index addresses that are registered in the +wallet. This mode is recommended for most users, as it provides a good balance between +comprehensiveness and resource usage for personal wallets. This is the default +mode for `walletd`. + +When adding addresses with existing history on chain, users will need to manually +initiate a rescan to index the new transactions. This can take some to complete, +depending on the number of blocks that need to be scanned. When adding addresses +with no existing history, a rescan is not necessary. + +**Full** + +In "full" index mode, `walletd` will index the entire blockchain including all addresses +and UTXOs. This is the most comprehensive mode, but it also requires the most +resources. This mode is recommended for exchanges or wallet builders that need +to support a large or unknown number of addresses. + +**None** + +In "none" index mode, `walletd` will treat the database as read-only and not +index any new data. This mode is only useful in situations where another process +is managing the database and `walletd` is only being used to read data. + +## Configuration + +`walletd` can be configured in multiple ways. Some settings, like the API password, +can be configured via environment variable. Others, like the API port, and data +directory, can be set via command line flags. To simplify more complex configurations, +`walletd` can also be configured via a YAML file. + +The priority of configuration settings is as follows: +1. Command line flags +2. YAML file +3. Environment variables + +### Default Ports ++ `9980` UI and API ++ `9981` Sia consensus + +### Environment Variables ++ `WALLETD_API_PASSWORD` - The password required to access the API. ++ `WALLETD_CONFIG_FILE` - The path to the YAML configuration file. Defaults to `walletd.yml` in the working directory. ++ `WALLETD_LOG_FILE` - The path to the log file. ++ `WALLETD_KEYSTORE_SECRET` - The secret to use for encrypting stored ed25519 signing keys. + +### Command Line Flags +``` +Usage: + walletd [flags] [action] + +Run 'walletd' with no arguments to start the blockchain node and API server. + +Actions: + version print walletd version + seed generate a recovery phrase + mine run CPU miner +Flags: + -addr string + p2p address to listen on (default ":9981") + -bootstrap + attempt to bootstrap the network (default true) + -debug + enable debug mode with additional profiling and mining endpoints + -dir string + directory to store node state in (default "/Users/username/Library/Application Support/walletd") + -http string + address to serve API on (default "localhost:9980") + -http.public + disables auth on endpoints that should be publicly accessible when running walletd as a service + -index.batch int + max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage. (default 1000) + -index.mode string + address index mode (personal, full, none) (default "full") + -network string + network to connect to (default "mainnet") + -upnp + attempt to forward ports and discover IP with UPnP + -keystore + enables the optional ed25519 key store. +``` + +### YAML +All configuration settings can be set in a YAML file. The default location of that file is +- `/etc/walletd/walletd.yml` on Linux +- `~/Library/Application Support/walletd/walletd.yml` on macOS +- `%APPDATA%\SiaFoundation\walletd.yml` on Windows +- `/data/walletd.yml` in the Docker container + +It can be generated using the `walletd config` command. Alternatively a local +configuration can be created manually by creating a file name `walletd.yml` in +the working directory. All fields are optional. +```yaml +directory: /etc/walletd +autoOpenWebUI: true +http: + address: :9980 + password: sia is cool + publicEndpoints: false # when true, auth will be disabled on endpoints that should be publicly accessible when running walletd as a service +consensus: + network: mainnet +syncer: + bootstrap: false + enableUPnP: false + peers: [] + address: :9981 +keystore: + enabled: false +index: + mode: personal # personal, full, none ("full" will index the entire blockchain, "personal" will only index addresses that are registered in the wallet, "none" will treat the database as read-only and not index any new data) + batchSize: 64 # max number of blocks to index at a time (increasing this will increase scan speed, but also increase memory and cpu usage) +log: + level: info # global log level + stdout: + enabled: true # enable logging to stdout + level: debug # override the global log level for stdout + enableANSI: false + format: human # human or JSON + file: + enabled: true # enable logging to a file + level: debug # override the global log level for the file + path: /var/log/walletd.log + format: json # human or JSON +``` + +## Building +`walletd` uses SQLite for its persistence. A gcc toolchain is required to build `walletd` + +```sh +go generate ./... +CGO_ENABLED=1 go build -o bin/ -tags='netgo timetzdata' -trimpath -a -ldflags '-s -w' ./cmd/walletd +``` + +## Docker Image +`walletd` includes a Dockerfile for building a Docker image. For building and +running `walletd` within a Docker container. The image can also be pulled from `ghcr.io/siafoundation/walletd`. + +```sh +docker run -d \ + --name walletd \ + -p 127.0.0.1:9980:9980 \ + -p 9981:9981 \ + -v /data:/data \ + ghcr.io/siafoundation/walletd:latest +``` + +### Docker Compose +```yml +services: + walletd: + image: ghcr.io/siafoundation/walletd:latest + ports: + - 127.0.0.1:9980:9980/tcp + - 9981:9981/tcp + volumes: + - /data:/data + restart: unless-stopped +``` + +### Building + +```sh +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/siafoundation/walletd:master . +``` \ No newline at end of file diff --git a/api/api.go b/api/api.go index d894469..645a04a 100644 --- a/api/api.go +++ b/api/api.go @@ -1,68 +1,94 @@ package api import ( + "encoding/json" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" ) +// A StateResponse returns information about the current state of the walletd +// daemon. +type StateResponse struct { + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + BuildTime time.Time `json:"buildTime"` + StartTime time.Time `json:"startTime"` + IndexMode wallet.IndexMode `json:"indexMode"` +} + // A GatewayPeer is a currently-connected peer. type GatewayPeer struct { - Addr string `json:"addr"` + Address string `json:"address"` Inbound bool `json:"inbound"` Version string `json:"version"` - FirstSeen time.Time `json:"firstSeen"` - ConnectedSince time.Time `json:"connectedSince"` - SyncedBlocks uint64 `json:"syncedBlocks"` - SyncDuration time.Duration `json:"syncDuration"` + FirstSeen time.Time `json:"firstSeen,omitempty"` + ConnectedSince time.Time `json:"connectedSince,omitempty"` + SyncedBlocks uint64 `json:"syncedBlocks,omitempty"` + SyncDuration time.Duration `json:"syncDuration,omitempty"` } // TxpoolBroadcastRequest is the request type for /txpool/broadcast. type TxpoolBroadcastRequest struct { + Basis types.ChainIndex `json:"basis"` Transactions []types.Transaction `json:"transactions"` V2Transactions []types.V2Transaction `json:"v2transactions"` } // TxpoolTransactionsResponse is the response type for /txpool/transactions. type TxpoolTransactionsResponse struct { + Basis types.ChainIndex `json:"basis"` Transactions []types.Transaction `json:"transactions"` V2Transactions []types.V2Transaction `json:"v2transactions"` } -// WalletBalanceResponse is the response type for /wallets/:name/balance. -type WalletBalanceResponse struct { - Siacoins types.Currency `json:"siacoins"` - Siafunds uint64 `json:"siafunds"` +// TxpoolUpdateV2TransactionsRequest is the request type for /txpool/transactions/v2/basis. +type TxpoolUpdateV2TransactionsRequest struct { + Basis types.ChainIndex `json:"basis"` + Target types.ChainIndex `json:"target"` + Transactions []types.V2Transaction `json:"transactions"` } -// WalletOutputsResponse is the response type for /wallets/:name/outputs. -type WalletOutputsResponse struct { - SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` - SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` +// TxpoolUpdateV2TransactionsResponse is the response type for /txpool/transactions/v2/basis. +type TxpoolUpdateV2TransactionsResponse struct { + Basis types.ChainIndex `json:"basis"` + Transactions []types.V2Transaction `json:"transactions"` } -// WalletReserveRequest is the request type for /wallets/:name/reserve. +// BalanceResponse is the response type for /wallets/:id/balance. +type BalanceResponse wallet.Balance + +// WalletReserveRequest is the request type for /wallets/:id/reserve. type WalletReserveRequest struct { SiacoinOutputs []types.SiacoinOutputID `json:"siacoinOutputs"` SiafundOutputs []types.SiafundOutputID `json:"siafundOutputs"` - Duration time.Duration `json:"duration"` } -// WalletReleaseRequest is the request type for /wallets/:name/release. +// A WalletUpdateRequest is a request to update a wallet +type WalletUpdateRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Metadata json.RawMessage `json:"metadata"` +} + +// WalletReleaseRequest is the request type for /wallets/:id/release. type WalletReleaseRequest struct { SiacoinOutputs []types.SiacoinOutputID `json:"siacoinOutputs"` SiafundOutputs []types.SiafundOutputID `json:"siafundOutputs"` } -// WalletFundRequest is the request type for /wallets/:name/fund. +// WalletFundRequest is the request type for /wallets/:id/fund. type WalletFundRequest struct { Transaction types.Transaction `json:"transaction"` Amount types.Currency `json:"amount"` ChangeAddress types.Address `json:"changeAddress"` } -// WalletFundSFRequest is the request type for /wallets/:name/fundsf. +// WalletFundSFRequest is the request type for /wallets/:id/fundsf. type WalletFundSFRequest struct { Transaction types.Transaction `json:"transaction"` Amount uint64 `json:"amount"` @@ -70,16 +96,168 @@ type WalletFundSFRequest struct { ClaimAddress types.Address `json:"claimAddress"` } -// WalletFundResponse is the response type for /wallets/:name/fund. +// WalletFundResponse is the response type for /wallets/:id/fund. type WalletFundResponse struct { + Basis types.ChainIndex `json:"basis"` Transaction types.Transaction `json:"transaction"` ToSign []types.Hash256 `json:"toSign"` DependsOn []types.Transaction `json:"dependsOn"` } +// WalletConstructRequest is the request type for /wallets/:id/construct. +type WalletConstructRequest struct { + Siacoins []types.SiacoinOutput `json:"siacoins"` + Siafunds []types.SiafundOutput `json:"siafunds"` + ChangeAddress types.Address `json:"changeAddress"` +} + +// SignaturePayload is a signature that is required to finalize a transaction. +type SignaturePayload struct { + PublicKey types.PublicKey `json:"publicKey"` + SigHash types.Hash256 `json:"sigHash"` +} + +// WalletConstructResponse is the response type for /wallets/:id/construct/transaction. +type WalletConstructResponse struct { + Basis types.ChainIndex `json:"basis"` + ID types.TransactionID `json:"id"` + Transaction types.Transaction `json:"transaction"` + EstimatedFee types.Currency `json:"estimatedFee"` +} + +// WalletConstructV2Response is the response type for /wallets/:id/construct/v2/transaction. +type WalletConstructV2Response struct { + Basis types.ChainIndex `json:"basis"` + ID types.TransactionID `json:"id"` + Transaction types.V2Transaction `json:"transaction"` + EstimatedFee types.Currency `json:"estimatedFee"` +} + // SeedSignRequest requests that a transaction be signed using the keys derived // from the given indices. type SeedSignRequest struct { Transaction types.Transaction `json:"transaction"` Keys []uint64 `json:"keys"` } + +// RescanResponse contains information about the state of a chain rescan. +type RescanResponse struct { + StartIndex types.ChainIndex `json:"startIndex"` + Index types.ChainIndex `json:"index"` + StartTime time.Time `json:"startTime"` + Error *string `json:"error,omitempty"` +} + +// An ApplyUpdate is a consensus update that was applied to the best chain. +type ApplyUpdate struct { + Update consensus.ApplyUpdate `json:"update"` + State consensus.State `json:"state"` + Block types.Block `json:"block"` +} + +// A RevertUpdate is a consensus update that was reverted from the best chain. +type RevertUpdate struct { + Update consensus.RevertUpdate `json:"update"` + State consensus.State `json:"state"` + Block types.Block `json:"block"` +} + +// ConsensusUpdatesResponse is the response type for /consensus/updates/:index. +type ConsensusUpdatesResponse struct { + Applied []ApplyUpdate `json:"applied"` + Reverted []RevertUpdate `json:"reverted"` +} + +// DebugMineRequest is the request type for /debug/mine. +type DebugMineRequest struct { + Blocks int `json:"blocks"` + Address types.Address `json:"address"` +} + +// SiacoinElementsResponse is the response type for any endpoint that returns +// siacoin UTXOs +type SiacoinElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []types.SiacoinElement `json:"outputs"` +} + +// SiafundElementsResponse is the response type for any endpoint that returns +// siafund UTXOs +type SiafundElementsResponse struct { + Basis types.ChainIndex `json:"basis"` + Outputs []types.SiafundElement `json:"outputs"` +} + +// ElementSpentResponse is the response type for /outputs/siacoin/:id/spent and +// /outputs/siafund/:id/spent. +type ElementSpentResponse struct { + Spent bool `json:"spent"` + Event *wallet.Event `json:"event,omitempty"` +} + +// MiningGetBlockTemplateRequest is the request type for +// /mining/getblocktemplate. +type MiningGetBlockTemplateRequest struct { + PayoutAddress types.Address `json:"payoutAddress,omitempty"` + LongPollID string `json:"longpollid,omitempty"` +} + +// MiningGetBlockTemplateResponse is the response type for +// /mining/getblocktemplate. +type MiningGetBlockTemplateResponse struct { + Transactions []MiningGetBlockTemplateResponseTxn `json:"transactions"` + MinerPayout []MiningGetBlockTemplateResponseTxn `json:"minerpayout"` + PreviousBlockHash string `json:"previousblockhash"` + + // Optional long polling from BIP 0022. + LongPollID string `json:"longpollid"` + + // Basic pool extension from BIP 0023. + Target string `json:"target"` + Height uint32 `json:"height"` + + // Mutations from BIP 0023. + Timestamp int32 `json:"curtime"` + + // Block proposal from BIP 0023. + Version uint32 `json:"version"` + Bits string `json:"bits"` +} + +// MiningGetBlockTemplateResponseTxn is a transaction in a block template. +type MiningGetBlockTemplateResponseTxn struct { + Data string `json:"data"` + Hash string `json:"hash"` + TxID string `json:"txid"` + Depends []int64 `json:"depends"` + Fee int64 `json:"fee"` + SigOps int64 `json:"sigops"` + TxType string `json:"txtype"` +} + +// MiningSubmitBlockRequest is the request type for /mining/submitblock. +type MiningSubmitBlockRequest struct { + // should contain only the hex-encoded block + Params []string `json:"params"` +} + +// An AddSigningKeyRequest is a request to add an ed25519 signing key to the +// key store. +type AddSigningKeyRequest struct { + PrivateKey types.PrivateKey `json:"privateKey"` +} + +// An AddSigningKeyResponse is the response to an AddSigningKeyRequest. +type AddSigningKeyResponse struct { + PublicKey types.PublicKey `json:"publicKey"` +} + +// A SignHashRequest is a request to sign a hash with a key. +type SignHashRequest struct { + Hash types.Hash256 `json:"hash"` +} + +// A SignHashResponse is the response to a SignHashRequest. +type SignHashResponse struct { + Signature types.Signature `json:"signature"` +} diff --git a/api/api_test.go b/api/api_test.go index 335807a..061e67e 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,83 +1,247 @@ package api_test import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" "net" "net/http" + "reflect" + "strings" "testing" "time" - "go.sia.tech/core/chain" "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils" "go.sia.tech/jape" "go.sia.tech/walletd/api" - "go.sia.tech/walletd/internal/syncerutil" - "go.sia.tech/walletd/internal/walletutil" - "go.sia.tech/walletd/syncer" + "go.sia.tech/walletd/internal/testutil" + "go.sia.tech/walletd/keys" "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" "lukechampine.com/frand" ) -func testNetwork() (*consensus.Network, types.Block) { - // use a modified version of Zen - n, genesisBlock := chain.TestnetZen() - n.InitialTarget = types.BlockID{0xFF} - n.HardforkDevAddr.Height = 1 - n.HardforkTax.Height = 1 - n.HardforkStorageProof.Height = 1 - n.HardforkOak.Height = 1 - n.HardforkASIC.Height = 1 - n.HardforkFoundation.Height = 1 - n.HardforkV2.AllowHeight = 5 - n.HardforkV2.RequireHeight = 10 - return n, genesisBlock -} +func startWalletServer(tb testing.TB, cn *testutil.ConsensusNode, log *zap.Logger, walletOpts ...wallet.Option) *api.Client { + tb.Helper() -func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Client, func()) { l, err := net.Listen("tcp", ":0") if err != nil { - panic(err) + tb.Fatal("failed to listen:", err) + } + tb.Cleanup(func() { l.Close() }) + + wm, err := wallet.NewManager(cn.Chain, cn.Store, append([]wallet.Option{wallet.WithLogger(log.Named("wallet"))}, walletOpts...)...) + if err != nil { + tb.Fatal("failed to create wallet manager:", err) + } + tb.Cleanup(func() { wm.Close() }) + + km, err := keys.NewManager(cn.Store, "foo") + if err != nil { + tb.Fatal("failed to create key manager:", err) + } + tb.Cleanup(func() { km.Close() }) + + server := &http.Server{ + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithKeyManager(km), api.WithDebug(), api.WithLogger(log)), + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, } - go func() { - srv := api.NewServer(cm, s, wm) - http.Serve(l, jape.BasicAuth("password")(srv)) - }() - c := api.NewClient("http://"+l.Addr().String(), "password") - return c, func() { l.Close() } + tb.Cleanup(func() { server.Close() }) + + go server.Serve(l) + return api.NewClient("http://"+l.Addr().String(), "password") } -func TestWallet(t *testing.T) { - n, genesisBlock := testNetwork() +func TestWalletAdd(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V1Network() giftPrivateKey := types.GeneratePrivateKey() giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ Value: types.Siacoins(1), Address: giftAddress, } + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + checkWalletResponse := func(wr api.WalletUpdateRequest, w wallet.Wallet, isUpdate bool) error { + // check wallet + if w.Name != wr.Name { + return fmt.Errorf("expected wallet name to be %v, got %v", wr.Name, w.Name) + } else if w.Description != wr.Description { + return fmt.Errorf("expected wallet description to be %v, got %v", wr.Description, w.Description) + } else if w.DateCreated.After(time.Now()) { + return fmt.Errorf("expected wallet creation date to be in the past, got %v", w.DateCreated) + } else if isUpdate && w.DateCreated == w.LastUpdated { + return fmt.Errorf("expected wallet last updated date to be after creation %v, got %v", w.DateCreated, w.LastUpdated) + } + + if wr.Metadata == nil && string(w.Metadata) == "null" { // zero value encodes as "null" + return nil + } + + // check metadata + var am, bm map[string]any + if err := json.Unmarshal(wr.Metadata, &am); err != nil { + return fmt.Errorf("failed to unmarshal metadata a %q: %v", wr.Metadata, err) + } else if err := json.Unmarshal(w.Metadata, &bm); err != nil { + return fmt.Errorf("failed to unmarshal metadata b: %v", err) + } + + if !reflect.DeepEqual(am, bm) { // not perfect, but probably enough for this test + return fmt.Errorf("expected metadata to be equal %v, got %v", wr.Metadata, w.Metadata) + } + return nil + } + + checkWallet := func(wa, wb wallet.Wallet) error { + // check wallet + if wa.Name != wb.Name { + return fmt.Errorf("expected wallet name to be %v, got %v", wa.Name, wb.Name) + } else if wa.Description != wb.Description { + return fmt.Errorf("expected wallet description to be %v, got %v", wa.Description, wb.Description) + } else if wa.DateCreated.Unix() != wb.DateCreated.Unix() { + return fmt.Errorf("expected wallet creation date to be %v, got %v", wa.DateCreated, wb.DateCreated) + } else if wa.LastUpdated.Unix() != wb.LastUpdated.Unix() { + return fmt.Errorf("expected wallet last updated date to be %v, got %v", wa.LastUpdated, wb.LastUpdated) + } + + if wa.Metadata == nil && string(wb.Metadata) == "null" { // zero value encodes as "null" + return nil + } + + // check metadata + var am, bm map[string]any + if err := json.Unmarshal(wa.Metadata, &am); err != nil { + return fmt.Errorf("failed to unmarshal metadata a %q: %v", wa.Metadata, err) + } else if err := json.Unmarshal(wb.Metadata, &bm); err != nil { + return fmt.Errorf("failed to unmarshal metadata b %q: %v", wb.Metadata, err) + } + + if !reflect.DeepEqual(am, bm) { // not perfect, but probably enough for this test + return fmt.Errorf("expected metadata to be equal %v, got %v", wa.Metadata, wb.Metadata) + } + return nil + } + + tests := []struct { + Initial api.WalletUpdateRequest + Update api.WalletUpdateRequest + }{ + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12))}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12))}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "hello, world!"}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "goodbye, world!"}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + { + Initial: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "hello, world!", Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: hex.EncodeToString(frand.Bytes(12)), Description: "goodbye, world!", Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + { + Initial: api.WalletUpdateRequest{Name: "constant name", Description: "constant description", Metadata: []byte(`{"foo": { "foo": "bar"}}`)}, + Update: api.WalletUpdateRequest{Name: "constant name", Description: "constant description", Metadata: []byte(`{"foo": { "foo": "baz"}}`)}, + }, + } + + var expectedWallets []wallet.Wallet + for i, test := range tests { + w, err := c.AddWallet(test.Initial) + if err != nil { + t.Fatal(err) + } else if err := checkWalletResponse(test.Initial, w, false); err != nil { + t.Fatalf("test %v: %v", i, err) + } + + expectedWallets = append(expectedWallets, w) + // check that the wallet was added + wallets, err := c.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != len(expectedWallets) { + t.Fatalf("test %v: expected %v wallets, got %v", i, len(expectedWallets), len(wallets)) + } + for j, w := range wallets { + if err := checkWallet(expectedWallets[j], w); err != nil { + t.Fatalf("test %v: wallet %v: %v", i, j, err) + } + } + + time.Sleep(time.Second) // ensure LastUpdated is different + + w, err = c.UpdateWallet(w.ID, test.Update) + if err != nil { + t.Fatal(err) + } else if err := checkWalletResponse(test.Update, w, true); err != nil { + t.Fatalf("test %v: %v", i, err) + } + + // check that the wallet was updated + expectedWallets[len(expectedWallets)-1] = w + wallets, err = c.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != len(expectedWallets) { + t.Fatalf("test %v: expected %v wallets, got %v", i, len(expectedWallets), len(wallets)) + } + for j, w := range wallets { + if err := checkWallet(expectedWallets[j], w); err != nil { + t.Fatalf("test %v: wallet %v: %v", i, j, err) + } + } + } +} + +func TestWallet(t *testing.T) { + log := zaptest.NewLogger(t) - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + // create syncer + syncerListener, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) } - cm := chain.NewManager(dbstore, tipState) - wm := walletutil.NewEphemeralWalletManager(cm) - sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) - c, shutdown := runServer(cm, nil, wm) - defer shutdown() - if err := c.AddWallet("primary", nil); err != nil { + defer syncerListener.Close() + + // create chain manager + n, genesisBlock := testutil.V1Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { t.Fatal(err) + } else if w.Name != "primary" { + t.Fatalf("expected wallet name to be 'primary', got %v", w.Name) } - wc := c.Wallet("primary") - if err := wc.Subscribe(0); err != nil { + wc := c.Wallet(w.ID) + if err := c.Rescan(0); err != nil { t.Fatal(err) } + cn.WaitForSync(t) balance, err := wc.Balance() if err != nil { t.Fatal(err) - } else if !balance.Siacoins.IsZero() || balance.Siafunds != 0 { + } else if !balance.Siacoins.IsZero() || !balance.ImmatureSiacoins.IsZero() || balance.Siafunds != 0 { t.Fatal("balance should be 0") } @@ -98,8 +262,12 @@ func TestWallet(t *testing.T) { } // create and add an address - addr, info := sav.NewAddress("primary") - if err := wc.AddAddress(addr, info); err != nil { + sk2 := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(sk2.PublicKey()) + err = wc.AddAddress(wallet.Address{ + Address: addr, + }) + if err != nil { t.Fatal(err) } @@ -107,8 +275,10 @@ func TestWallet(t *testing.T) { addresses, err = wc.Addresses() if err != nil { t.Fatal(err) - } else if _, ok := addresses[addr]; !ok || len(addresses) != 1 { - t.Fatal("bad address list", addresses) + } else if len(addresses) != 1 { + t.Fatal("address list should have one address") + } else if addresses[0].Address != addr { + t.Fatalf("address should be %v, got %v", addr, addresses[0]) } // send gift to wallet @@ -127,530 +297,1440 @@ func TestWallet(t *testing.T) { CoveredFields: types.CoveredFields{WholeTransaction: true}, }}, } - sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) txn.Signatures[0].Signature = sig[:] - cs := cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: []types.Transaction{txn}, + // broadcast the transaction to the transaction pool + if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { + t.Fatal(err) } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() + + // shouldn't have any events yet + events, err = wc.Events(0, -1) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatal("event history should be empty") } - if err := cm.AddBlocks([]types.Block{b}); err != nil { + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatal("txpool should have one transaction") } + // confirm the transaction + cn.MineBlocks(t, types.VoidAddress, 1) // get new balance balance, err = wc.Balance() if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Error("balance should be 1 SC, got", balance.Siacoins) + t.Fatal("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } // transaction should appear in history - events, err = wc.Events(0, -1) + events, err = wc.Events(0, 100) if err != nil { t.Fatal(err) } else if len(events) == 0 { - t.Error("transaction should appear in history") + t.Fatal("transaction should appear in history") } - outputs, _, err := wc.Outputs() + outputs, basis, err := wc.SiacoinOutputs(0, 100) if err != nil { t.Fatal(err) } else if len(outputs) != 2 { - t.Error("should have two UTXOs, got", len(outputs)) + t.Fatal("should have two UTXOs, got", len(outputs)) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) } -} -func TestV2(t *testing.T) { - n, genesisBlock := testNetwork() - // gift primary wallet some coins - primaryPrivateKey := types.GeneratePrivateKey() - primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress - // secondary wallet starts with nothing - secondaryPrivateKey := types.GeneratePrivateKey() - secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) + // mine a block to add an immature balance + expectedPayout := cn.Chain.TipState().BlockReward() + cn.MineBlocks(t, addr, 1) - // create wallets - dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + // get new balance + balance, err = wc.Balance() if err != nil { t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Fatal("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("immature balance should be %d SC, got %d SC", expectedPayout, balance.ImmatureSiacoins) } - cm := chain.NewManager(dbstore, tipState) - wm := walletutil.NewEphemeralWalletManager(cm) - c, shutdown := runServer(cm, nil, wm) - defer shutdown() - if err := c.AddWallet("primary", nil); err != nil { + + // mine enough blocks for the miner payout to mature + expectedBalance := types.Siacoins(1).Add(expectedPayout) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + + // get new balance + balance, err = wc.Balance() + if err != nil { t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedBalance) { + t.Fatalf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } - primary := c.Wallet("primary") - if err := primary.AddAddress(primaryAddress, nil); err != nil { - t.Fatal(err) +} + +func TestAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V1Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, } - if err := primary.Subscribe(0); err != nil { + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + sk2 := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(sk2.PublicKey()) + + // personal index mode requires a wallet for indexing + w, err := c.AddWallet(api.WalletUpdateRequest{Name: "primary"}) + if err != nil { t.Fatal(err) } - if err := c.AddWallet("secondary", nil); err != nil { + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{Address: addr}) + if err != nil { t.Fatal(err) } - secondary := c.Wallet("secondary") - if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + + // send gift to wallet + giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: giftSCOID, + UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(1).Div64(2)}, + {Address: addr, Value: types.Siacoins(1).Div64(2)}, + }, + Signatures: []types.TransactionSignature{{ + ParentID: types.Hash256(giftSCOID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + + cs, err := c.ConsensusTipState() + if err != nil { t.Fatal(err) } - if err := secondary.Subscribe(0); err != nil { + + sig := giftPrivateKey.SignHash(cs.WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction to the transaction pool + if err := c.TxpoolBroadcast(cs.Index, []types.Transaction{txn}, nil); err != nil { t.Fatal(err) } + cn.MineBlocks(t, types.VoidAddress, 1) - // define some helper functions - addBlock := func(txns []types.Transaction, v2txns []types.V2Transaction) error { - cs := cm.TipState() - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: txns, - } - if v2txns != nil { - b.V2 = &types.V2BlockData{ - Height: cs.Index.Height + 1, - Transactions: v2txns, - } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - return cm.AddBlocks([]types.Block{b}) + // get new balance + balance, err := c.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Fatal("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } - checkBalances := func(p, s types.Currency) { - t.Helper() - if primaryBalance, err := primary.Balance(); err != nil { - t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) - } - if secondaryBalance, err := secondary.Balance(); err != nil { - t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) - } + + // transaction should appear in history + events, err := c.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) == 0 { + t.Fatal("transaction should appear in history") } - sendV1 := func() error { - t.Helper() - // which wallet is sending? - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() - if err != nil { - t.Fatal(err) - } - if len(sces) == 0 { - sces, _, err = secondary.Outputs() - if err != nil { - t.Fatal(err) - } - key = secondaryPrivateKey - dest = primaryAddress - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - - txn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - Signatures: []types.TransactionSignature{{ - ParentID: sce.ID, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }}, - } - sig := key.SignHash(cm.TipState().WholeSigHash(txn, sce.ID, 0, 0, nil)) - txn.Signatures[0].Signature = sig[:] - if err := addBlock([]types.Transaction{txn}, nil); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil + outputs, basis, err := c.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(outputs) != 2 { + t.Fatal("should have two UTXOs, got", len(outputs)) + } else if basis != cn.Chain.Tip() { + t.Fatalf("basis should be %v, got %v", cn.Chain.Tip(), basis) } - sendV2 := func() error { - t.Helper() - // which wallet is sending? - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() - if err != nil { - t.Fatal(err) - } - if len(sces) == 0 { - sces, _, err = secondary.Outputs() - if err != nil { - t.Fatal(err) - } - key = secondaryPrivateKey - dest = primaryAddress - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] + // mine a block to add an immature balance + expectedPayout := cn.Chain.TipState().BlockReward() + cn.MineBlocks(t, addr, 1) - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cm.TipState().InputSigHash(txn))} - if err := addBlock(nil, []types.V2Transaction{txn}); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil + // get new balance + balance, err = c.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(1)) { + t.Fatal("balance should be 1 SC, got", balance.Siacoins) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("immature balance should be %d SC, got %d SC", expectedPayout, balance.ImmatureSiacoins) } - // attempt to send primary->secondary with a v2 txn; should fail - if err := sendV2(); err == nil { - t.Fatal("expected v2 txn to be rejected") - } - // use a v1 transaction instead - if err := sendV1(); err != nil { + // mine enough blocks for the miner payout to mature + expectedBalance := types.Siacoins(1).Add(expectedPayout) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + + // get new balance + balance, err = c.AddressBalance(addr) + if err != nil { t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedBalance) { + t.Fatalf("balance should be %d, got %d", expectedBalance, balance.Siacoins) + } else if !balance.ImmatureSiacoins.IsZero() { + t.Fatal("immature balance should be 0 SC, got", balance.ImmatureSiacoins) } +} - // mine past v2 allow height - for cm.Tip().Height <= n.HardforkV2.AllowHeight { - if err := addBlock(nil, nil); err != nil { - t.Fatal(err) - } +func TestConsensus(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, } - // now send coins back with a v2 transaction - if err := sendV2(); err != nil { + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + // mine a block + minedBlock, ok := coreutils.MineBlock(cn.Chain, types.Address{}, time.Minute) + if !ok { + t.Fatal("no block found") + } else if err := cn.Chain.AddBlocks([]types.Block{minedBlock}); err != nil { t.Fatal(err) } - // v1 transactions should also still work - if err := sendV1(); err != nil { + + // block should be tip now + ci, err := c.ConsensusTip() + if err != nil { t.Fatal(err) + } else if ci.ID != minedBlock.ID() { + t.Fatalf("expected consensus tip to be %v, got %v", minedBlock.ID(), ci.ID) } - // mine past v2 require height - for cm.Tip().Height <= n.HardforkV2.RequireHeight { - if err := addBlock(nil, nil); err != nil { - t.Fatal(err) - } + // fetch block + b, err := c.ConsensusBlocksID(minedBlock.ID()) + if err != nil { + t.Fatal(err) + } else if b.ID() != minedBlock.ID() { + t.Fatal("mismatch") } - // v1 transactions should no longer work - if err := sendV1(); err == nil { - t.Fatal("expected v1 txn to be rejected") +} + +func TestConsensusUpdates(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V1Network() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, } - // use a v2 transaction instead - if err := sendV2(); err != nil { + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + cn.MineBlocks(t, types.VoidAddress, 10) + + reverted, applied, err := c.ConsensusUpdates(types.ChainIndex{}, 10) + if err != nil { t.Fatal(err) + } else if len(reverted) != 0 { + t.Fatal("expected no reverted blocks") + } else if len(applied) != 11 { // genesis + 10 mined blocks (chain manager off-by-one) + t.Fatalf("expected 11 applied blocks, got %v", len(applied)) + } + + for i, cau := range applied { + // using i for height since we're testing the update contents + expected, ok := cn.Chain.BestIndex(uint64(i)) + if !ok { + t.Fatalf("failed to get expected index for block %v", i) + } else if cau.State.Index != expected { + t.Fatalf("expected index %v, got %v", expected, cau.State.Index) + } else if cau.State.Network.Name != n.Name { // TODO: better comparison. reflect.DeepEqual is failing in CI, but passing local. + t.Fatalf("expected network to be %q, got %q", n.Name, cau.State.Network.Name) + } } } -func TestP2P(t *testing.T) { - n, genesisBlock := testNetwork() - // gift primary wallet some coins - primaryPrivateKey := types.GeneratePrivateKey() - primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) - genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress - // secondary wallet starts with nothing - secondaryPrivateKey := types.GeneratePrivateKey() - secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) +func TestConstructSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V1Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() - // create wallets - dbstore1, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) if err != nil { t.Fatal(err) } - cm1 := chain.NewManager(dbstore1, tipState) - wm1 := walletutil.NewEphemeralWalletManager(cm1) - l1, err := net.Listen("tcp", ":0") + + wc := c.Wallet(w.ID) + // add an address with no spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + }) if err != nil { t.Fatal(err) } - defer l1.Close() - s1 := syncer.New(l1, cm1, syncerutil.NewEphemeralPeerStore(), gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l1.Addr().String(), - }) - go s1.Run() - c1, shutdown := runServer(cm1, s1, wm1) - defer shutdown() - if err := c1.AddWallet("primary", nil); err != nil { + + if err := c.Rescan(0); err != nil { t.Fatal(err) } - primary := c1.Wallet("primary") - if err := primary.AddAddress(primaryAddress, nil); err != nil { + cn.MineBlocks(t, types.VoidAddress, 1) + + // try to construct a valid transaction with no spend policy + _, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if !strings.Contains(err.Error(), "no spend policy") { + t.Fatalf("expected error to contain %q, got %q", "no spend policy", err) + } + + // add the spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey())), + }, + }) + if err != nil { t.Fatal(err) } - if err := primary.Subscribe(0); err != nil { + + // try to construct a transaction with more siafunds than the wallet holds. + // this will lock all of the wallet's siacoins + resp, err := wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, []types.SiafundOutput{ + {Value: 100000, Address: senderAddr}, + }, senderAddr) + if !strings.Contains(err.Error(), "insufficient funds") { t.Fatal(err) } - dbstore2, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + // construct a transaction with a single siacoin output + // this will fail if the utxos were not unlocked + resp, err = wc.Construct([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) if err != nil { t.Fatal(err) } - cm2 := chain.NewManager(dbstore2, tipState) - wm2 := walletutil.NewEphemeralWalletManager(cm2) - l2, err := net.Listen("tcp", ":0") + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(1)): + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiacoinOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[1].Address) + case !resp.Transaction.SiacoinOutputs[1].Value.Equals(types.Siacoins(99).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[1].Value) + } + + cs, err := c.ConsensusTipState() if err != nil { t.Fatal(err) } - defer l2.Close() - s2 := syncer.New(l2, cm2, syncerutil.NewEphemeralPeerStore(), gateway.Header{ - GenesisID: genesisBlock.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: l2.Addr().String(), - }) - go s2.Run() - c2, shutdown2 := runServer(cm2, s2, wm2) - defer shutdown2() - if err := c2.AddWallet("secondary", nil); err != nil { + + // sign the transaction + for i, sig := range resp.Transaction.Signatures { + sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.Signatures[i].Signature = sig[:] + } + + if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { t.Fatal(err) } - secondary := c2.Wallet("secondary") - if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) } - if err := secondary.Subscribe(0); err != nil { + expectedValue := types.Siacoins(1).Add(resp.EstimatedFee) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected confirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected confirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected confirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) } +} - // define some helper functions - addBlock := func() error { - // choose a client at random - c := c1 - if frand.Intn(2) == 0 { - c = c2 - } +func TestConstructSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) - cs, err := c.ConsensusTipState() - if err != nil { - return err - } + n, genesisBlock := testutil.V1Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + resp, err := wc.Construct(nil, []types.SiafundOutput{ + {Value: 1, Address: receiverAddr}, + }, senderAddr) + if err != nil { + t.Fatal(err) + } + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(100).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiafundOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiafundOutputs[0].Address) + case resp.Transaction.SiafundOutputs[0].Value != 1: + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiafundOutputs[0].Value) + case resp.Transaction.SiafundOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiafundOutputs[1].Address) + case resp.Transaction.SiafundOutputs[1].Value != 9999: + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiafundOutputs[1].Value) + case resp.Transaction.SiafundInputs[0].ClaimAddress != senderAddr: + t.Fatalf("expected transaction to have siafund input claim address %q, got %q", senderAddr, resp.Transaction.SiafundInputs[0].ClaimAddress) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + for i, sig := range resp.Transaction.Signatures { + sigHash := cs.WholeSigHash(resp.Transaction, sig.ParentID, 0, 0, nil) + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.Signatures[i].Signature = sig[:] + } + + if err := c.TxpoolBroadcast(resp.Basis, []types.Transaction{resp.Transaction}, nil); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV1Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV1Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } +} + +func TestConstructV2Siacoins(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + + cm := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cm, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + // add an address without a spend policy + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + // try to construct a transaction + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if !strings.Contains(err.Error(), "no spend policy") { + t.Fatalf("expected spend policy error, got %q", err) + } + + // add a spend policy to the address + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + // try to construct a transaction with more siafunds than the wallet holds. + // this will lock all of the wallet's Siacoin UTXOs + resp, err = wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, []types.SiafundOutput{ + {Value: 100000, Address: senderAddr}, + }, senderAddr) + if !strings.Contains(err.Error(), "insufficient funds") { + t.Fatal(err) + } + + // this will fail if the utxos were not properly + // unlocked when the previous request failed + resp, err = wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(1), Address: receiverAddr}, + }, nil, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + switch { + case resp.Transaction.SiacoinOutputs[0].Address != receiverAddr: + t.Fatalf("expected transaction to have output address %q, got %q", receiverAddr, resp.Transaction.SiacoinOutputs[0].Address) + case !resp.Transaction.SiacoinOutputs[0].Value.Equals(types.Siacoins(1)): + t.Fatalf("expected transaction to have output value of %v, got %v", types.Siacoins(1), resp.Transaction.SiacoinOutputs[0].Value) + case resp.Transaction.SiacoinOutputs[1].Address != senderAddr: + t.Fatalf("expected transaction to have change address %q, got %q", senderAddr, resp.Transaction.SiacoinOutputs[1].Address) + case !resp.Transaction.SiacoinOutputs[1].Value.Equals(types.Siacoins(99).Sub(resp.EstimatedFee)): + t.Fatalf("expected transaction to have change value of %v, got %v", types.Siacoins(99).Sub(resp.EstimatedFee), resp.Transaction.SiacoinOutputs[1].Value) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i := range resp.Transaction.SiacoinInputs { + sig := senderPrivateKey.SignHash(sigHash) + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + expectedValue := types.Siacoins(1).Add(resp.EstimatedFee) + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } + cm.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected confirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected confirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(expectedValue): + t.Fatalf("expected confirmed event to have outflow of %v, got %v", expectedValue, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + } +} + +func TestConstructV2Siafunds(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + err = wc.AddAddress(wallet.Address{ + Address: senderAddr, + SpendPolicy: &senderPolicy, + }) + if err != nil { + t.Fatal(err) + } + + if err := c.Rescan(0); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + resp, err := wc.ConstructV2(nil, []types.SiafundOutput{ + {Value: 1, Address: receiverAddr}, + }, senderAddr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + sig := senderPrivateKey.SignHash(sigHash) + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + for i := range resp.Transaction.SiafundInputs { + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + + unconfirmed, err := wc.UnconfirmedEvents() + if err != nil { + t.Fatal(err) + } else if len(unconfirmed) != 1 { + t.Fatalf("expected 1 unconfirmed event, got %v", len(unconfirmed)) + } + sent := unconfirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + confirmed, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(confirmed) != 2 { + t.Fatalf("expected 2 confirmed events, got %v", len(confirmed)) // initial gift + sent transaction + } + + sent = confirmed[0] + switch { + case types.TransactionID(sent.ID) != resp.ID: + t.Fatalf("expected unconfirmed event to have transaction ID %q, got %q", resp.ID, sent.ID) + case sent.Type != wallet.EventTypeV2Transaction: + t.Fatalf("expected unconfirmed event to have type %q, got %q", wallet.EventTypeV2Transaction, sent.Type) + case !sent.SiacoinOutflow().Sub(sent.SiacoinInflow()).Equals(resp.EstimatedFee): + t.Fatalf("expected unconfirmed event to have outflow of %v, got %v", resp.EstimatedFee, sent.SiacoinOutflow().Sub(sent.SiacoinInflow())) + case sent.SiafundOutflow()-sent.SiafundInflow() != 1: + t.Fatalf("expected unconfirmed event to have siafund outflow of 1, got %v", sent.SiafundOutflow()-sent.SiafundInflow()) + } +} + +func TestSpentElement(t *testing.T) { + log := zaptest.NewLogger(t) + + n, genesisBlock := testutil.V2Network() + senderPrivateKey := types.GeneratePrivateKey() + senderPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(senderPrivateKey.PublicKey()))} + senderAddr := senderPolicy.Address() + + receiverPrivateKey := types.GeneratePrivateKey() + receiverPolicy := types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(receiverPrivateKey.PublicKey()))} + receiverAddr := receiverPolicy.Address() + + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(100), + Address: senderAddr, + } + genesisBlock.Transactions[0].SiafundOutputs[0].Address = senderAddr + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log, wallet.WithIndexMode(wallet.IndexModeFull)) + + // trigger initial scan + cn.MineBlocks(t, types.VoidAddress, 1) + + sce, basis, err := c.AddressSiacoinOutputs(senderAddr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sce) != 1 { + t.Fatalf("expected 1 siacoin element, got %v", len(sce)) + } + + // check if the element is spent + spent, err := c.SpentSiacoinElement(sce[0].ID) + if err != nil { + t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siacoin element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siacoin element to have no event, got %v", spent.Event) + } + + // spend the element + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + { + Value: sce[0].SiacoinOutput.Value, + Address: receiverAddr, + }, + }, + } + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + // check if the element is spent + spent, err = c.SpentSiacoinElement(sce[0].ID) + if err != nil { + t.Fatal(err) + } else if !spent.Spent { + t.Fatal("expected siacoin element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siacoin element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siacoin element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) + } + + // mine until the utxo is pruned + cn.MineBlocks(t, types.VoidAddress, 144) + + _, err = c.SpentSiacoinElement(sce[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) + } + + sfe, basis, err := c.AddressSiafundOutputs(senderAddr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(sfe) != 1 { + t.Fatalf("expected 1 siafund element, got %v", len(sfe)) + } + + // check if the siafund element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) + if err != nil { + t.Fatal(err) + } else if spent.Spent { + t.Fatal("expected siafund element to be unspent") + } else if spent.Event != nil { + t.Fatalf("expected siafund element to have no event, got %v", spent.Event) + } + + // spend the element + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfe[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: senderPolicy, + }, + ClaimAddress: senderAddr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + { + Address: receiverAddr, + Value: sfe[0].SiafundOutput.Value, + }, + }, + } + cs, err = c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{ + senderPrivateKey.SignHash(cs.InputSigHash(txn)), + } + + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + // check if the element is spent + spent, err = c.SpentSiafundElement(sfe[0].ID) + if err != nil { + t.Fatal(err) + } else if !spent.Spent { + t.Fatal("expected siafund element to be spent") + } else if types.TransactionID(spent.Event.ID) != txn.ID() { + t.Fatalf("expected siafund element to have event %q, got %q", txn.ID(), spent.Event.ID) + } else if spent.Event.Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected siafund element to have type %q, got %q", wallet.EventTypeV2Transaction, spent.Event.Type) + } + + // mine until the utxo is pruned + cn.MineBlocks(t, types.VoidAddress, 144) + + _, err = c.SpentSiafundElement(sfe[0].ID) + if !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected error to contain %q, got %q", "not found", err) + } +} + +func TestDebugMine(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V1Network() + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + jc := jape.Client{ + BaseURL: c.BaseURL(), + Password: "password", + } + + err := jc.POST("/debug/mine", api.DebugMineRequest{ + Blocks: 5, + Address: types.VoidAddress, + }, nil) + if err != nil { + t.Fatal(err) + } + cn.WaitForSync(t) - txns, v2txns, err := c.TxpoolTransactions() + tip, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } else if tip.Height != 5 { + t.Fatalf("expected tip height to be 5, got %v", tip.Height) + } +} + +func TestAPISecurity(t *testing.T) { + n, genesisBlock := testutil.V1Network() + log := zaptest.NewLogger(t) + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + wm, err := wallet.NewManager(cn.Chain, cn.Store, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + km, err := keys.NewManager(cn.Store, "foo") + if err != nil { + t.Fatal(err) + } + defer km.Close() + + httpListener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal("failed to listen:", err) + } + defer httpListener.Close() + + server := &http.Server{ + Handler: api.NewServer(cn.Chain, cn.Syncer, wm, api.WithDebug(), api.WithKeyManager(km), api.WithLogger(zaptest.NewLogger(t)), api.WithBasicAuth("test")), + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + } + defer server.Close() + go server.Serve(httpListener) + + replaceHandler := func(apiOpts ...api.ServerOption) { + server.Handler = api.NewServer(cn.Chain, cn.Syncer, wm, apiOpts...) + } + + // create a client with correct credentials + c := api.NewClient("http://"+httpListener.Addr().String(), "test") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // check that the signing key endpoints are working + if _, err := c.GenerateSigningKey(); err != nil { + t.Fatal(err) + } + + // create a client with incorrect credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "wrong") + if _, err := c.ConsensusTip(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + + // check that the signing key endpoints are working + if _, err := c.GenerateSigningKey(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + + // replace the handler with a new one that doesn't require auth + replaceHandler() + + // create a client without credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // create a client with incorrect credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "test") + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // replace the handler with one that requires auth and has public endpoints + replaceHandler(api.WithBasicAuth("test"), api.WithPublicEndpoints(true)) + + // create a client without credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "") + + // check that a public endpoint is accessible + if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } + + // check that the signing endpoint returns 404 when public mode is enabled + if _, err := c.SignHash(frand.Entropy256(), frand.Entropy256()); err == nil { + t.Fatal("expected 404 error") + } else if !strings.Contains(err.Error(), "404") { + t.Fatal("expected 404 error, got", err) + } + + // check that a private endpoint is still protected + if _, err := c.Wallets(); err == nil { + t.Fatal("expected auth error") + } else if err.Error() == "unauthorized" { + t.Fatal("expected auth error, got", err) + } + + // create a client with credentials + c = api.NewClient("http://"+httpListener.Addr().String(), "test") + + // check that both public and private endpoints are accessible + if _, err := c.Wallets(); err != nil { + t.Fatal(err) + } else if _, err := c.ConsensusTip(); err != nil { + t.Fatal(err) + } +} + +func TestAPINoContent(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V1Network() + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + buf, err := json.Marshal(api.TxpoolBroadcastRequest{ + Transactions: []types.Transaction{}, + V2Transactions: []types.V2Transaction{}, + }) + if err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodPost, c.BaseURL()+"/txpool/broadcast", bytes.NewReader(buf)) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("expected status %v, got %v", http.StatusNoContent, resp.StatusCode) + } else if resp.ContentLength != 0 { + t.Fatalf("expected no content, got %v bytes", resp.ContentLength) + } +} + +func TestV2TransactionUpdateBasis(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V2Network() + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + // create a wallet + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + + pk := types.GeneratePrivateKey() + policy := types.SpendPolicy{Type: types.PolicyTypePublicKey(pk.PublicKey())} + addr := policy.Address() + + err = wc.AddAddress(wallet.Address{ + Address: addr, + SpendPolicy: &policy, + }) + if err != nil { + t.Fatal(err) + } + + // fund the wallet + cn.MineBlocks(t, addr, 5+int(n.MaturityDelay)) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(100), Address: addr}, + }, nil, addr) + if err != nil { + t.Fatal(err) + } + parentTxn, basis := resp.Transaction, resp.Basis + + // sign the transaction + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + sigHash := cs.InputSigHash(parentTxn) + sig := pk.SignHash(sigHash) + for i := range parentTxn.SiacoinInputs { + parentTxn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + // broadcast the transaction + if err := c.TxpoolBroadcast(basis, nil, []types.V2Transaction{parentTxn}); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + // create a child transaction + sce := parentTxn.EphemeralSiacoinOutput(0) + childTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + childSigHash := cs.InputSigHash(childTxn) + childTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(childSigHash)} + + txnset := []types.V2Transaction{parentTxn, childTxn} + + tip, err := c.ConsensusTip() + if err != nil { + t.Fatal(err) + } + + basis, txnset, err = c.V2UpdateTransactionSetBasis(txnset, basis, tip) + if err != nil { + t.Fatal(err) + } else if len(txnset) != 1 { + t.Fatalf("expected 1 transactions, got %v", len(txnset)) + } else if txnset[0].ID() != childTxn.ID() { + t.Fatalf("expected parent transaction to be removed") + } else if basis != tip { + t.Fatalf("expected basis to be %v, got %v", tip, basis) + } + + if err := c.TxpoolBroadcast(basis, nil, txnset); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) +} + +func TestSigning(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesisBlock := testutil.V2Network() + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + pk, err := c.GenerateSigningKey() + if err != nil { + t.Fatal(err) + } + + // create a wallet + w, err := c.AddWallet(api.WalletUpdateRequest{ + Name: "primary", + }) + if err != nil { + t.Fatal(err) + } + + wc := c.Wallet(w.ID) + + policy := types.SpendPolicy{Type: types.PolicyTypePublicKey(pk)} + addr := policy.Address() + + err = wc.AddAddress(wallet.Address{ + Address: addr, + SpendPolicy: &policy, + }) + if err != nil { + t.Fatal(err) + } + + // fund the wallet + cn.MineBlocks(t, addr, 1) + cn.MineBlocks(t, types.VoidAddress, int(n.MaturityDelay)) + + resp, err := wc.ConstructV2([]types.SiacoinOutput{ + {Value: types.Siacoins(100), Address: addr}, + }, nil, addr) + if err != nil { + t.Fatal(err) + } + + cs, err := c.ConsensusTipState() + if err != nil { + t.Fatal(err) + } + + // sign the transaction + sigHash := cs.InputSigHash(resp.Transaction) + for i, si := range resp.Transaction.SiacoinInputs { + pk := types.PublicKey(si.SatisfiedPolicy.Policy.Type.(types.PolicyTypePublicKey)) + + sig, err := c.SignHash(pk, sigHash) if err != nil { - return err - } - b := types.Block{ - ParentID: cs.Index.ID, - Timestamp: types.CurrentTimestamp(), - MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, - Transactions: txns, - } - if len(v2txns) > 0 { - b.V2 = &types.V2BlockData{ - Height: cs.Index.Height + 1, - Transactions: v2txns, - } - b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) - } - for b.ID().CmpWork(cs.ChildTarget) < 0 { - b.Nonce += cs.NonceFactor() - } - if err := c.SyncerBroadcastBlock(b); err != nil { - return err - } - // wait for tips to update - again: - time.Sleep(10 * time.Millisecond) - if tip1, err := c1.ConsensusTip(); err != nil { - return err - } else if tip2, err := c2.ConsensusTip(); err != nil { - return err - } else if tip1 == cs.Index || tip2 == cs.Index { - goto again + t.Fatal(err) + } else if !pk.VerifyHash(sigHash, sig) { + t.Fatal("signature verification failed") } - return nil + resp.Transaction.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{sig} + } + + if err := c.TxpoolBroadcast(resp.Basis, nil, []types.V2Transaction{resp.Transaction}); err != nil { + t.Fatal(err) + } + cn.MineBlocks(t, types.VoidAddress, 1) + + events, err := wc.Events(0, 5) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected event type %q, got %q", wallet.EventTypeV2Transaction, events[0].Type) + } else if types.TransactionID(events[0].ID) != resp.ID { + t.Fatalf("expected event ID %q, got %q", resp.ID, events[0].ID) } - checkBalances := func(p, s types.Currency) { +} + +func TestMineGetBlockTemplate(t *testing.T) { + log := zaptest.NewLogger(t) + + test := func(n *consensus.Network, genesisBlock types.Block) { t.Helper() - if primaryBalance, err := primary.Balance(); err != nil { + + cn := testutil.NewConsensusNode(t, n, genesisBlock, log) + c := startWalletServer(t, cn, log) + + // mine a few blocks to avoid starting at 0 + cn.MineBlocks(t, types.Address{}, 10) + + // get block template + minerAddr := types.Address{1, 2, 3} + resp, err := c.MiningGetBlockTemplate(minerAddr, "") + if err != nil { t.Fatal(err) - } else if !primaryBalance.Siacoins.Equals(p) { - t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) } - if secondaryBalance, err := secondary.Balance(); err != nil { + + var parentID types.BlockID + if err := parentID.UnmarshalText([]byte(resp.PreviousBlockHash)); err != nil { t.Fatal(err) - } else if !secondaryBalance.Siacoins.Equals(s) { - t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) } - } - sendV1 := func() error { - t.Helper() - // which wallet is sending? - c := c1 - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() + rawMinerPayout, err := hex.DecodeString(resp.MinerPayout[0].Data) if err != nil { t.Fatal(err) } - if len(sces) == 0 { - c = c2 - key = secondaryPrivateKey - dest = primaryAddress - sces, _, err = secondary.Outputs() + dec := types.NewBufDecoder(rawMinerPayout) + + var minerPayout types.SiacoinOutput + switch resp.Version { + case 1: + (*types.V1SiacoinOutput)(&minerPayout).DecodeFrom(dec) + case 2: + (*types.V2SiacoinOutput)(&minerPayout).DecodeFrom(dec) + default: + t.Fatal("unknown version", resp.Version) + } + if err := dec.Err(); err != nil { + t.Fatal(err) + } + + var txns []types.Transaction + var v2Txns []types.V2Transaction + for _, templateTxn := range resp.Transactions { + rawTxn, err := hex.DecodeString(templateTxn.Data) if err != nil { t.Fatal(err) } - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value - } - sce := sces[0] - - txn := types.Transaction{ - SiacoinInputs: []types.SiacoinInput{{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - Signatures: []types.TransactionSignature{{ - ParentID: sce.ID, - CoveredFields: types.CoveredFields{WholeTransaction: true}, - }}, - } - cs, err := c.ConsensusTipState() - if err != nil { - return err - } - sig := key.SignHash(cs.WholeSigHash(txn, sce.ID, 0, 0, nil)) - txn.Signatures[0].Signature = sig[:] - if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { - return err - } else if err := addBlock(); err != nil { - return err - } - checkBalances(pbal, sbal) - return nil - } - sendV2 := func() error { - t.Helper() - // which wallet is sending? - c := c1 - key := primaryPrivateKey - dest := secondaryAddress - pbal, sbal := types.ZeroCurrency, types.ZeroCurrency - sces, _, err := primary.Outputs() - if err != nil { - t.Fatal(err) + dec := types.NewBufDecoder(rawTxn) + switch templateTxn.TxType { + case "1": + var txn types.Transaction + txn.DecodeFrom(dec) + if err := dec.Err(); err != nil { + t.Fatal(err) + } + txns = append(txns, txn) + case "2": + var txn types.V2Transaction + txn.DecodeFrom(dec) + if err := dec.Err(); err != nil { + t.Fatal(err) + } + v2Txns = append(v2Txns, txn) + default: + t.Fatal("unknown type", templateTxn.TxType) + } } - if len(sces) == 0 { - c = c2 - key = secondaryPrivateKey - dest = primaryAddress - sces, _, err = secondary.Outputs() + + var v2BlockData *types.V2BlockData + if resp.Version == 2 { + v2BlockData = &types.V2BlockData{ + Height: uint64(resp.Height), + Transactions: v2Txns, + } + + cs, err := c.ConsensusTipState() if err != nil { t.Fatal(err) } - pbal = sces[0].SiacoinOutput.Value - } else { - sbal = sces[0].SiacoinOutput.Value + v2BlockData.Commitment = cs.Commitment(cs.TransactionsCommitment(txns, v2Txns), minerAddr) } - sce := sces[0] - txn := types.V2Transaction{ - SiacoinInputs: []types.V2SiacoinInput{{ - Parent: sce, - SatisfiedPolicy: types.SatisfiedPolicy{ - Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, - }, - }}, - SiacoinOutputs: []types.SiacoinOutput{{ - Address: dest, - Value: sce.SiacoinOutput.Value, - }}, - } - cs, err := c.ConsensusTipState() - if err != nil { - return err - } - txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cs.InputSigHash(txn))} - if err := c.TxpoolBroadcast(nil, []types.V2Transaction{txn}); err != nil { - return err - } else if err := addBlock(); err != nil { - return err + // construct block + b := types.Block{ + ParentID: parentID, + Timestamp: time.Unix(int64(resp.Timestamp), 0), + MinerPayouts: []types.SiacoinOutput{minerPayout}, + V2: v2BlockData, + Transactions: txns, } - checkBalances(pbal, sbal) - return nil - } - // connect the syncers - if _, err := s1.Connect(s2.Addr()); err != nil { - t.Fatal(err) - } + var target types.BlockID + if err := target.UnmarshalText([]byte(resp.Target)); err != nil { + t.Fatal(err) + } - // attempt to send primary->secondary with a v2 txn; should fail - if err := sendV2(); err == nil { - t.Fatal("expected v2 txn to be rejected") - } - // use a v1 transaction instead - if err := sendV1(); err != nil { - t.Fatal(err) - } + // mine block + mineBlock := func(b *types.Block, target types.BlockID) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + factor := 1009 + for b.ID().CmpWork(target) < 0 { + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + default: + } + b.Nonce += uint64(factor) + } + } + mineBlock(&b, target) - // mine past v2 allow height - for cm1.Tip().Height <= n.HardforkV2.AllowHeight { - if err := addBlock(); err != nil { + // submit block + if err := c.MiningSubmitBlock(b); err != nil { t.Fatal(err) } - } - // now send coins back with a v2 transaction - if err := sendV2(); err != nil { - t.Fatal(err) - } - // v1 transactions should also still work - if err := sendV1(); err != nil { - t.Fatal(err) - } - // mine past v2 require height - for cm1.Tip().Height <= n.HardforkV2.RequireHeight { - if err := addBlock(); err != nil { + // the block should be the new tip + tip, err := c.ConsensusTip() + if err != nil { t.Fatal(err) + } else if tip.ID != b.ID() { + t.Fatalf("expected tip to be %v, got %v", b.ID(), tip.ID) } } - // v1 transactions should no longer work - if err := sendV1(); err == nil { - t.Fatal("expected v1 txn to be rejected") - } - // use a v2 transaction instead - if err := sendV2(); err != nil { - t.Fatal(err) - } + + t.Run("v1", func(t *testing.T) { + network, genesisBlock := testutil.V1Network() + test(network, genesisBlock) + }) + + t.Run("v2", func(t *testing.T) { + network, genesisBlock := testutil.V2Network() + test(network, genesisBlock) + }) } diff --git a/api/client.go b/api/client.go index 973f194..f4f1704 100644 --- a/api/client.go +++ b/api/client.go @@ -1,12 +1,16 @@ package api import ( - "encoding/json" + "bytes" + "encoding/hex" "fmt" + "net/url" + "sync" "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/jape" "go.sia.tech/walletd/wallet" ) @@ -14,20 +18,70 @@ import ( // A Client provides methods for interacting with a walletd API server. type Client struct { c jape.Client - n *consensus.Network // for ConsensusTipState + + mu sync.Mutex // protects n + n *consensus.Network +} + +func (c *Client) getNetwork() (*consensus.Network, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.n == nil { + var err error + c.n, err = c.ConsensusNetwork() + if err != nil { + return nil, err + } + } + return c.n, nil +} + +// BaseURL returns the URL of the walletd server. +func (c *Client) BaseURL() string { + return c.c.BaseURL +} + +// State returns information about the current state of the walletd daemon. +func (c *Client) State() (resp StateResponse, err error) { + err = c.c.GET("/state", &resp) + return } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(txns []types.Transaction, v2txns []types.V2Transaction) (err error) { - err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) +func (c *Client) TxpoolBroadcast(basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction) (err error) { + err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{ + Basis: basis, + Transactions: txns, + V2Transactions: v2txns, + }, nil) return } // TxpoolTransactions returns all transactions in the transaction pool. -func (c *Client) TxpoolTransactions() (txns []types.Transaction, v2txns []types.V2Transaction, err error) { +func (c *Client) TxpoolTransactions() (basis types.ChainIndex, txns []types.Transaction, v2txns []types.V2Transaction, err error) { var resp TxpoolTransactionsResponse err = c.c.GET("/txpool/transactions", &resp) - return resp.Transactions, resp.V2Transactions, err + return resp.Basis, resp.Transactions, resp.V2Transactions, err +} + +// V2UpdateTransactionSetBasis updates a V2 transaction set's basis to the target index. +func (c *Client) V2UpdateTransactionSetBasis(txnset []types.V2Transaction, from, to types.ChainIndex) (types.ChainIndex, []types.V2Transaction, error) { + req := TxpoolUpdateV2TransactionsRequest{ + Basis: from, + Target: to, + Transactions: txnset, + } + var resp TxpoolUpdateV2TransactionsResponse + err := c.c.POST("/txpool/transactions/v2/basis", req, &resp) + return resp.Basis, resp.Transactions, err +} + +// TxpoolParents returns the parents of a transaction that are currently in the +// transaction pool. +func (c *Client) TxpoolParents(txn types.Transaction) (resp []types.Transaction, err error) { + err = c.c.POST("/txpool/parents", txn, &resp) + return } // TxpoolFee returns the recommended fee (per weight unit) to ensure a high @@ -44,25 +98,102 @@ func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { return } -// ConsensusTip returns the current tip index. -func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { - err = c.c.GET("/consensus/tip", &resp) +// ConsensusBlocksID returns the block with the given id. +func (c *Client) ConsensusBlocksID(bid types.BlockID) (resp types.Block, err error) { + err = c.c.GET(fmt.Sprintf("/consensus/blocks/%v", bid), &resp) return } +// ConsensusIndex returns the consensus index at the specified height. +func (c *Client) ConsensusIndex(height uint64) (resp types.ChainIndex, err error) { + err = c.c.GET(fmt.Sprintf("/consensus/index/%d", height), &resp) + return +} + +// ConsensusUpdates returns at most n consensus updates that have occurred since +// the specified index +func (c *Client) ConsensusUpdates(index types.ChainIndex, limit int) ([]chain.RevertUpdate, []chain.ApplyUpdate, error) { + // index.String() is a short-hand representation. We need the full text + indexBuf, err := index.MarshalText() + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal index: %w", err) + } + + var resp ConsensusUpdatesResponse + if err := c.c.GET(fmt.Sprintf("/consensus/updates/%s?limit=%d", indexBuf, limit), &resp); err != nil { + return nil, nil, err + } + + network, err := c.getNetwork() + if err != nil { + return nil, nil, fmt.Errorf("failed to get network metadata: %w", err) + } + + reverted := make([]chain.RevertUpdate, 0, len(resp.Reverted)) + for _, u := range resp.Reverted { + revert := chain.RevertUpdate{ + RevertUpdate: u.Update, + State: u.State, + Block: u.Block, + } + revert.State.Network = network + reverted = append(reverted, revert) + } + + applied := make([]chain.ApplyUpdate, 0, len(resp.Applied)) + for _, u := range resp.Applied { + apply := chain.ApplyUpdate{ + ApplyUpdate: u.Update, + State: u.State, + Block: u.Block, + } + apply.State.Network = network + applied = append(applied, apply) + } + return reverted, applied, nil +} + // ConsensusTipState returns the current tip state. func (c *Client) ConsensusTipState() (resp consensus.State, err error) { - if c.n == nil { - c.n, err = c.ConsensusNetwork() - if err != nil { - return - } + if err = c.c.GET("/consensus/tipstate", &resp); err != nil { + return } - err = c.c.GET("/consensus/tipstate", &resp) - resp.Network = c.n + resp.Network, err = c.getNetwork() + return +} + +// ConsensusTip returns the current tip index. +func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { + err = c.c.GET("/consensus/tip", &resp) return } +// MiningGetBlockTemplate returns a block template for mining. +func (c *Client) MiningGetBlockTemplate(payoutAddr types.Address, longPollID string) (resp MiningGetBlockTemplateResponse, err error) { + err = c.c.POST("/mining/getblocktemplate", MiningGetBlockTemplateRequest{ + LongPollID: longPollID, + PayoutAddress: payoutAddr, + }, &resp) + return +} + +// MiningSubmitBlock submits a mined block to the network. +func (c *Client) MiningSubmitBlock(b types.Block) error { + buf := new(bytes.Buffer) + enc := types.NewEncoder(buf) + if b.V2 == nil { + types.V1Block(b).EncodeTo(enc) + } else { + types.V2Block(b).EncodeTo(enc) + } + if err := enc.Flush(); err != nil { + return fmt.Errorf("failed to encode block: %w", err) + } + return c.c.POST("/mining/submitblock", MiningSubmitBlockRequest{ + Params: []string{hex.EncodeToString(buf.Bytes())}, + }, nil) +} + // SyncerPeers returns the current peers of the syncer. func (c *Client) SyncerPeers() (resp []GatewayPeer, err error) { err = c.c.GET("/syncer/peers", &resp) @@ -82,100 +213,199 @@ func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { } // Wallets returns the set of tracked wallets. -func (c *Client) Wallets() (ws map[string]json.RawMessage, err error) { +func (c *Client) Wallets() (ws []wallet.Wallet, err error) { err = c.c.GET("/wallets", &ws) return } // AddWallet adds a wallet to the set of tracked wallets. -func (c *Client) AddWallet(name string, info json.RawMessage) (err error) { - err = c.c.PUT(fmt.Sprintf("/wallets/%v", name), info) +func (c *Client) AddWallet(uw WalletUpdateRequest) (w wallet.Wallet, err error) { + err = c.c.POST("/wallets", uw, &w) + return +} + +// UpdateWallet updates a wallet. +func (c *Client) UpdateWallet(id wallet.ID, uw WalletUpdateRequest) (w wallet.Wallet, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v", id), uw, &w) return } // RemoveWallet deletes a wallet. If the wallet is currently subscribed, it will // be unsubscribed. -func (c *Client) RemoveWallet(name string) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v", name)) +func (c *Client) RemoveWallet(id wallet.ID) (err error) { + err = c.c.DELETE(fmt.Sprintf("/wallets/%v", id)) return } // Wallet returns a client for interacting with the specified wallet. -func (c *Client) Wallet(name string) *WalletClient { - return &WalletClient{c: c.c, name: name} +func (c *Client) Wallet(id wallet.ID) *WalletClient { + return &WalletClient{c: c.c, id: id} } -// A WalletClient provides methods for interacting with a particular wallet on a -// walletd API server. -type WalletClient struct { - c jape.Client - name string +// ScanStatus returns the current state of wallet scanning. +func (c *Client) ScanStatus() (resp RescanResponse, err error) { + err = c.c.GET("/rescan", &resp) + return +} + +// Rescan rescans the blockchain starting from the specified height. +func (c *Client) Rescan(height uint64) (err error) { + err = c.c.POST("/rescan", height, nil) + return +} + +// AddressBalance returns the balance of a single address. +func (c *Client) AddressBalance(addr types.Address) (resp BalanceResponse, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/balance", addr), &resp) + return +} + +// AddressEvents returns the events of a single address. +func (c *Client) AddressEvents(addr types.Address, offset, limit int) (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/events?offset=%d&limit=%d", addr, offset, limit), &resp) + return +} + +// AddressUnconfirmedEvents returns the unconfirmed events for a single address. +func (c *Client) AddressUnconfirmedEvents(addr types.Address) (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/addresses/%v/events/unconfirmed", addr), &resp) + return +} + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (c *Client) AddressSiacoinOutputs(addr types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + var resp SiacoinElementsResponse + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siacoin?offset=%d&limit=%d", addr, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } -// Subscribe subscribes the wallet to consensus updates, starting at the -// specified height. This can only be done once. -func (c *WalletClient) Subscribe(height uint64) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/subscribe", c.name), height, nil) +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (c *Client) AddressSiafundOutputs(addr types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + var resp SiafundElementsResponse + err := c.c.GET(fmt.Sprintf("/addresses/%v/outputs/siafund?offset=%d&limit=%d", addr, offset, limit), &resp) + return resp.Outputs, resp.Basis, err +} + +// Event returns the event with the specified ID. +func (c *Client) Event(id types.Hash256) (resp wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/events/%v", id), &resp) return } +// SpentSiacoinElement returns whether a siacoin output has been spent and the +// event that spent it. +func (c *Client) SpentSiacoinElement(id types.SiacoinOutputID) (resp ElementSpentResponse, err error) { + err = c.c.GET(fmt.Sprintf("/outputs/siacoin/%v/spent", id), &resp) + return +} + +// SpentSiafundElement returns whether a siafund output has been spent and the +// event that spent it. +func (c *Client) SpentSiafundElement(id types.SiafundOutputID) (resp ElementSpentResponse, err error) { + err = c.c.GET(fmt.Sprintf("/outputs/siafund/%v/spent", id), &resp) + return +} + +// GenerateSigningKey generates a new ed25519 private key +// on the server and adds it to the key store. Returns the +// public key. +func (c *Client) GenerateSigningKey() (types.PublicKey, error) { + var resp AddSigningKeyResponse + err := c.c.POST("/keys/generate/ed25519", nil, &resp) + return resp.PublicKey, err +} + +// ImportSigningKey imports an ed25519 signing key into the key store. +// Returns the public key. +func (c *Client) ImportSigningKey(sk types.PrivateKey) (types.PublicKey, error) { + var resp AddSigningKeyResponse + err := c.c.POST("/keys/ed25519", AddSigningKeyRequest{PrivateKey: sk}, &resp) + return resp.PublicKey, err +} + +// DeleteSigningKey deletes an ed25519 signing key from the key store. +func (c *Client) DeleteSigningKey(pk types.PublicKey) error { + return c.c.DELETE(fmt.Sprintf("/keys/ed25519/%s", pk)) +} + +// SignHash signs a hash with the specified key. If the key is not found, it +// returns 404 and [keys.ErrNotFound]. +func (c *Client) SignHash(key types.PublicKey, hash types.Hash256) (types.Signature, error) { + var resp SignHashResponse + err := c.c.POST(fmt.Sprintf("/keys/ed25519/%s/sign", url.PathEscape(key.String())), SignHashRequest{hash}, &resp) + return resp.Signature, err +} + +// A WalletClient provides methods for interacting with a particular wallet on a +// walletd API server. +type WalletClient struct { + c jape.Client + id wallet.ID +} + // AddAddress adds the specified address and associated metadata to the // wallet. -func (c *WalletClient) AddAddress(addr types.Address, info json.RawMessage) (err error) { - err = c.c.PUT(fmt.Sprintf("/wallets/%v/addresses/%v", c.name, addr), info) +func (c *WalletClient) AddAddress(a wallet.Address) (err error) { + err = c.c.PUT(fmt.Sprintf("/wallets/%v/addresses", c.id), a) return } // RemoveAddress removes the specified address from the wallet. func (c *WalletClient) RemoveAddress(addr types.Address) (err error) { - err = c.c.DELETE(fmt.Sprintf("/wallets/%v/addresses/%v", c.name, addr)) + err = c.c.DELETE(fmt.Sprintf("/wallets/%v/addresses/%v", c.id, addr)) return } // Addresses the addresses controlled by the wallet. -func (c *WalletClient) Addresses() (resp map[types.Address]json.RawMessage, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/addresses", c.name), &resp) +func (c *WalletClient) Addresses() (resp []wallet.Address, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/addresses", c.id), &resp) return } // Balance returns the current wallet balance. -func (c *WalletClient) Balance() (resp WalletBalanceResponse, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.name), &resp) +func (c *WalletClient) Balance() (resp BalanceResponse, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/balance", c.id), &resp) return } // Events returns all events relevant to the wallet. func (c *WalletClient) Events(offset, limit int) (resp []wallet.Event, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.name, offset, limit), &resp) + err = c.c.GET(fmt.Sprintf("/wallets/%v/events?offset=%d&limit=%d", c.id, offset, limit), &resp) return } -// PoolTransactions returns all txpool transactions relevant to the wallet. -func (c *WalletClient) PoolTransactions() (resp []wallet.PoolTransaction, err error) { - err = c.c.GET(fmt.Sprintf("/wallets/%v/txpool", c.name), &resp) +// UnconfirmedEvents returns all unconfirmed events relevant to the wallet. +func (c *WalletClient) UnconfirmedEvents() (resp []wallet.Event, err error) { + err = c.c.GET(fmt.Sprintf("/wallets/%v/events/unconfirmed", c.id), &resp) return } -// Outputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) Outputs() (sc []types.SiacoinElement, sf []types.SiafundElement, err error) { - var resp WalletOutputsResponse - err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs", c.name), &resp) - return resp.SiacoinOutputs, resp.SiafundOutputs, err +// SiacoinOutputs returns the set of unspent outputs controlled by the wallet. +func (c *WalletClient) SiacoinOutputs(offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + var resp SiacoinElementsResponse + err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siacoin?offset=%d&limit=%d", c.id, offset, limit), &resp) + return resp.Outputs, resp.Basis, err +} + +// SiafundOutputs returns the set of unspent outputs controlled by the wallet. +func (c *WalletClient) SiafundOutputs(offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + var resp SiafundElementsResponse + err := c.c.GET(fmt.Sprintf("/wallets/%v/outputs/siafund?offset=%d&limit=%d", c.id, offset, limit), &resp) + return resp.Outputs, resp.Basis, err } // Reserve reserves a set outputs for use in a transaction. func (c *WalletClient) Reserve(sc []types.SiacoinOutputID, sf []types.SiafundOutputID, duration time.Duration) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.name), WalletReserveRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/reserve", c.id), WalletReserveRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, - Duration: duration, }, nil) return } // Release releases a set of previously-reserved outputs. func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOutputID) (err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/release", c.name), WalletReleaseRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/release", c.id), WalletReleaseRequest{ SiacoinOutputs: sc, SiafundOutputs: sf, }, nil) @@ -184,7 +414,7 @@ func (c *WalletClient) Release(sc []types.SiacoinOutputID, sf []types.SiafundOut // Fund funds a siacoin transaction. func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, changeAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fund", c.name), WalletFundRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/fund", c.id), WalletFundRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, @@ -194,7 +424,7 @@ func (c *WalletClient) Fund(txn types.Transaction, amount types.Currency, change // FundSF funds a siafund transaction. func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, claimAddr types.Address) (resp WalletFundResponse, err error) { - err = c.c.POST(fmt.Sprintf("/wallets/%v/fundsf", c.name), WalletFundSFRequest{ + err = c.c.POST(fmt.Sprintf("/wallets/%v/fundsf", c.id), WalletFundSFRequest{ Transaction: txn, Amount: amount, ChangeAddress: changeAddr, @@ -203,6 +433,28 @@ func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, return } +// Construct constructs a transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned +// along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. +func (c *WalletClient) Construct(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructResponse, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/transaction", c.id), WalletConstructRequest{ + Siacoins: siacoins, + Siafunds: siafunds, + ChangeAddress: change, + }, &resp) + return +} + +// ConstructV2 constructs a V2 transaction sending the specified Siacoins or Siafunds to the recipients. The transaction is returned +// along with its ID and calculated miner fee. The transaction will need to be signed before broadcasting. +func (c *WalletClient) ConstructV2(siacoins []types.SiacoinOutput, siafunds []types.SiafundOutput, change types.Address) (resp WalletConstructV2Response, err error) { + err = c.c.POST(fmt.Sprintf("/wallets/%v/construct/v2/transaction", c.id), WalletConstructRequest{ + Siacoins: siacoins, + Siafunds: siafunds, + ChangeAddress: change, + }, &resp) + return +} + // NewClient returns a client that communicates with a walletd server listening // on the specified address. func NewClient(addr, password string) *Client { diff --git a/api/mine.go b/api/mine.go new file mode 100644 index 0000000..862d5dd --- /dev/null +++ b/api/mine.go @@ -0,0 +1,206 @@ +package api + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "math/big" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "lukechampine.com/frand" +) + +func generateBlockTemplate(cm ChainManager, addr types.Address) (MiningGetBlockTemplateResponse, error) { + block, cs := unsolvedBlock(cm, addr) + + // sanity check miner payouts + if len(block.MinerPayouts) != 1 { + return MiningGetBlockTemplateResponse{}, fmt.Errorf("expected 1 miner payout got %d", len(block.MinerPayouts)) + } + + // figure out encoding version + version := uint32(1) + if block.V2 != nil { + version = 2 + } + + // encode payout + buf := new(bytes.Buffer) + enc := types.NewEncoder(buf) + if block.V2 == nil { + types.V1SiacoinOutput(block.MinerPayouts[0]).EncodeTo(enc) + } else { + types.V2SiacoinOutput(block.MinerPayouts[0]).EncodeTo(enc) + } + if err := enc.Flush(); err != nil { + return MiningGetBlockTemplateResponse{}, err + } + minerPayout := MiningGetBlockTemplateResponseTxn{ + Data: hex.EncodeToString(buf.Bytes()), + } + + // encode transactions + var txns []MiningGetBlockTemplateResponseTxn + for _, txn := range block.Transactions { + buf.Reset() + txn.EncodeTo(enc) + if err := enc.Flush(); err != nil { + return MiningGetBlockTemplateResponse{}, err + } + txns = append(txns, MiningGetBlockTemplateResponseTxn{ + Data: hex.EncodeToString(buf.Bytes()), + TxID: txn.ID().String(), + TxType: "1", // types.Transaction encoding + }) + } + if block.V2 != nil { + for _, txn := range block.V2.Transactions { + buf.Reset() + txn.EncodeTo(enc) + if err := enc.Flush(); err != nil { + return MiningGetBlockTemplateResponse{}, err + } + txns = append(txns, MiningGetBlockTemplateResponseTxn{ + Data: hex.EncodeToString(buf.Bytes()), + TxID: txn.ID().String(), + TxType: "2", // types.V2Transaction encoding + }) + } + } + + return MiningGetBlockTemplateResponse{ + Transactions: txns, + MinerPayout: []MiningGetBlockTemplateResponseTxn{minerPayout}, + PreviousBlockHash: block.ParentID.String(), + LongPollID: hex.EncodeToString(frand.Bytes(16)), + Target: cs.ChildTarget.String(), + Height: uint32(cs.Index.Height) + 1, + Timestamp: int32(block.Timestamp.Unix()), + Version: version, + Bits: compressDifficulty(cs.Difficulty), + }, nil +} + +func compressDifficulty(w consensus.Work) string { + buf := new(bytes.Buffer) + enc := types.NewEncoder(buf) + w.EncodeTo(enc) + if err := enc.Flush(); err != nil { + panic("failed to flush encoder") // can't fail + } + b := new(big.Int).SetBytes(buf.Bytes()) + return fmt.Sprintf("%08X", bigToCompact(b)) +} + +// bigToCompact converts a whole number N to a compact representation using an +// unsigned 32-bit number. +func bigToCompact(n *big.Int) uint32 { + // No need to do any work if it's zero. + if n.Sign() == 0 { + return 0 + } + + // Since the base for the exponent is 256, the exponent can be treated + // as the number of bytes. So, shift the number right or left + // accordingly. This is equivalent to: + // mantissa = mantissa / 256^(exponent-3) + var mantissa uint32 + exponent := uint(len(n.Bytes())) + if exponent <= 3 { + mantissa = uint32(n.Bits()[0]) + mantissa <<= 8 * (3 - exponent) + } else { + // Use a copy to avoid modifying the caller's original number. + tn := new(big.Int).Set(n) + mantissa = uint32(tn.Rsh(tn, 8*(exponent-3)).Bits()[0]) + } + + // When the mantissa already has the sign bit set, the number is too + // large to fit into the available 23-bits, so divide the number by 256 + // and increment the exponent accordingly. + if mantissa&0x00800000 != 0 { + mantissa >>= 8 + exponent++ + } + + // Pack the exponent, sign bit, and mantissa into an unsigned 32-bit + // int and return it. + compact := uint32(exponent<<24) | mantissa + if n.Sign() < 0 { + compact |= 0x00800000 + } + return compact +} + +// mineBlock constructs a block from the provided address and the transactions +// in the txpool, and attempts to find a nonce for it that meets the PoW target. +func mineBlock(ctx context.Context, cm ChainManager, addr types.Address) (types.Block, error) { + b, cs := unsolvedBlock(cm, addr) + factor := cs.NonceFactor() + for b.ID().CmpWork(cs.ChildTarget) < 0 { + select { + case <-ctx.Done(): + return types.Block{}, ctx.Err() + default: + } + + // tip changed, abort mining + if cm.Tip() != cs.Index { + return types.Block{}, errors.New("tip changed") + } + + b.Nonce += factor + } + return b, nil +} + +func unsolvedBlock(cm ChainManager, addr types.Address) (types.Block, consensus.State) { +retry: + cs := cm.TipState() + txns := cm.PoolTransactions() + v2Txns := cm.V2PoolTransactions() + if cs.Index != cm.Tip() { + goto retry + } + + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{ + Value: cs.BlockReward(), + Address: addr, + }}, + } + + if cs.Index.Height >= cs.Network.HardforkV2.AllowHeight { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + } + } + if cs.Index.Height >= cs.Network.HardforkV2.RequireHeight { + txns = nil // ignore potential v1 transactions + } + + var weight uint64 + for _, txn := range txns { + if weight += cs.TransactionWeight(txn); weight > cs.MaxBlockWeight() { + break + } + b.Transactions = append(b.Transactions, txn) + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) + } + for _, txn := range v2Txns { + if weight += cs.V2TransactionWeight(txn); weight > cs.MaxBlockWeight() { + break + } + b.V2.Transactions = append(b.V2.Transactions, txn) + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) + } + if b.V2 != nil { + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), addr) + } + return b, cs +} diff --git a/api/server.go b/api/server.go index 83321ea..6938da8 100644 --- a/api/server.go +++ b/api/server.go @@ -1,73 +1,181 @@ package api import ( - "encoding/json" + "context" + "encoding/hex" "errors" "fmt" "net/http" - "reflect" + "net/http/pprof" + "runtime" "sync" "time" "go.sia.tech/jape" - "lukechampine.com/frand" + "go.uber.org/zap" "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" - "go.sia.tech/walletd/syncer" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/walletd/build" + "go.sia.tech/walletd/keys" "go.sia.tech/walletd/wallet" ) +// A ServerOption sets an optional parameter for the server. +type ServerOption func(*server) + +// WithLogger sets the logger used by the server. +func WithLogger(log *zap.Logger) ServerOption { + return func(s *server) { + s.log = log + } +} + +// WithDebug enables debug endpoints. +func WithDebug() ServerOption { + return func(s *server) { + s.debugEnabled = true + } +} + +// WithKeyManager sets the key manager used by the server. +func WithKeyManager(ks SigningKeyManager) ServerOption { + return func(s *server) { + s.km = ks + } +} + +// WithPublicEndpoints sets whether the server should disable authentication +// on endpoints that are safe for use when running walletd as a service. +func WithPublicEndpoints(public bool) ServerOption { + return func(s *server) { + s.publicEndpoints = public + } +} + +// WithBasicAuth sets the password for basic authentication. +func WithBasicAuth(password string) ServerOption { + return func(s *server) { + s.password = password + } +} + type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { + UpdatesSince(types.ChainIndex, int) ([]chain.RevertUpdate, []chain.ApplyUpdate, error) + + Tip() types.ChainIndex + BestIndex(height uint64) (types.ChainIndex, bool) + Block(id types.BlockID) (types.Block, bool) TipState() consensus.State AddBlocks([]types.Block) error RecommendedFee() types.Currency PoolTransactions() []types.Transaction V2PoolTransactions() []types.V2Transaction - AddPoolTransactions(txns []types.Transaction) error - AddV2PoolTransactions(txns []types.V2Transaction) error + AddPoolTransactions(txns []types.Transaction) (bool, error) + AddV2PoolTransactions(index types.ChainIndex, txns []types.V2Transaction) (bool, error) UnconfirmedParents(txn types.Transaction) []types.Transaction + UpdateV2TransactionSet(txns []types.V2Transaction, from types.ChainIndex, to types.ChainIndex) ([]types.V2Transaction, error) } // A Syncer can connect to other peers and synchronize the blockchain. Syncer interface { Addr() string - Peers() []*gateway.Peer - PeerInfo(peer string) (syncer.PeerInfo, bool) - Connect(addr string) (*gateway.Peer, error) - BroadcastHeader(bh gateway.BlockHeader) + Peers() []*syncer.Peer + PeerInfo(addr string) (syncer.PeerInfo, error) + Connect(ctx context.Context, addr string) (*syncer.Peer, error) + BroadcastHeader(types.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) - BroadcastV2TransactionSet(txns []types.V2Transaction) + BroadcastV2TransactionSet(index types.ChainIndex, txns []types.V2Transaction) BroadcastV2BlockOutline(bo gateway.V2BlockOutline) } // A WalletManager manages wallets, keyed by name. WalletManager interface { - AddWallet(name string, info json.RawMessage) error - DeleteWallet(name string) error - Wallets() map[string]json.RawMessage - SubscribeWallet(name string, startHeight uint64) error - - AddAddress(name string, addr types.Address, info json.RawMessage) error - RemoveAddress(name string, addr types.Address) error - Addresses(name string) (map[types.Address]json.RawMessage, error) - Events(name string, offset, limit int) ([]wallet.Event, error) - UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) - Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) + IndexMode() wallet.IndexMode + Tip() (types.ChainIndex, error) + Scan(_ context.Context, index types.ChainIndex) error + + AddWallet(wallet.Wallet) (wallet.Wallet, error) + UpdateWallet(wallet.Wallet) (wallet.Wallet, error) + DeleteWallet(wallet.ID) error + Wallets() ([]wallet.Wallet, error) + + AddAddress(id wallet.ID, addr wallet.Address) error + RemoveAddress(id wallet.ID, addr types.Address) error + Addresses(id wallet.ID) ([]wallet.Address, error) + WalletAddress(wallet.ID, types.Address) (wallet.Address, error) + WalletEvents(id wallet.ID, offset, limit int) ([]wallet.Event, error) + WalletUnconfirmedEvents(id wallet.ID) ([]wallet.Event, error) + SelectSiacoinElements(walletID wallet.ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) + SelectSiafundElements(walletID wallet.ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) + UnspentSiacoinOutputs(id wallet.ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + UnspentSiafundOutputs(id wallet.ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + WalletBalance(id wallet.ID) (wallet.Balance, error) + + AddressBalance(address types.Address) (wallet.Balance, error) + AddressEvents(address types.Address, offset, limit int) ([]wallet.Event, error) + AddressUnconfirmedEvents(address types.Address) ([]wallet.Event, error) + AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + + Events(eventIDs []types.Hash256) ([]wallet.Event, error) + + SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) + SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + // SiacoinElementSpentEvent returns the event of a spent siacoin element. + // If the element is not spent, the return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiacoinElementSpentEvent(types.SiacoinOutputID) (wallet.Event, bool, error) + // SiafundElementSpentEvent returns the event of a spent siafund element. + // If the element is not spent, the second return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiafundElementSpentEvent(types.SiafundOutputID) (wallet.Event, bool, error) + + Reserve([]types.Hash256) error + Release([]types.Hash256) + } + + // A SigningKeyManager manages ed25519 signing keys. + SigningKeyManager interface { + Add(types.PrivateKey) error + Delete(types.PublicKey) error + Sign(types.PublicKey, types.Hash256) (types.Signature, error) } ) type server struct { - cm ChainManager - s Syncer - wm WalletManager + startTime time.Time + debugEnabled bool + publicEndpoints bool + password string + + log *zap.Logger + cm ChainManager + s Syncer + wm WalletManager + km SigningKeyManager + + scanMu sync.Mutex // for resubscribe + scanInProgress bool + scanInfo RescanResponse +} - // for walletsReserveHandler - mu sync.Mutex - used map[types.Hash256]bool +func (s *server) stateHandler(jc jape.Context) { + jc.Encode(StateResponse{ + Version: build.Version(), + Commit: build.Commit(), + OS: runtime.GOOS, + BuildTime: build.Time(), + StartTime: s.startTime, + IndexMode: s.wm.IndexMode(), + }) } func (s *server) consensusNetworkHandler(jc jape.Context) { @@ -82,23 +190,146 @@ func (s *server) consensusTipStateHandler(jc jape.Context) { jc.Encode(s.cm.TipState()) } +func (s *server) consensusBlocksIDHandler(jc jape.Context) { + var bid types.BlockID + if jc.DecodeParam("id", &bid) != nil { + return + } + block, found := s.cm.Block(bid) + if !found { + jc.Error(errors.New("couldn't find block"), http.StatusNotFound) + return + } + jc.Encode(block) +} + +func (s *server) consensusIndexHeightHandler(jc jape.Context) { + var height uint64 + if jc.DecodeParam("height", &height) != nil { + return + } + index, ok := s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + jc.Encode(index) +} + +func (s *server) consensusUpdatesIndexHandler(jc jape.Context) { + var index types.ChainIndex + if jc.DecodeParam("index", &index) != nil { + return + } + + limit := 10 + if jc.DecodeForm("limit", &limit) != nil { + return + } else if limit <= 0 || limit > 100 { + jc.Error(errors.New("limit must be between 0 and 100"), http.StatusBadRequest) + return + } + + reverted, applied, err := s.cm.UpdatesSince(index, limit) + if jc.Check("couldn't get updates", err) != nil { + return + } + + var res ConsensusUpdatesResponse + for _, ru := range reverted { + res.Reverted = append(res.Reverted, RevertUpdate{ + Update: ru.RevertUpdate, + State: ru.State, + Block: ru.Block, + }) + } + for _, au := range applied { + res.Applied = append(res.Applied, ApplyUpdate{ + Update: au.ApplyUpdate, + State: au.State, + Block: au.Block, + }) + } + jc.Encode(res) +} + +func (s *server) miningGetBlockTemplateHandler(jc jape.Context) { + var req MiningGetBlockTemplateRequest + if jc.Decode(&req) != nil { + return + } else if req.PayoutAddress == types.VoidAddress { + jc.Error(errors.New("payout address can't be empty"), http.StatusBadRequest) + return + } + + // TODO: add polling + + template, err := generateBlockTemplate(s.cm, req.PayoutAddress) + if jc.Check("failed to generate block template", err) != nil { + return + } + jc.Encode(template) +} + +func (s *server) miningSubmitBlockTemplateHandler(jc jape.Context) { + var req MiningSubmitBlockRequest + if jc.Decode(&req) != nil { + return + } else if len(req.Params) < 1 { + jc.Error(errors.New("expected block hex in request params array"), http.StatusBadRequest) + return + } + rawBlock, err := hex.DecodeString(req.Params[0]) + if jc.Check("couldn't decode block hex", err) != nil { + return + } + + // decode block + var block types.Block + isV2 := s.cm.Tip().Height >= s.cm.TipState().Network.HardforkV2.AllowHeight + dec := types.NewBufDecoder(rawBlock) + if !isV2 { + (*types.V1Block)(&block).DecodeFrom(dec) + } else { + (*types.V2Block)(&block).DecodeFrom(dec) + } + if jc.Check("couldn't decode block", dec.Err()) != nil { + return + } + + // verify and broadcast block + if jc.Check("failed to add block to chain manager", s.cm.AddBlocks([]types.Block{block})) != nil { + return + } + if !isV2 { + s.s.BroadcastHeader(block.Header()) + } else { + s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(block, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + } + jc.EmptyResonse() +} + func (s *server) syncerPeersHandler(jc jape.Context) { var peers []GatewayPeer for _, p := range s.s.Peers() { - info, ok := s.s.PeerInfo(p.Addr) - if !ok { - continue - } - peers = append(peers, GatewayPeer{ - Addr: p.Addr, + // create peer response with known fields + peer := GatewayPeer{ + Address: p.Addr(), Inbound: p.Inbound, - Version: p.Version, - - FirstSeen: info.FirstSeen, - ConnectedSince: info.LastConnect, - SyncedBlocks: info.SyncedBlocks, - SyncDuration: info.SyncDuration, - }) + Version: p.Version(), + } + // add more info if available + info, err := s.s.PeerInfo(p.Addr()) + if err != nil && !errors.Is(err, syncer.ErrPeerNotFound) { + jc.Error(err, http.StatusInternalServerError) + return + } else if err == nil { + peer.FirstSeen = info.FirstSeen + peer.ConnectedSince = info.LastConnect + peer.SyncedBlocks = info.SyncedBlocks + peer.SyncDuration = info.SyncDuration + } + peers = append(peers, peer) } jc.Encode(peers) } @@ -108,8 +339,11 @@ func (s *server) syncerConnectHandler(jc jape.Context) { if jc.Decode(&addr) != nil { return } - _, err := s.s.Connect(addr) - jc.Check("couldn't connect to peer", err) + _, err := s.s.Connect(jc.Request.Context(), addr) + if jc.Check("couldn't connect to peer", err) != nil { + return + } + jc.EmptyResonse() } func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { @@ -120,19 +354,25 @@ func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { return } if b.V2 == nil { - s.s.BroadcastHeader(gateway.BlockHeader{ - ParentID: b.ParentID, - Nonce: b.Nonce, - Timestamp: b.Timestamp, - MerkleRoot: b.MerkleRoot(), - }) + s.s.BroadcastHeader(b.Header()) } else { s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) } + jc.EmptyResonse() +} + +func (s *server) txpoolParentsHandler(jc jape.Context) { + var txn types.Transaction + if jc.Decode(&txn) != nil { + return + } + + jc.Encode(s.cm.UnconfirmedParents(txn)) } func (s *server) txpoolTransactionsHandler(jc jape.Context) { jc.Encode(TxpoolTransactionsResponse{ + Basis: s.cm.Tip(), Transactions: s.cm.PoolTransactions(), V2Transactions: s.cm.V2PoolTransactions(), }) @@ -148,79 +388,197 @@ func (s *server) txpoolBroadcastHandler(jc jape.Context) { return } if len(tbr.Transactions) != 0 { - if jc.Check("invalid transaction set", s.cm.AddPoolTransactions(tbr.Transactions)) != nil { + _, err := s.cm.AddPoolTransactions(tbr.Transactions) + if err != nil { + jc.Error(fmt.Errorf("invalid transaction set: %w", err), http.StatusBadRequest) return } s.s.BroadcastTransactionSet(tbr.Transactions) } if len(tbr.V2Transactions) != 0 { - if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(tbr.V2Transactions)) != nil { + _, err := s.cm.AddV2PoolTransactions(tbr.Basis, tbr.V2Transactions) + if err != nil { + jc.Error(fmt.Errorf("invalid v2 transaction set: %w", err), http.StatusBadRequest) return } - s.s.BroadcastV2TransactionSet(tbr.V2Transactions) + s.s.BroadcastV2TransactionSet(tbr.Basis, tbr.V2Transactions) + } + + jc.EmptyResonse() +} + +func (s *server) txpoolV2TransactionsBasisHandler(jc jape.Context) { + var req TxpoolUpdateV2TransactionsRequest + if jc.Decode(&req) != nil { + return } + + txnset, err := s.cm.UpdateV2TransactionSet(req.Transactions, req.Basis, req.Target) + if jc.Check("couldn't update v2 transaction set", err) != nil { + return + } + jc.Encode(TxpoolUpdateV2TransactionsResponse{ + Basis: req.Target, + Transactions: txnset, + }) } func (s *server) walletsHandler(jc jape.Context) { - jc.Encode(s.wm.Wallets()) + wallets, err := s.wm.Wallets() + if jc.Check("couldn't load wallets", err) != nil { + return + } + jc.Encode(wallets) +} + +func (s *server) walletsHandlerPOST(jc jape.Context) { + var req WalletUpdateRequest + if jc.Decode(&req) != nil { + return + } + w := wallet.Wallet{ + Name: req.Name, + Description: req.Description, + Metadata: req.Metadata, + } + + w, err := s.wm.AddWallet(w) + if jc.Check("couldn't add wallet", err) != nil { + return + } + jc.Encode(w) } -func (s *server) walletsNameHandlerPUT(jc jape.Context) { - var name string - var info json.RawMessage - if jc.DecodeParam("name", &name) != nil || jc.Decode(&info) != nil { +func (s *server) walletsIDHandlerPOST(jc jape.Context) { + var id wallet.ID + var req WalletUpdateRequest + if jc.DecodeParam("id", &id) != nil || jc.Decode(&req) != nil { + return + } + w := wallet.Wallet{ + ID: id, + Name: req.Name, + Description: req.Description, + Metadata: req.Metadata, + } + + w, err := s.wm.UpdateWallet(w) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) return - } else if jc.Check("couldn't add wallet", s.wm.AddWallet(name, info)) != nil { + } else if jc.Check("couldn't update wallet", err) != nil { return } + jc.Encode(w) } -func (s *server) walletsNameHandlerDELETE(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { +func (s *server) walletsIDHandlerDELETE(jc jape.Context) { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { + return + } + err := s.wm.DeleteWallet(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't remove wallet", err) != nil { return - } else if jc.Check("couldn't remove wallet", s.wm.DeleteWallet(name)) != nil { + } + jc.EmptyResonse() +} + +func (s *server) rescanHandlerGET(jc jape.Context) { + index, err := s.wm.Tip() + if jc.Check("couldn't get tip", err) != nil { return } + + s.scanMu.Lock() + defer s.scanMu.Unlock() + s.scanInfo.Index = index + jc.Encode(s.scanInfo) } -func (s *server) walletsSubscribeHandler(jc jape.Context) { - var name string +func (s *server) rescanHandlerPOST(jc jape.Context) { var height uint64 - if jc.DecodeParam("name", &name) != nil || jc.Decode(&height) != nil { + if jc.Decode(&height) != nil { return - } else if jc.Check("couldn't subscribe wallet", s.wm.SubscribeWallet(name, height)) != nil { + } + + s.scanMu.Lock() + defer s.scanMu.Unlock() + + if s.scanInProgress { + jc.Error(errors.New("scan already in progress"), http.StatusConflict) return } + + var index types.ChainIndex + if height > 0 { + var ok bool + index, ok = s.cm.BestIndex(height) + if !ok { + jc.Error(errors.New("height not found"), http.StatusNotFound) + return + } + } + + s.scanInProgress = true + s.scanInfo = RescanResponse{ + StartIndex: index, + Index: index, + StartTime: time.Now(), + Error: nil, + } + + go func() { + err := s.wm.Scan(context.Background(), index) + + // update the scan state + s.scanMu.Lock() + defer s.scanMu.Unlock() + s.scanInProgress = false + if err != nil { + msg := err.Error() + s.scanInfo.Error = &msg + } + }() + + jc.EmptyResonse() } func (s *server) walletsAddressHandlerPUT(jc jape.Context) { - var name string - var addr types.Address - var info json.RawMessage - if jc.DecodeParam("name", &name) != nil || jc.DecodeParam("addr", &addr) != nil || jc.Decode(&info) != nil { + var id wallet.ID + var addr wallet.Address + if jc.DecodeParam("id", &id) != nil || jc.Decode(&addr) != nil { return - } else if jc.Check("couldn't add address", s.wm.AddAddress(name, addr, info)) != nil { + } else if jc.Check("couldn't add address", s.wm.AddAddress(id, addr)) != nil { return } + jc.EmptyResonse() } func (s *server) walletsAddressHandlerDELETE(jc jape.Context) { - var name string + var id wallet.ID var addr types.Address - if jc.DecodeParam("name", &name) != nil || jc.DecodeParam("addr", &addr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.DecodeParam("addr", &addr) != nil { return - } else if jc.Check("couldn't remove address", s.wm.RemoveAddress(name, addr)) != nil { + } + + err := s.wm.RemoveAddress(id, addr) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + } else if jc.Check("couldn't remove address", err) != nil { return } + jc.EmptyResonse() } func (s *server) walletsAddressesHandlerGET(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { return } - addrs, err := s.wm.Addresses(name) + addrs, err := s.wm.Addresses(id) if jc.Check("couldn't load addresses", err) != nil { return } @@ -228,192 +586,218 @@ func (s *server) walletsAddressesHandlerGET(jc jape.Context) { } func (s *server) walletsBalanceHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { return } - scos, sfos, err := s.wm.UnspentOutputs(name) - if jc.Check("couldn't load outputs", err) != nil { + + b, err := s.wm.WalletBalance(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load balance", err) != nil { return } - var sc types.Currency - var sf uint64 - for _, sco := range scos { - sc = sc.Add(sco.SiacoinOutput.Value) + jc.Encode(BalanceResponse(b)) +} + +func (s *server) walletsEventsHandler(jc jape.Context) { + var id wallet.ID + offset, limit := 0, 500 + if jc.DecodeParam("id", &id) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return } - for _, sfo := range sfos { - sf += sfo.SiafundOutput.Value + events, err := s.wm.WalletEvents(id, offset, limit) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load events", err) != nil { + return } - jc.Encode(WalletBalanceResponse{ - Siacoins: sc, - Siafunds: sf, - }) + jc.Encode(events) } -func (s *server) walletsEventsHandler(jc jape.Context) { - var name string - offset, limit := 0, -1 - if jc.DecodeParam("name", &name) != nil || jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { +func (s *server) walletsEventsUnconfirmedHandlerGET(jc jape.Context) { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { return } - events, err := s.wm.Events(name, offset, limit) - if jc.Check("couldn't load events", err) != nil { + + events, err := s.wm.WalletUnconfirmedEvents(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if err != nil { + jc.Error(err, http.StatusInternalServerError) return } jc.Encode(events) } -func (s *server) walletsTxpoolHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { +func (s *server) walletsOutputsSiacoinHandler(jc jape.Context) { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - pool, err := s.wm.Annotate(name, s.cm.PoolTransactions()) - if jc.Check("couldn't annotate pool", err) != nil { + + scos, basis, err := s.wm.UnspentSiacoinOutputs(id, offset, limit) + if jc.Check("couldn't load siacoin outputs", err) != nil { return } - jc.Encode(pool) + + jc.Encode(SiacoinElementsResponse{ + Basis: basis, + Outputs: scos, + }) } -func (s *server) walletsOutputsHandler(jc jape.Context) { - var name string - if jc.DecodeParam("name", &name) != nil { +func (s *server) walletsOutputsSiafundHandler(jc jape.Context) { + var id wallet.ID + if jc.DecodeParam("id", &id) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { return } - scos, sfos, err := s.wm.UnspentOutputs(name) - if jc.Check("couldn't load outputs", err) != nil { + + sfos, basis, err := s.wm.UnspentSiafundOutputs(id, offset, limit) + if jc.Check("couldn't load siacoin outputs", err) != nil { return } - jc.Encode(WalletOutputsResponse{ - SiacoinOutputs: scos, - SiafundOutputs: sfos, + jc.Encode(SiafundElementsResponse{ + Basis: basis, + Outputs: sfos, }) } +func (s *server) outputsSiacoinSpentHandlerGET(jc jape.Context) { + var id types.SiacoinOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + event, spent, err := s.wm.SiacoinElementSpentEvent(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load siacoin element", err) != nil { + return + } + + resp := ElementSpentResponse{ + Spent: spent, + } + if spent { + resp.Event = &event + } + + jc.Encode(resp) +} + +func (s *server) outputsSiafundSpentHandlerGET(jc jape.Context) { + var id types.SiafundOutputID + if jc.DecodeParam("id", &id) != nil { + return + } + + event, spent, err := s.wm.SiafundElementSpentEvent(id) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("couldn't load siafund element", err) != nil { + return + } + + resp := ElementSpentResponse{ + Spent: spent, + } + if spent { + resp.Event = &event + } + + jc.Encode(resp) +} + func (s *server) walletsReserveHandler(jc jape.Context) { - var name string var wrr WalletReserveRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wrr) != nil { + if jc.Decode(&wrr) != nil { return } - s.mu.Lock() + ids := make([]types.Hash256, 0, len(wrr.SiacoinOutputs)+len(wrr.SiafundOutputs)) for _, id := range wrr.SiacoinOutputs { - if s.used[types.Hash256(id)] { - s.mu.Unlock() - jc.Error(fmt.Errorf("output %v is already reserved", id), http.StatusBadRequest) - return - } - s.used[types.Hash256(id)] = true + ids = append(ids, types.Hash256(id)) } + for _, id := range wrr.SiafundOutputs { - if s.used[types.Hash256(id)] { - s.mu.Unlock() - jc.Error(fmt.Errorf("output %v is already reserved", id), http.StatusBadRequest) - return - } - s.used[types.Hash256(id)] = true + ids = append(ids, types.Hash256(id)) } - s.mu.Unlock() - if wrr.Duration == 0 { - wrr.Duration = 10 * time.Minute + if jc.Check("couldn't reserve outputs", s.wm.Reserve(ids)) != nil { + return } - time.AfterFunc(wrr.Duration, func() { - s.mu.Lock() - defer s.mu.Unlock() - for _, id := range wrr.SiacoinOutputs { - delete(s.used, types.Hash256(id)) - } - for _, id := range wrr.SiafundOutputs { - delete(s.used, types.Hash256(id)) - } - }) + jc.EmptyResonse() } func (s *server) walletsReleaseHandler(jc jape.Context) { - var name string var wrr WalletReleaseRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wrr) != nil { + if jc.Decode(&wrr) != nil { return } - s.mu.Lock() - defer s.mu.Unlock() + + ids := make([]types.Hash256, 0, len(wrr.SiacoinOutputs)+len(wrr.SiafundOutputs)) for _, id := range wrr.SiacoinOutputs { - delete(s.used, types.Hash256(id)) + ids = append(ids, types.Hash256(id)) } for _, id := range wrr.SiafundOutputs { - delete(s.used, types.Hash256(id)) + ids = append(ids, types.Hash256(id)) } + s.wm.Release(ids) + jc.EmptyResonse() } func (s *server) walletsFundHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount types.Currency, utxos []types.SiacoinElement, changeAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { - s.mu.Lock() - defer s.mu.Unlock() - if amount.IsZero() { - return nil, nil - } - inPool := make(map[types.Hash256]bool) - for _, ptxn := range pool { - for _, in := range ptxn.SiacoinInputs { - inPool[types.Hash256(in.ParentID)] = true - } - } - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) - var outputSum types.Currency - var fundingElements []types.SiacoinElement - for _, sce := range utxos { - if s.used[types.Hash256(sce.ID)] || inPool[types.Hash256(sce.ID)] { - continue - } - fundingElements = append(fundingElements, sce) - outputSum = outputSum.Add(sce.SiacoinOutput.Value) - if outputSum.Cmp(amount) >= 0 { - break - } - } - if outputSum.Cmp(amount) < 0 { - return nil, errors.New("insufficient balance") - } else if outputSum.Cmp(amount) > 0 { - if changeAddr == types.VoidAddress { - return nil, errors.New("change address must be specified") - } - txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ - Value: outputSum.Sub(amount), - Address: changeAddr, - }) - } - - toSign := make([]types.Hash256, len(fundingElements)) - for i, sce := range fundingElements { - txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ - ParentID: types.SiacoinOutputID(sce.ID), - // UnlockConditions left empty for client to fill in - }) - toSign[i] = types.Hash256(sce.ID) - s.used[types.Hash256(sce.ID)] = true - } - - return toSign, nil - } - - var name string + var id wallet.ID var wfr WalletFundRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - utxos, _, err := s.wm.UnspentOutputs(name) + utxos, basis, change, err := s.wm.SelectSiacoinElements(id, wfr.Amount, false) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } txn := wfr.Transaction - toSign, err := fundTxn(&txn, wfr.Amount, utxos, wfr.ChangeAddress, s.cm.PoolTransactions()) - if jc.Check("couldn't fund transaction", err) != nil { - return + if !change.IsZero() { + if wfr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ + Value: change, + Address: wfr.ChangeAddress, + }) } + + toSign := make([]types.Hash256, 0, len(utxos)) + for _, sce := range utxos { + txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ + ParentID: sce.ID, + // UnlockConditions left empty for client to fill in + }) + toSign = append(toSign, types.Hash256(sce.ID)) + } + jc.Encode(WalletFundResponse{ + Basis: basis, Transaction: txn, ToSign: toSign, DependsOn: s.cm.UnconfirmedParents(txn), @@ -421,114 +805,772 @@ func (s *server) walletsFundHandler(jc jape.Context) { } func (s *server) walletsFundSFHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount uint64, utxos []types.SiafundElement, changeAddr, claimAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { - s.mu.Lock() - defer s.mu.Unlock() - if amount == 0 { - return nil, nil - } - inPool := make(map[types.Hash256]bool) - for _, ptxn := range pool { - for _, in := range ptxn.SiafundInputs { - inPool[types.Hash256(in.ParentID)] = true - } - } - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) - var outputSum uint64 - var fundingElements []types.SiafundElement - for _, sfe := range utxos { - if s.used[types.Hash256(sfe.ID)] || inPool[types.Hash256(sfe.ID)] { - continue - } - fundingElements = append(fundingElements, sfe) - outputSum += sfe.SiafundOutput.Value - if outputSum >= amount { - break - } - } - if outputSum < amount { - return nil, errors.New("insufficient balance") - } else if outputSum > amount { - if changeAddr == types.VoidAddress { - return nil, errors.New("change address must be specified") - } - txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ - Value: outputSum - amount, - Address: changeAddr, - }) - } - - toSign := make([]types.Hash256, len(fundingElements)) - for i, sfe := range fundingElements { - txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ - ParentID: types.SiafundOutputID(sfe.ID), - ClaimAddress: claimAddr, - // UnlockConditions left empty for client to fill in - }) - toSign[i] = types.Hash256(sfe.ID) - s.used[types.Hash256(sfe.ID)] = true - } - - return toSign, nil - } - - var name string + var id wallet.ID var wfr WalletFundSFRequest - if jc.DecodeParam("name", &name) != nil || jc.Decode(&wfr) != nil { + if jc.DecodeParam("id", &id) != nil || jc.Decode(&wfr) != nil { return } - _, utxos, err := s.wm.UnspentOutputs(name) + utxos, basis, change, err := s.wm.SelectSiafundElements(id, wfr.Amount) if jc.Check("couldn't get utxos to fund transaction", err) != nil { return } txn := wfr.Transaction - toSign, err := fundTxn(&txn, wfr.Amount, utxos, wfr.ChangeAddress, wfr.ClaimAddress, s.cm.PoolTransactions()) - if jc.Check("couldn't fund transaction", err) != nil { - return + if change > 0 { + if wfr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + txn.SiafundOutputs = append(txn.SiafundOutputs, types.SiafundOutput{ + Value: change, + Address: wfr.ChangeAddress, + }) + } + + toSign := make([]types.Hash256, 0, len(utxos)) + for _, sce := range utxos { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sce.ID, + ClaimAddress: wfr.ChangeAddress, + // UnlockConditions left empty for client to fill in + }) + toSign = append(toSign, types.Hash256(sce.ID)) } jc.Encode(WalletFundResponse{ + Basis: basis, Transaction: txn, ToSign: toSign, DependsOn: s.cm.UnconfirmedParents(txn), }) } -// NewServer returns an HTTP handler that serves the walletd API. -func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { - srv := server{ - cm: cm, - s: s, - wm: wm, - used: make(map[types.Hash256]bool), - } - return jape.Mux(map[string]jape.Handler{ - "GET /consensus/network": srv.consensusNetworkHandler, - "GET /consensus/tip": srv.consensusTipHandler, - "GET /consensus/tipstate": srv.consensusTipStateHandler, - - "GET /syncer/peers": srv.syncerPeersHandler, - "POST /syncer/connect": srv.syncerConnectHandler, - "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, - - "GET /txpool/transactions": srv.txpoolTransactionsHandler, - "GET /txpool/fee": srv.txpoolFeeHandler, - "POST /txpool/broadcast": srv.txpoolBroadcastHandler, - - "GET /wallets": srv.walletsHandler, - "PUT /wallets/:name": srv.walletsNameHandlerPUT, - "DELETE /wallets/:name": srv.walletsNameHandlerDELETE, - "POST /wallets/:name/subscribe": srv.walletsSubscribeHandler, - "PUT /wallets/:name/addresses/:addr": srv.walletsAddressHandlerPUT, - "DELETE /wallets/:name/addresses/:addr": srv.walletsAddressHandlerDELETE, - "GET /wallets/:name/addresses": srv.walletsAddressesHandlerGET, - "GET /wallets/:name/balance": srv.walletsBalanceHandler, - "GET /wallets/:name/events": srv.walletsEventsHandler, - "GET /wallets/:name/txpool": srv.walletsTxpoolHandler, - "GET /wallets/:name/outputs": srv.walletsOutputsHandler, - "POST /wallets/:name/reserve": srv.walletsReserveHandler, - "POST /wallets/:name/release": srv.walletsReleaseHandler, - "POST /wallets/:name/fund": srv.walletsFundHandler, - "POST /wallets/:name/fundsf": srv.walletsFundSFHandler, - }) +func (s *server) walletsConstructHandler(jc jape.Context) { + cs := s.cm.TipState() + if cs.Index.Height >= cs.Network.HardforkV2.RequireHeight { + jc.Error(errors.New("v1 transactions are not allowed after the v2 require height"), http.StatusBadRequest) + } + + var walletID wallet.ID + if err := jc.DecodeParam("id", &walletID); err != nil { + return + } + + _, err := s.wm.WalletBalance(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to get wallet", err) != nil { + return + } + + var wcr WalletConstructRequest + if err := jc.Decode(&wcr); err != nil { + return + } + + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + var siacoinInput types.Currency + for i, sco := range wcr.Siacoins { + switch { + case sco.Value.IsZero(): + jc.Error(fmt.Errorf("siacoin output %d has zero value", i), http.StatusBadRequest) + return + case sco.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siacoin output %d has void address", i), http.StatusBadRequest) + return + } + siacoinInput = siacoinInput.Add(sco.Value) + } + + var siafundInput uint64 + for i, sfo := range wcr.Siafunds { + switch { + case sfo.Value == 0: + jc.Error(fmt.Errorf("siafund output %d has zero value", i), http.StatusBadRequest) + return + case sfo.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siafund output %d has void address", i), http.StatusBadRequest) + return + } + siafundInput += sfo.Value + } + + if siacoinInput.IsZero() && siafundInput == 0 { + jc.Error(errors.New("no inputs provided"), http.StatusBadRequest) + } + + fee := s.cm.RecommendedFee().Mul64(2000) // use a const for simplicity + + var sent bool + var locked []types.Hash256 + defer func() { + if sent { + return + } + s.wm.Release(locked) + }() + + sces, basis, siacoinChange, err := s.wm.SelectSiacoinElements(walletID, siacoinInput.Add(fee), false) + if err != nil { + jc.Error(fmt.Errorf("failed to select siacoin elements: %w", err), http.StatusInternalServerError) + return + } + for _, sce := range sces { + locked = append(locked, types.Hash256(sce.ID)) + } + + if !siacoinChange.IsZero() { + wcr.Siacoins = append(wcr.Siacoins, types.SiacoinOutput{ + Value: siacoinChange, + Address: wcr.ChangeAddress, + }) + } + + sfes, _, siafundChange, err := s.wm.SelectSiafundElements(walletID, siafundInput) + if err != nil { + jc.Error(fmt.Errorf("failed to select siafund elements: %w", err), http.StatusInternalServerError) + return + } + for _, sfe := range sfes { + locked = append(locked, types.Hash256(sfe.ID)) + } + + if siafundChange > 0 { + wcr.Siafunds = append(wcr.Siafunds, types.SiafundOutput{ + Value: siafundChange, + Address: wcr.ChangeAddress, + }) + } + + knownAddresses := make(map[types.Address]types.UnlockConditions) + getAddressUnlockConditions := func(jc jape.Context, addr types.Address) (types.UnlockConditions, bool) { + if a, ok := knownAddresses[addr]; ok { + return a, true + } + a, err := s.wm.WalletAddress(walletID, addr) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return types.UnlockConditions{}, false + } else if a.SpendPolicy == nil { + jc.Error(fmt.Errorf("address %q has no spend policy", addr), http.StatusBadRequest) + return types.UnlockConditions{}, false + } + uc, ok := a.SpendPolicy.Type.(types.PolicyTypeUnlockConditions) + if !ok { + jc.Error(fmt.Errorf("address %q has v2-only spend policy", addr), http.StatusBadRequest) + return types.UnlockConditions{}, false + } + knownAddresses[addr] = types.UnlockConditions(uc) + return knownAddresses[addr], true + } + + resp := WalletConstructResponse{ + Basis: basis, + EstimatedFee: fee, + } + + txn := types.Transaction{ + MinerFees: []types.Currency{fee}, + SiacoinInputs: make([]types.SiacoinInput, 0, len(sces)), + SiacoinOutputs: wcr.Siacoins, + SiafundInputs: make([]types.SiafundInput, 0, len(sfes)), + SiafundOutputs: wcr.Siafunds, + } + + for _, sce := range sces { + uc, ok := getAddressUnlockConditions(jc, sce.SiacoinOutput.Address) + if !ok { + return + } + + sci := types.SiacoinInput{ + ParentID: sce.ID, + UnlockConditions: uc, + } + + txn.SiacoinInputs = append(txn.SiacoinInputs, sci) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sce.ID), + CoveredFields: types.CoveredFields{ + WholeTransaction: true, + }, + }) + } + + for _, sfe := range sfes { + uc, ok := getAddressUnlockConditions(jc, sfe.SiafundOutput.Address) + if !ok { + return + } + + sfi := types.SiafundInput{ + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: wcr.ChangeAddress, + } + txn.SiafundInputs = append(txn.SiafundInputs, sfi) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{ + WholeTransaction: true, + }, + }) + } + + resp.ID = txn.ID() + resp.Transaction = txn + sent = true // locks are released in defer + jc.Encode(resp) +} + +func (s *server) walletsConstructV2Handler(jc jape.Context) { + cs := s.cm.TipState() + if cs.Index.Height < cs.Network.HardforkV2.AllowHeight { + jc.Error(errors.New("v2 transactions are not allowed before the v2 allow height"), http.StatusBadRequest) + } + + var walletID wallet.ID + if err := jc.DecodeParam("id", &walletID); err != nil { + return + } + + _, err := s.wm.WalletBalance(walletID) + if errors.Is(err, wallet.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to get wallet", err) != nil { + return + } + + var wcr WalletConstructRequest + if err := jc.Decode(&wcr); err != nil { + return + } + + if wcr.ChangeAddress == types.VoidAddress { + jc.Error(errors.New("change address must be specified"), http.StatusBadRequest) + return + } + + var siacoinInput types.Currency + for i, sco := range wcr.Siacoins { + switch { + case sco.Value.IsZero(): + jc.Error(fmt.Errorf("siacoin output %d has zero value", i), http.StatusBadRequest) + return + case sco.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siacoin output %d has void address", i), http.StatusBadRequest) + return + } + siacoinInput = siacoinInput.Add(sco.Value) + } + + var siafundInput uint64 + for i, sfo := range wcr.Siafunds { + switch { + case sfo.Value == 0: + jc.Error(fmt.Errorf("siafund output %d has zero value", i), http.StatusBadRequest) + return + case sfo.Address == types.VoidAddress: + jc.Error(fmt.Errorf("siafund output %d has void address", i), http.StatusBadRequest) + return + } + siafundInput += sfo.Value + } + + if siacoinInput.IsZero() && siafundInput == 0 { + jc.Error(errors.New("no inputs provided"), http.StatusBadRequest) + } + + fee := s.cm.RecommendedFee().Mul64(2000) // use a const for simplicity + + var sent bool + var locked []types.Hash256 + defer func() { + if sent { + return + } + s.wm.Release(locked) + }() + + sces, basis, siacoinChange, err := s.wm.SelectSiacoinElements(walletID, siacoinInput.Add(fee), false) + if err != nil { + jc.Error(fmt.Errorf("failed to select siacoin elements: %w", err), http.StatusInternalServerError) + return + } + for _, sce := range sces { + locked = append(locked, types.Hash256(sce.ID)) + } + + if !siacoinChange.IsZero() { + wcr.Siacoins = append(wcr.Siacoins, types.SiacoinOutput{ + Value: siacoinChange, + Address: wcr.ChangeAddress, + }) + } + + sfes, sfBasis, siafundChange, err := s.wm.SelectSiafundElements(walletID, siafundInput) + if err != nil { + jc.Error(fmt.Errorf("failed to select siafund elements: %w", err), http.StatusInternalServerError) + return + } + for _, sfe := range sfes { + locked = append(locked, types.Hash256(sfe.ID)) + } + + if siafundChange > 0 { + wcr.Siafunds = append(wcr.Siafunds, types.SiafundOutput{ + Value: siafundChange, + Address: wcr.ChangeAddress, + }) + } + + knownAddresses := make(map[types.Address]types.SpendPolicy) + getAddressSpendPolicy := func(jc jape.Context, addr types.Address) (types.SpendPolicy, bool) { + if a, ok := knownAddresses[addr]; ok { + return a, true + } + a, err := s.wm.WalletAddress(walletID, addr) + if err != nil { + jc.Error(fmt.Errorf("failed to get address: %w", err), http.StatusInternalServerError) + return types.SpendPolicy{}, false + } + + if a.SpendPolicy == nil { + jc.Error(fmt.Errorf("address %q has no spend policy", addr), http.StatusBadRequest) + return types.SpendPolicy{}, false + } + knownAddresses[addr] = *a.SpendPolicy + return knownAddresses[addr], true + } + + resp := WalletConstructV2Response{ + Basis: basis, + EstimatedFee: fee, + } + + txn := types.V2Transaction{ + MinerFee: fee, + SiacoinInputs: make([]types.V2SiacoinInput, 0, len(sces)), + SiacoinOutputs: wcr.Siacoins, + SiafundInputs: make([]types.V2SiafundInput, 0, len(sfes)), + SiafundOutputs: wcr.Siafunds, + } + + // the siafund elements are added to the transaction first because `UpdateV2TransactionSet` takes + // a V2 transaction as an argument. The Siacoin basis is our target because the transaction is + // guaranteed to have a non-zero Siacoin basis while the Siafund basis will be zero when not + // sending Siafunds. + for _, sfe := range sfes { + sp, ok := getAddressSpendPolicy(jc, sfe.SiafundOutput.Address) + if !ok { + return + } + + sfi := types.V2SiafundInput{ + Parent: sfe, + ClaimAddress: wcr.ChangeAddress, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + } + txn.SiafundInputs = append(txn.SiafundInputs, sfi) + } + + if len(sfes) > 0 && basis != sfBasis { + txnset, err := s.cm.UpdateV2TransactionSet([]types.V2Transaction{txn}, sfBasis, basis) + if err != nil { + jc.Error(fmt.Errorf("failed to update transaction set: %w", err), http.StatusInternalServerError) + return + } + txn = txnset[0] + } + + for _, sce := range sces { + sp, ok := getAddressSpendPolicy(jc, sce.SiacoinOutput.Address) + if !ok { + return + } + + sci := types.V2SiacoinInput{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + } + txn.SiacoinInputs = append(txn.SiacoinInputs, sci) + } + + resp.ID = txn.ID() + resp.Transaction = txn + sent = true // locks are released in defer + jc.Encode(resp) +} + +func (s *server) addressesAddrBalanceHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + b, err := s.wm.AddressBalance(addr) + if jc.Check("couldn't load balance", err) != nil { + return + } + jc.Encode(BalanceResponse(b)) +} + +func (s *server) addressesAddrEventsHandlerGET(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + events, err := s.wm.AddressEvents(addr, offset, limit) + if jc.Check("couldn't load events", err) != nil { + return + } + jc.Encode(events) +} + +func (s *server) addressesAddrEventsUnconfirmedHandlerGET(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + events, err := s.wm.AddressUnconfirmedEvents(addr) + if jc.Check("couldn't load events", err) != nil { + return + } + jc.Encode(events) +} + +func (s *server) addressesAddrOutputsSCHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, basis, err := s.wm.AddressSiacoinOutputs(addr, offset, limit) + if jc.Check("couldn't load utxos", err) != nil { + return + } + jc.Encode(SiacoinElementsResponse{ + Basis: basis, + Outputs: utxos, + }) +} + +func (s *server) addressesAddrOutputsSFHandler(jc jape.Context) { + var addr types.Address + if jc.DecodeParam("addr", &addr) != nil { + return + } + + offset, limit := 0, 1000 + if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { + return + } + + utxos, basis, err := s.wm.AddressSiafundOutputs(addr, offset, limit) + if jc.Check("couldn't load utxos", err) != nil { + return + } + jc.Encode(SiafundElementsResponse{ + Basis: basis, + Outputs: utxos, + }) +} + +func (s *server) eventsHandlerGET(jc jape.Context) { + var eventID types.Hash256 + if jc.DecodeParam("id", &eventID) != nil { + return + } + events, err := s.wm.Events([]types.Hash256{eventID}) + if jc.Check("couldn't load events", err) != nil { + return + } else if len(events) == 0 { + jc.Error(errors.New("event not found"), http.StatusNotFound) + return + } + jc.Encode(events[0]) +} + +func (s *server) outputsSiacoinHandlerGET(jc jape.Context) { + var outputID types.SiacoinOutputID + if jc.DecodeParam("id", &outputID) != nil { + return + } + + output, err := s.wm.SiacoinElement(outputID) + if jc.Check("couldn't load output", err) != nil { + return + } + jc.Encode(output) +} + +func (s *server) outputsSiafundHandlerGET(jc jape.Context) { + var outputID types.SiafundOutputID + if jc.DecodeParam("id", &outputID) != nil { + return + } + + output, err := s.wm.SiafundElement(outputID) + if jc.Check("couldn't load output", err) != nil { + return + } + jc.Encode(output) +} + +func (s *server) keysEd25519GenerateHandlerPOST(jc jape.Context) { + sk := types.GeneratePrivateKey() + defer clear(sk) + if jc.Check("failed to add key", s.km.Add(sk)) != nil { + return + } + jc.Encode(AddSigningKeyResponse{ + PublicKey: sk.PublicKey(), + }) +} + +func (s *server) keysEd25519HandlerPUT(jc jape.Context) { + var req AddSigningKeyRequest + defer clear(req.PrivateKey) + if jc.Decode(&req) != nil { + return + } else if jc.Check("failed to add key", s.km.Add(req.PrivateKey)) != nil { + return + } + + jc.Encode(AddSigningKeyResponse{ + PublicKey: req.PrivateKey.PublicKey(), + }) +} + +func (s *server) keysEd25519HandlerDELETE(jc jape.Context) { + var pk types.PublicKey + if jc.DecodeParam("pub", &pk) != nil { + return + } else if jc.Check("failed to remove key", s.km.Delete(pk)) != nil { + return + } + jc.EmptyResonse() +} + +func (s *server) keysEd25519SignHandlerPOST(jc jape.Context) { + var pub types.PublicKey + if jc.DecodeParam("pub", &pub) != nil { + return + } + var req SignHashRequest + if jc.Decode(&req) != nil { + return + } + + sig, err := s.km.Sign(pub, req.Hash) + if errors.Is(err, keys.ErrNotFound) { + jc.Error(err, http.StatusNotFound) + return + } else if jc.Check("failed to sign message", err) != nil { + return + } + jc.Encode(SignHashResponse{ + Signature: sig, + }) +} + +func (s *server) debugMineHandler(jc jape.Context) { + var req DebugMineRequest + if jc.Decode(&req) != nil { + return + } + + log := s.log.Named("miner") + ctx := jc.Request.Context() + + for n := req.Blocks; n > 0; { + b, err := mineBlock(ctx, s.cm, req.Address) + if errors.Is(err, context.Canceled) { + return + } else if err != nil { + log.Warn("failed to mine block", zap.Error(err)) + } else if err := s.cm.AddBlocks([]types.Block{b}); err != nil { + log.Warn("failed to add block", zap.Error(err)) + } + + if b.V2 == nil { + s.s.BroadcastHeader(b.Header()) + } else { + s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + } + + log.Debug("mined block", zap.Stringer("blockID", b.ID())) + n-- + } + jc.EmptyResonse() +} + +func (s *server) pprofHandler(jc jape.Context) { + var handler string + if err := jc.DecodeParam("handler", &handler); err != nil { + return + } + + switch handler { + case "cmdline": + pprof.Cmdline(jc.ResponseWriter, jc.Request) + case "profile": + pprof.Profile(jc.ResponseWriter, jc.Request) + case "symbol": + pprof.Symbol(jc.ResponseWriter, jc.Request) + case "trace": + pprof.Trace(jc.ResponseWriter, jc.Request) + default: + pprof.Index(jc.ResponseWriter, jc.Request) + } + pprof.Index(jc.ResponseWriter, jc.Request) +} + +// NewServer returns an HTTP handler that serves the walletd API. +func NewServer(cm ChainManager, s Syncer, wm WalletManager, opts ...ServerOption) http.Handler { + srv := server{ + log: zap.NewNop(), + debugEnabled: false, + publicEndpoints: false, + startTime: time.Now(), + + cm: cm, + s: s, + wm: wm, + } + for _, opt := range opts { + opt(&srv) + } + + // checkAuth checks the request for basic authentication. + checkAuth := func(jc jape.Context) bool { + if srv.password == "" { + // unset password is equivalent to no auth + return true + } + + // verify auth header + _, pass, ok := jc.Request.BasicAuth() + if ok && pass == srv.password { + return true + } + + jc.Error(errors.New("unauthorized"), http.StatusUnauthorized) + return false + } + + // wrapAuthHandler wraps a jape handler with an authentication check. + wrapAuthHandler := func(h jape.Handler) jape.Handler { + return func(jc jape.Context) { + if !checkAuth(jc) { + return + } + h(jc) + } + } + + // wrapPublicAuthHandler wraps a jape handler with an authentication check + // unless publicEndpoints is true. + wrapPublicAuthHandler := func(h jape.Handler) jape.Handler { + return func(jc jape.Context) { + if !srv.publicEndpoints && !checkAuth(jc) { + return + } + h(jc) + } + } + + handlers := map[string]jape.Handler{ + "GET /state": wrapPublicAuthHandler(srv.stateHandler), + + "GET /consensus/network": wrapPublicAuthHandler(srv.consensusNetworkHandler), + "GET /consensus/tip": wrapPublicAuthHandler(srv.consensusTipHandler), + "GET /consensus/tipstate": wrapPublicAuthHandler(srv.consensusTipStateHandler), + "GET /consensus/blocks/:id": wrapPublicAuthHandler(srv.consensusBlocksIDHandler), + "GET /consensus/updates/:index": wrapPublicAuthHandler(srv.consensusUpdatesIndexHandler), + "GET /consensus/index/:height": wrapPublicAuthHandler(srv.consensusIndexHeightHandler), + + "POST /mining/getblocktemplate": wrapAuthHandler(srv.miningGetBlockTemplateHandler), + "POST /mining/submitblock": wrapAuthHandler(srv.miningSubmitBlockTemplateHandler), + + "POST /syncer/connect": wrapAuthHandler(srv.syncerConnectHandler), + "GET /syncer/peers": wrapPublicAuthHandler(srv.syncerPeersHandler), + "POST /syncer/broadcast/block": wrapPublicAuthHandler(srv.syncerBroadcastBlockHandler), + + "GET /txpool/transactions": wrapPublicAuthHandler(srv.txpoolTransactionsHandler), + "POST /txpool/transactions/v2/basis": wrapPublicAuthHandler(srv.txpoolV2TransactionsBasisHandler), + "GET /txpool/fee": wrapPublicAuthHandler(srv.txpoolFeeHandler), + "POST /txpool/parents": wrapPublicAuthHandler(srv.txpoolParentsHandler), + "POST /txpool/broadcast": wrapPublicAuthHandler(srv.txpoolBroadcastHandler), + + "GET /addresses/:addr/balance": wrapPublicAuthHandler(srv.addressesAddrBalanceHandler), + "GET /addresses/:addr/events": wrapPublicAuthHandler(srv.addressesAddrEventsHandlerGET), + "GET /addresses/:addr/events/unconfirmed": wrapPublicAuthHandler(srv.addressesAddrEventsUnconfirmedHandlerGET), + "GET /addresses/:addr/outputs/siacoin": wrapPublicAuthHandler(srv.addressesAddrOutputsSCHandler), + "GET /addresses/:addr/outputs/siafund": wrapPublicAuthHandler(srv.addressesAddrOutputsSFHandler), + + "GET /outputs/siacoin/:id": wrapPublicAuthHandler(srv.outputsSiacoinHandlerGET), + "GET /outputs/siacoin/:id/spent": wrapPublicAuthHandler(srv.outputsSiacoinSpentHandlerGET), + "GET /outputs/siafund/:id": wrapPublicAuthHandler(srv.outputsSiafundHandlerGET), + "GET /outputs/siafund/:id/spent": wrapPublicAuthHandler(srv.outputsSiafundSpentHandlerGET), + + "GET /events/:id": wrapPublicAuthHandler(srv.eventsHandlerGET), + + "GET /rescan": wrapAuthHandler(srv.rescanHandlerGET), + "POST /rescan": wrapAuthHandler(srv.rescanHandlerPOST), + + "GET /wallets": wrapAuthHandler(srv.walletsHandler), + "POST /wallets": wrapAuthHandler(srv.walletsHandlerPOST), + "POST /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerPOST), + "DELETE /wallets/:id": wrapAuthHandler(srv.walletsIDHandlerDELETE), + "PUT /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressHandlerPUT), + "DELETE /wallets/:id/addresses/:addr": wrapAuthHandler(srv.walletsAddressHandlerDELETE), + "GET /wallets/:id/addresses": wrapAuthHandler(srv.walletsAddressesHandlerGET), + "GET /wallets/:id/balance": wrapAuthHandler(srv.walletsBalanceHandler), + "GET /wallets/:id/events": wrapAuthHandler(srv.walletsEventsHandler), + "POST /wallets/:id/construct/transaction": wrapAuthHandler(srv.walletsConstructHandler), + "POST /wallets/:id/construct/v2/transaction": wrapAuthHandler(srv.walletsConstructV2Handler), + "GET /wallets/:id/events/unconfirmed": wrapAuthHandler(srv.walletsEventsUnconfirmedHandlerGET), + "GET /wallets/:id/outputs/siacoin": wrapAuthHandler(srv.walletsOutputsSiacoinHandler), + "GET /wallets/:id/outputs/siafund": wrapAuthHandler(srv.walletsOutputsSiafundHandler), + "POST /wallets/:id/reserve": wrapAuthHandler(srv.walletsReserveHandler), + "POST /wallets/:id/release": wrapAuthHandler(srv.walletsReleaseHandler), + "POST /wallets/:id/fund": wrapAuthHandler(srv.walletsFundHandler), + "POST /wallets/:id/fundsf": wrapAuthHandler(srv.walletsFundSFHandler), + } + + if srv.km != nil && !srv.publicEndpoints { + // key management endpoints are disabled on public nodes + handlers["POST /keys/generate/ed25519"] = wrapAuthHandler(srv.keysEd25519GenerateHandlerPOST) + handlers["PUT /keys/ed25519"] = wrapAuthHandler(srv.keysEd25519HandlerPUT) + handlers["DELETE /keys/ed25519/:pub"] = wrapAuthHandler(srv.keysEd25519HandlerDELETE) + handlers["POST /keys/ed25519/:pub/sign"] = wrapAuthHandler(srv.keysEd25519SignHandlerPOST) + } + + if srv.debugEnabled { + handlers["POST /debug/mine"] = wrapAuthHandler(srv.debugMineHandler) + handlers["GET /debug/pprof/:handler"] = wrapAuthHandler(srv.pprofHandler) + } + return jape.Mux(handlers) } diff --git a/build/build.go b/build/build.go new file mode 100644 index 0000000..119b32f --- /dev/null +++ b/build/build.go @@ -0,0 +1,21 @@ +// Package build contains build-time information. +package build + +//go:generate go run gen.go + +import "time" + +// Commit returns the commit hash of walletd +func Commit() string { + return commit +} + +// Version returns the version of walletd +func Version() string { + return version +} + +// Time returns the time at which the binary was built. +func Time() time.Time { + return time.Unix(buildTime, 0) +} diff --git a/build/gen.go b/build/gen.go new file mode 100644 index 0000000..3bf7dff --- /dev/null +++ b/build/gen.go @@ -0,0 +1,114 @@ +//go:build ignore + +// This script generates meta.go which contains version info for the walletd binary. It can be run with `go generate`. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "strings" + "text/template" + "time" +) + +const logFormat = `{%n "commit": "%H",%n "shortCommit": "%h",%n "timestamp": "%cD",%n "tag": "%(describe:tags=true)"%n}` + +type ( + gitTime time.Time + + gitMeta struct { + Commit string `json:"commit"` + ShortCommit string `json:"shortCommit"` + Timestamp gitTime `json:"timestamp"` + Tag string `json:"tag"` + } +) + +var buildTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT. +// This file was generated by go generate at {{ .RunTime }}. +package build + +const ( + commit = "{{ .Commit }}" + version = "{{ .Version }}" + buildTime = {{ .UnixTimestamp }} +) +`)) + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (t *gitTime) UnmarshalJSON(buf []byte) error { + timeFormats := []string{ + time.RFC1123Z, + "Mon, 2 Jan 2006 15:04:05 -0700", + "2006-01-02 15:04:05 -0700", + time.UnixDate, + time.ANSIC, + time.RFC3339, + time.RFC1123, + } + + for _, format := range timeFormats { + parsed, err := time.Parse(format, strings.Trim(string(buf), `"`)) + if err == nil { + *t = gitTime(parsed) + return nil + } + } + return errors.New("failed to parse time") +} + +func getGitMeta() (meta gitMeta, _ error) { + cmd := exec.Command("git", "log", "-1", "--pretty=format:"+logFormat+"") + buf, err := cmd.Output() + if err != nil { + if err, ok := err.(*exec.ExitError); ok && len(err.Stderr) > 0 { + return gitMeta{}, fmt.Errorf("command failed: %w", errors.New(string(err.Stderr))) + } + return gitMeta{}, fmt.Errorf("failed to execute command: %w", err) + } else if err := json.Unmarshal(buf, &meta); err != nil { + return gitMeta{}, fmt.Errorf("failed to unmarshal json: %w", err) + } + return +} + +func main() { + meta, err := getGitMeta() + if err != nil { + log.Fatalln(err) + } + + commit := meta.ShortCommit + version := meta.Tag + if len(version) == 0 { + // no version, use commit and current time for development + version = commit + meta.Timestamp = gitTime(time.Now()) + } + + f, err := os.Create("meta.go") + if err != nil { + log.Fatalln(err) + } + defer f.Close() + + err = buildTemplate.Execute(f, struct { + Commit string + Version string + UnixTimestamp int64 + + RunTime string + }{ + Commit: commit, + Version: version, + UnixTimestamp: time.Time(meta.Timestamp).Unix(), + + RunTime: time.Now().Format(time.RFC3339), + }) + if err != nil { + log.Fatalln(err) + } +} diff --git a/build/meta.go b/build/meta.go new file mode 100644 index 0000000..05dd9fc --- /dev/null +++ b/build/meta.go @@ -0,0 +1,7 @@ +package build + +const ( + commit = "?" + version = "?" + buildTime = 0 +) diff --git a/cmd/walletd/config.go b/cmd/walletd/config.go new file mode 100644 index 0000000..353998f --- /dev/null +++ b/cmd/walletd/config.go @@ -0,0 +1,271 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + + "go.sia.tech/walletd/wallet" + "golang.org/x/term" + "gopkg.in/yaml.v3" +) + +// readPasswordInput reads a password from stdin. +func readPasswordInput(context string) string { + fmt.Printf("%s: ", context) + input, err := term.ReadPassword(int(os.Stdin.Fd())) + checkFatalError("failed to read password input", err) + fmt.Println("") + return string(input) +} + +func readInput(context string) string { + fmt.Printf("%s: ", context) + r := bufio.NewReader(os.Stdin) + input, err := r.ReadString('\n') + checkFatalError("failed to read input", err) + return strings.TrimSpace(input) +} + +// wrapANSI wraps the output in ANSI escape codes if enabled. +func wrapANSI(prefix, output, suffix string) string { + if cfg.Log.StdOut.EnableANSI { + return prefix + output + suffix + } + return output +} + +func humanList(s []string, sep string) string { + if len(s) == 0 { + return "" + } else if len(s) == 1 { + return fmt.Sprintf(`%q`, s[0]) + } else if len(s) == 2 { + return fmt.Sprintf(`%q %s %q`, s[0], sep, s[1]) + } + + var sb strings.Builder + for i, v := range s { + if i != 0 { + sb.WriteString(", ") + } + if i == len(s)-1 { + sb.WriteString("or ") + } + sb.WriteString(`"`) + sb.WriteString(v) + sb.WriteString(`"`) + } + return sb.String() +} + +func promptQuestion(question string, answers []string) string { + for { + input := readInput(fmt.Sprintf("%s (%s)", question, strings.Join(answers, "/"))) + for _, answer := range answers { + if strings.EqualFold(input, answer) { + return answer + } + } + fmt.Println(wrapANSI("\033[31m", fmt.Sprintf("Answer must be %s", humanList(answers, "or")), "\033[0m")) + } +} + +func promptYesNo(question string) bool { + answer := promptQuestion(question, []string{"yes", "no"}) + return strings.EqualFold(answer, "yes") +} + +// stdoutError prints an error message to stdout +func stdoutError(msg string) { + if cfg.Log.StdOut.EnableANSI { + fmt.Println(wrapANSI("\033[31m", msg, "\033[0m")) + } else { + fmt.Println(msg) + } +} + +func setAPIPassword() { + // retry until a valid API password is entered + for { + fmt.Println("Please choose a password to unlock walletd.") + fmt.Println("This password will be required to access the admin UI in your web browser.") + fmt.Println("(The password must be at least 4 characters.)") + cfg.HTTP.Password = readPasswordInput("Enter password") + if len(cfg.HTTP.Password) >= 4 { + break + } + + fmt.Println(wrapANSI("\033[31m", "Password must be at least 4 characters!", "\033[0m")) + fmt.Println("") + } +} + +func setDataDirectory() { + if cfg.Directory == "" { + cfg.Directory = "." + } + + dir, err := filepath.Abs(cfg.Directory) + checkFatalError("failed to get absolute path of data directory", err) + + fmt.Println("The data directory is where walletd will store its metadata and consensus data.") + fmt.Println("This directory should be on a fast, reliable storage device, preferably an SSD.") + fmt.Println("") + + _, existsErr := os.Stat(filepath.Join(cfg.Directory, "walletd.sqlite3")) + dataExists := existsErr == nil + if dataExists { + fmt.Println(wrapANSI("\033[33m", "There is existing data in the data directory.", "\033[0m")) + fmt.Println(wrapANSI("\033[33m", "If you change your data directory, you will need to manually move consensus, gateway, tpool, and walletd.sqlite3 to the new directory.", "\033[0m")) + } + + if !promptYesNo("Would you like to change the data directory? (Current: " + dir + ")") { + return + } + cfg.Directory = readInput("Enter data directory") +} + +func setListenAddress(context string, value *string) { + // will continue to prompt until a valid value is entered + for { + input := readInput(fmt.Sprintf("%s (currently %q)", context, *value)) + if input == "" { + return + } + + host, port, err := net.SplitHostPort(input) + if err != nil { + stdoutError(fmt.Sprintf("Invalid %s port %q: %s", context, input, err.Error())) + continue + } + + n, err := strconv.Atoi(port) + if err != nil { + stdoutError(fmt.Sprintf("Invalid %s port %q: %s", context, input, err.Error())) + continue + } else if n < 0 || n > 65535 { + stdoutError(fmt.Sprintf("Invalid %s port %q: must be between 0 and 65535", context, input)) + continue + } + *value = net.JoinHostPort(host, port) + return + } +} + +func setAdvancedConfig() { + if !promptYesNo("Would you like to configure advanced settings?") { + return + } + + fmt.Println("") + fmt.Println("Advanced settings are used to configure walletd's behavior.") + fmt.Println("You can leave these settings blank to use the defaults.") + fmt.Println("") + + fmt.Println("The HTTP address is used to serve the host's admin API.") + fmt.Println("The admin API is used to configure the host.") + fmt.Println("It should not be exposed to the public internet without setting up a reverse proxy.") + setListenAddress("HTTP Address", &cfg.HTTP.Address) + + fmt.Println("") + fmt.Println("The syncer address is used to connect to the Sia network.") + fmt.Println("It should be reachable from other Sia nodes.") + setListenAddress("Syncer Address", &cfg.Syncer.Address) + + fmt.Println("") + fmt.Println("Index mode determines how much of the blockchain to store.") + fmt.Println(`"personal" mode stores events only relevant to addresses associated with a wallet.`) + fmt.Println("To add new addresses, the wallet must be rescanned. This is the default mode.") + fmt.Println("") + fmt.Println(`"full" mode stores all blockchain events. This mode is useful for exchanges and shared wallet clients.`) + fmt.Println("This mode requires significantly more disk space, but does not require rescanning when adding new addresses.") + fmt.Println("") + fmt.Println("This cannot be changed later without resetting walletd.") + fmt.Printf("Currently %q\n", cfg.Index.Mode) + mode := readInput(`Enter index mode ("personal" or "full")`) + switch { + case strings.EqualFold(mode, "personal"): + cfg.Index.Mode = wallet.IndexModePersonal + case strings.EqualFold(mode, "full"): + cfg.Index.Mode = wallet.IndexModeFull + default: + checkFatalError("invalid index mode", errors.New("must be either 'personal' or 'full'")) + } + + fmt.Println("") + fmt.Println("The network is the blockchain network that walletd will connect to.") + fmt.Println("Mainnet is the default network.") + fmt.Println("Zen is a production-like testnet.") + fmt.Println("This cannot be changed later without resetting walletd.") + fmt.Printf("Currently %q\n", cfg.Consensus.Network) + cfg.Consensus.Network = readInput(`Enter network ("mainnet" or "zen")`) +} + +func configPath() string { + if str := os.Getenv(configFileEnvVar); str != "" { + return str + } + + switch runtime.GOOS { + case "windows": + return filepath.Join(os.Getenv("APPDATA"), "walletd", "walletd.yml") + case "darwin": + return filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd", "walletd.yml") + case "linux", "freebsd", "openbsd": + return filepath.Join(string(filepath.Separator), "etc", "walletd", "walletd.yml") + default: + return "walletd.yml" + } +} + +func buildConfig(fp string) { + fmt.Println("walletd Configuration Wizard") + fmt.Println("This wizard will help you configure walletd for the first time.") + fmt.Println("You can always change these settings with the config command or by editing the config file.") + + // write the config file + if fp == "" { + fp = configPath() + } + + fmt.Println("") + fmt.Printf("Config Location %q\n", fp) + + if _, err := os.Stat(fp); err == nil { + if !promptYesNo(fmt.Sprintf("%q already exists. Would you like to overwrite it?", fp)) { + return + } + } else if !errors.Is(err, os.ErrNotExist) { + checkFatalError("failed to check if config file exists", err) + } else { + // ensure the config directory exists + checkFatalError("failed to create config directory", os.MkdirAll(filepath.Dir(fp), 0700)) + } + + fmt.Println("") + setDataDirectory() + + fmt.Println("") + setAPIPassword() + + fmt.Println("") + setAdvancedConfig() + + // write the config file + f, err := os.Create(fp) + checkFatalError("failed to create config file", err) + defer f.Close() + + enc := yaml.NewEncoder(f) + defer enc.Close() + + checkFatalError("failed to encode config file", enc.Encode(cfg)) + checkFatalError("failed to sync config file", f.Sync()) +} diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index da11cc1..727ee72 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -1,98 +1,369 @@ package main import ( - "flag" + "context" + "errors" "fmt" - "log" - "net" "os" "os/signal" - "runtime/debug" + "path/filepath" + "runtime" + "syscall" - "golang.org/x/term" + "go.sia.tech/core/types" + cwallet "go.sia.tech/coreutils/wallet" + "go.sia.tech/walletd/api" + "go.sia.tech/walletd/build" + "go.sia.tech/walletd/config" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "lukechampine.com/flagg" ) -var commit = "?" -var timestamp = "?" +const ( + apiPasswordEnvVar = "WALLETD_API_PASSWORD" + configFileEnvVar = "WALLETD_CONFIG_FILE" + dataDirEnvVar = "WALLETD_DATA_DIR" + logFileEnvVar = "WALLETD_LOG_FILE_PATH" + keystoreSecretEnvVar = "WALLETD_KEYSTORE_SECRET" +) + +const ( + rootUsage = `Usage: + walletd [flags] [action] + +Run 'walletd' with no arguments to start the blockchain node and API server. + +Actions: + version print walletd version + seed generate a recovery phrase + mine run CPU miner` + + versionUsage = `Usage: + walletd version + +Prints the version of the walletd binary. +` + seedUsage = `Usage: + walletd seed + +Generates a secure BIP-39 recovery phrase. +` + mineUsage = `Usage: + walletd mine + +Runs a CPU miner. Not intended for production use. +` +) -func init() { - info, ok := debug.ReadBuildInfo() - if !ok { +var cfg = config.Config{ + Name: "walletd", + Directory: os.Getenv(dataDirEnvVar), + AutoOpenWebUI: true, + HTTP: config.HTTP{ + Address: "localhost:9980", + Password: os.Getenv(apiPasswordEnvVar), + PublicEndpoints: false, + }, + Syncer: config.Syncer{ + Address: ":9981", + Bootstrap: true, + }, + Consensus: config.Consensus{ + Network: "mainnet", + }, + Index: config.Index{ + Mode: wallet.IndexModePersonal, + BatchSize: 1000, + }, + KeyStore: config.KeyStore{ + Enabled: false, + Secret: os.Getenv(keystoreSecretEnvVar), + }, + Log: config.Log{ + Level: "info", + File: config.LogFile{ + Enabled: true, + Format: "json", + Path: os.Getenv(logFileEnvVar), + }, + StdOut: config.StdOut{ + Enabled: true, + Format: "human", + EnableANSI: runtime.GOOS != "windows", + }, + }, +} + +func mustSetAPIPassword() { + if cfg.HTTP.Password != "" { return } - modified := false - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - commit = setting.Value[:8] - case "vcs.time": - timestamp = setting.Value - case "vcs.modified": - modified = setting.Value == "true" + + // retry until a valid API password is entered + for { + fmt.Println("Please choose a password to unlock walletd.") + fmt.Println("This password will be required to access the admin UI in your web browser.") + fmt.Println("(The password must be at least 4 characters.)") + cfg.HTTP.Password = readPasswordInput("Enter password") + if len(cfg.HTTP.Password) >= 4 { + break } + + fmt.Println(wrapANSI("\033[31m", "Password must be at least 4 characters!", "\033[0m")) + fmt.Println("") } - if modified { - commit += " (modified)" +} + +// checkFatalError prints an error message to stderr and exits with a 1 exit code. If err is nil, this is a no-op. +func checkFatalError(context string, err error) { + if err == nil { + return } + os.Stderr.WriteString(fmt.Sprintf("%s: %s\n", context, err)) + os.Exit(1) } -func check(context string, err error) { - if err != nil { - log.Fatalf("%v: %v", context, err) +// tryLoadConfig tries to load the config file. It will try multiple locations +// based on GOOS starting with PWD/walletd.yml. If the file does not exist, it will +// try the next location. If an error occurs while loading the file, it will +// print the error and exit. If the config is successfully loaded, the path to +// the config file is returned. +func tryLoadConfig() string { + for _, fp := range tryConfigPaths() { + if err := config.LoadFile(fp, &cfg); err == nil { + return fp + } else if !errors.Is(err, os.ErrNotExist) { + checkFatalError("failed to load config file", err) + } } + return "" } -func getAPIPassword() string { - apiPassword := os.Getenv("WALLETD_API_PASSWORD") - if apiPassword != "" { - fmt.Println("env: Using WALLETD_API_PASSWORD environment variable") +// jsonEncoder returns a zapcore.Encoder that encodes logs as JSON intended for +// parsing. +func jsonEncoder() zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.TimeKey = "timestamp" + return zapcore.NewJSONEncoder(cfg) +} + +// humanEncoder returns a zapcore.Encoder that encodes logs as human-readable +// text. +func humanEncoder(showColors bool) zapcore.Encoder { + cfg := zap.NewProductionEncoderConfig() + cfg.EncodeTime = zapcore.RFC3339TimeEncoder + cfg.EncodeDuration = zapcore.StringDurationEncoder + + if showColors { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder } else { - fmt.Print("Enter API password: ") - pw, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - check("Could not read API password:", err) - if err != nil { - log.Fatal(err) - } - apiPassword = string(pw) + cfg.EncodeLevel = zapcore.CapitalLevelEncoder } - return apiPassword + + cfg.StacktraceKey = "" + cfg.CallerKey = "" + return zapcore.NewConsoleEncoder(cfg) } -func main() { - log.SetFlags(0) - gatewayAddr := flag.String("addr", ":9981", "p2p address to listen on") - apiAddr := flag.String("http", "localhost:9980", "address to serve API on") - dir := flag.String("dir", ".", "directory to store node state in") - network := flag.String("network", "mainnet", "network to connect to") - upnp := flag.Bool("upnp", true, "attempt to forward ports and discover IP with UPnP") - flag.Parse() - - log.Println("walletd v0.1.0") - if flag.Arg(0) == "version" { - log.Println("Commit Hash:", commit) - log.Println("Commit Date:", timestamp) - return +func parseLogLevel(level string) zap.AtomicLevel { + switch level { + case "debug": + return zap.NewAtomicLevelAt(zap.DebugLevel) + case "info": + return zap.NewAtomicLevelAt(zap.InfoLevel) + case "warn": + return zap.NewAtomicLevelAt(zap.WarnLevel) + case "error": + return zap.NewAtomicLevelAt(zap.ErrorLevel) + default: + fmt.Printf("invalid log level %q", level) + os.Exit(1) } + panic("unreachable") +} - apiPassword := getAPIPassword() - l, err := net.Listen("tcp", *apiAddr) - if err != nil { - log.Fatal(err) +func initStdoutLog(colored bool, levelStr string) *zap.Logger { + level := parseLogLevel(levelStr) + core := zapcore.NewCore(humanEncoder(colored), zapcore.Lock(os.Stdout), level) + return zap.New(core, zap.AddCaller()) +} + +func main() { + log := initStdoutLog(cfg.Log.StdOut.EnableANSI, cfg.Log.Level) + defer log.Sync() + + // attempt to load the config file, command line flags will override any + // values set in the config file + configPath := tryLoadConfig() + if configPath != "" { + log.Info("loaded config file", zap.String("path", configPath)) } + // set the data directory to the default if it is not set + cfg.Directory = defaultDataDirectory(cfg.Directory) + + indexModeStr := cfg.Index.Mode.String() + + var minerAddrStr string + var minerBlocks int + var enableDebug bool + + rootCmd := flagg.Root + rootCmd.Usage = flagg.SimpleUsage(rootCmd, rootUsage) + rootCmd.BoolVar(&enableDebug, "debug", false, "enable debug mode with additional profiling and mining endpoints") + rootCmd.StringVar(&cfg.Directory, "dir", cfg.Directory, "directory to store node state in") + rootCmd.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "address to serve API on") + rootCmd.BoolVar(&cfg.HTTP.PublicEndpoints, "http.public", cfg.HTTP.PublicEndpoints, "disables auth on endpoints that should be publicly accessible when running walletd as a service") + rootCmd.BoolVar(&cfg.KeyStore.Enabled, "keystore", cfg.KeyStore.Enabled, "enables the keystore") + + rootCmd.StringVar(&cfg.Syncer.Address, "addr", cfg.Syncer.Address, "p2p address to listen on") + rootCmd.StringVar(&cfg.Consensus.Network, "network", cfg.Consensus.Network, "network to connect to") + rootCmd.BoolVar(&cfg.Syncer.EnableUPnP, "upnp", cfg.Syncer.EnableUPnP, "attempt to forward ports and discover IP with UPnP") + rootCmd.BoolVar(&cfg.Syncer.Bootstrap, "bootstrap", cfg.Syncer.Bootstrap, "attempt to bootstrap the network") + + rootCmd.StringVar(&indexModeStr, "index.mode", indexModeStr, "address index mode (personal, full, none)") + rootCmd.IntVar(&cfg.Index.BatchSize, "index.batch", cfg.Index.BatchSize, "max number of blocks to index at a time. Increasing this will increase scan speed, but also increase memory and cpu usage.") + + versionCmd := flagg.New("version", versionUsage) + seedCmd := flagg.New("seed", seedUsage) + configCmd := flagg.New("config", "interactively configure walletd") + + mineCmd := flagg.New("mine", mineUsage) + mineCmd.IntVar(&minerBlocks, "n", -1, "mine this many blocks. If negative, mine indefinitely") + mineCmd.StringVar(&minerAddrStr, "addr", "", "address to send block rewards to (required)") + + cmd := flagg.Parse(flagg.Tree{ + Cmd: rootCmd, + Sub: []flagg.Tree{ + {Cmd: configCmd}, + {Cmd: versionCmd}, + {Cmd: seedCmd}, + {Cmd: mineCmd}, + }, + }) + + switch cmd { + case rootCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) + defer cancel() + + if cfg.Directory != "" { + checkFatalError("failed to create data directory", os.MkdirAll(cfg.Directory, 0700)) + } + + mustSetAPIPassword() + + checkFatalError("failed to parse index mode", cfg.Index.Mode.UnmarshalText([]byte(indexModeStr))) + + if cfg.KeyStore.Enabled && cfg.KeyStore.Secret == "" { + checkFatalError("keystore is enabled but no secret was provided", errors.New("missing keystore secret")) + } + + var logCores []zapcore.Core + if cfg.Log.StdOut.Enabled { + // if no log level is set for stdout, use the global log level + if cfg.Log.StdOut.Level == "" { + cfg.Log.StdOut.Level = cfg.Log.Level + } + + var encoder zapcore.Encoder + switch cfg.Log.StdOut.Format { + case "json": + encoder = jsonEncoder() + default: // stdout defaults to human + encoder = humanEncoder(cfg.Log.StdOut.EnableANSI) + } + + // create the stdout logger + level := parseLogLevel(cfg.Log.StdOut.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), level)) + } + + if cfg.Log.File.Enabled { + // if no log level is set for file, use the global log level + if cfg.Log.File.Level == "" { + cfg.Log.File.Level = cfg.Log.Level + } + + // normalize log path + if cfg.Log.File.Path == "" { + cfg.Log.File.Path = filepath.Join(cfg.Directory, "walletd.log") + } + + // configure file logging + var encoder zapcore.Encoder + switch cfg.Log.File.Format { + case "human": + encoder = humanEncoder(false) // disable colors in file log + default: // log file defaults to JSON + encoder = jsonEncoder() + } + + fileWriter, closeFn, err := zap.Open(cfg.Log.File.Path) + checkFatalError("failed to open log file", err) + defer closeFn() + + // create the file logger + level := parseLogLevel(cfg.Log.File.Level) + logCores = append(logCores, zapcore.NewCore(encoder, zapcore.Lock(fileWriter), level)) + } + + var log *zap.Logger + if len(logCores) == 1 { + log = zap.New(logCores[0], zap.AddCaller()) + } else { + log = zap.New(zapcore.NewTee(logCores...), zap.AddCaller()) + } + defer log.Sync() + + // redirect stdlib log to zap + zap.RedirectStdLog(log.Named("stdlib")) + + checkFatalError("failed to run node", runNode(ctx, cfg, log, enableDebug)) + case versionCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + fmt.Println("walletd", build.Version()) + fmt.Println("Commit:", build.Commit()) + fmt.Println("Build Date:", build.Time()) + case seedCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + recoveryPhrase := cwallet.NewSeedPhrase() + var seed [32]byte + checkFatalError("failed to parse mnemonic phrase", cwallet.SeedFromPhrase(&seed, recoveryPhrase)) + addr := types.StandardUnlockHash(cwallet.KeyFromSeed(&seed, 0).PublicKey()) + + fmt.Println("Recovery Phrase:", recoveryPhrase) + fmt.Println("Address", addr) + case configCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } + + buildConfig(configPath) + case mineCmd: + if len(cmd.Args()) != 0 { + cmd.Usage() + return + } - n, err := newNode(*gatewayAddr, *dir, *network, *upnp) - if err != nil { - log.Fatal(err) + minerAddr, err := types.ParseAddress(minerAddrStr) + checkFatalError("failed to parse miner address", err) + mustSetAPIPassword() + c := api.NewClient("http://"+cfg.HTTP.Address+"/api", cfg.HTTP.Password) + runCPUMiner(c, minerAddr, minerBlocks) } - log.Println("p2p: Listening on", n.s.Addr()) - stop := n.Start() - log.Println("api: Listening on", l.Addr()) - go startWeb(l, n, apiPassword) - - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt) - <-signalCh - log.Println("Shutting down...") - stop() } diff --git a/cmd/walletd/miner.go b/cmd/walletd/miner.go new file mode 100644 index 0000000..a4e6767 --- /dev/null +++ b/cmd/walletd/miner.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "log" + "math/big" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/walletd/api" + "lukechampine.com/frand" +) + +func runCPUMiner(c *api.Client, minerAddr types.Address, n int) { + log.Println("Started mining into", minerAddr) + start := time.Now() + + var blocksFound int + for { + if n >= 0 && blocksFound >= n { + break + } + elapsed := time.Since(start) + cs, err := c.ConsensusTipState() + checkFatalError("failed to get consensus tip state:", err) + d, _ := new(big.Int).SetString(cs.Difficulty.String(), 10) + d.Mul(d, big.NewInt(int64(1+elapsed))) + fmt.Printf("\rMining block %4v...(%.2f blocks/day), difficulty %v)", cs.Index.Height+1, float64(blocksFound)*float64(24*time.Hour)/float64(elapsed), cs.Difficulty) + + _, txns, v2txns, err := c.TxpoolTransactions() + checkFatalError("failed to get pool transactions:", err) + b := types.Block{ + ParentID: cs.Index.ID, + Nonce: cs.NonceFactor() * frand.Uint64n(100), + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: cs.BlockReward()}}, + Transactions: txns, + } + for _, txn := range txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.TotalFees()) + } + for _, txn := range v2txns { + b.MinerPayouts[0].Value = b.MinerPayouts[0].Value.Add(txn.MinerFee) + } + if len(v2txns) > 0 || cs.Index.Height+1 >= cs.Network.HardforkV2.RequireHeight { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + Transactions: v2txns, + } + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + } + if !coreutils.FindBlockNonce(cs, &b, time.Minute) { + continue + } + blocksFound++ + index := types.ChainIndex{Height: cs.Index.Height + 1, ID: b.ID()} + tip, err := c.ConsensusTip() + checkFatalError("failed to get consensus tip:", err) + if tip != cs.Index { + fmt.Printf("\nMined %v but tip changed, starting over\n", index) + } else if err := c.SyncerBroadcastBlock(b); err != nil { + fmt.Printf("\nMined invalid block: %v\n", err) + } else if b.V2 == nil { + fmt.Printf("\nFound v1 block %v\n", index) + } else { + fmt.Printf("\nFound v2 block %v\n", index) + } + } +} diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index ba367ea..38ca50b 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -3,225 +3,240 @@ package main import ( "context" "errors" - "log" + "fmt" "net" + "net/http" + "os" "path/filepath" + "runtime" "strconv" + "strings" "time" - bolt "go.etcd.io/bbolt" - "go.sia.tech/core/chain" "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" "go.sia.tech/core/types" - "go.sia.tech/walletd/internal/syncerutil" - "go.sia.tech/walletd/internal/walletutil" - "go.sia.tech/walletd/syncer" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/walletd/api" + "go.sia.tech/walletd/build" + "go.sia.tech/walletd/config" + "go.sia.tech/walletd/keys" + "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/wallet" + "go.sia.tech/web/walletd" + "go.uber.org/zap" "lukechampine.com/upnp" ) -var mainnetBootstrap = []string{ - "108.227.62.195:9981", - "139.162.81.190:9991", - "144.217.7.188:9981", - "147.182.196.252:9981", - "15.235.85.30:9981", - "167.235.234.84:9981", - "173.235.144.230:9981", - "198.98.53.144:7791", - "199.27.255.169:9981", - "2.136.192.200:9981", - "213.159.50.43:9981", - "24.253.116.61:9981", - "46.249.226.103:9981", - "5.165.236.113:9981", - "5.252.226.131:9981", - "54.38.120.222:9981", - "62.210.136.25:9981", - "63.135.62.123:9981", - "65.21.93.245:9981", - "75.165.149.114:9981", - "77.51.200.125:9981", - "81.6.58.121:9981", - "83.194.193.156:9981", - "84.39.246.63:9981", - "87.99.166.34:9981", - "91.214.242.11:9981", - "93.105.88.181:9981", - "93.180.191.86:9981", - "94.130.220.162:9981", -} - -var zenBootstrap = []string{ - "147.135.16.182:9881", - "147.135.39.109:9881", - "51.81.208.10:9881", -} - -type boltDB struct { - tx *bolt.Tx - db *bolt.DB -} - -func (db *boltDB) newTx() (err error) { - if db.tx == nil { - db.tx, err = db.db.Begin(true) +func tryConfigPaths() []string { + if str := os.Getenv(configFileEnvVar); str != "" { + return []string{str} } - return -} -func (db *boltDB) Bucket(name []byte) chain.DBBucket { - if err := db.newTx(); err != nil { - panic(err) + paths := []string{ + "walletd.yml", } - - b := db.tx.Bucket(name) - if b == nil { - return nil - } - return b -} - -func (db *boltDB) CreateBucket(name []byte) (chain.DBBucket, error) { - if err := db.newTx(); err != nil { - return nil, err + if str := os.Getenv(dataDirEnvVar); str != "" { + paths = append(paths, filepath.Join(str, "walletd.yml")) } - b, err := db.tx.CreateBucket(name) - if b == nil { - return nil, err + switch runtime.GOOS { + case "windows": + paths = append(paths, filepath.Join(os.Getenv("APPDATA"), "walletd", "walletd.yml")) + case "darwin": + paths = append(paths, filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd", "walletd.yml")) + case "linux", "freebsd", "openbsd": + paths = append(paths, + filepath.Join(string(filepath.Separator), "etc", "walletd", "walletd.yml"), + filepath.Join(string(filepath.Separator), "var", "lib", "walletd", "walletd.yml"), // old default for the Linux service + ) } - return b, nil + return paths } -func (db *boltDB) Flush() error { - if db.tx == nil { - return nil +func defaultDataDirectory(fp string) string { + // use the provided path if it's not empty + if fp != "" { + return fp } - if err := db.tx.Commit(); err != nil { - return err + // check for databases in the current directory + if _, err := os.Stat("walletd.db"); err == nil { + return "." + } else if _, err := os.Stat("walletd.sqlite3"); err == nil { + return "." } - db.tx = nil - return nil -} -func (db *boltDB) Cancel() { - if db.tx == nil { - return + // default to the operating system's application directory + switch runtime.GOOS { + case "windows": + return filepath.Join(os.Getenv("APPDATA"), "walletd") + case "darwin": + return filepath.Join(os.Getenv("HOME"), "Library", "Application Support", "walletd") + case "linux", "freebsd", "openbsd": + return filepath.Join(string(filepath.Separator), "var", "lib", "walletd") + default: + return "." } - - db.tx.Rollback() - db.tx = nil -} - -func (db *boltDB) Close() error { - db.Flush() - return db.db.Close() } -type node struct { - cm *chain.Manager - s *syncer.Syncer - wm *walletutil.JSONWalletManager - - Start func() (stop func()) +func setupUPNP(ctx context.Context, port uint16, log *zap.Logger) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + d, err := upnp.Discover(ctx) + if err != nil { + return "", fmt.Errorf("couldn't discover UPnP router: %w", err) + } else if !d.IsForwarded(port, "TCP") { + if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { + log.Debug("couldn't forward port", zap.Error(err)) + } else { + log.Debug("upnp: forwarded p2p port", zap.Uint16("port", port)) + } + } + return d.ExternalIP() } -func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) { +func runNode(ctx context.Context, cfg config.Config, log *zap.Logger, enableDebug bool) error { var network *consensus.Network var genesisBlock types.Block var bootstrapPeers []string - switch chainNetwork { + switch cfg.Consensus.Network { case "mainnet": network, genesisBlock = chain.Mainnet() - bootstrapPeers = mainnetBootstrap + bootstrapPeers = syncer.MainnetBootstrapPeers case "zen": network, genesisBlock = chain.TestnetZen() - bootstrapPeers = zenBootstrap + bootstrapPeers = syncer.ZenBootstrapPeers + case "anagami": + network, genesisBlock = chain.TestnetAnagami() + bootstrapPeers = syncer.AnagamiBootstrapPeers default: - return nil, errors.New("invalid network: must be one of 'mainnet' or 'zen'") + return errors.New("invalid network: must be one of 'mainnet', 'zen', or 'anagami'") } - bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(cfg.Directory, "consensus.db")) if err != nil { - log.Fatal(err) + return fmt.Errorf("failed to open consensus database: %w", err) } - db := &boltDB{db: bdb} - dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) + defer bdb.Close() + + dbstore, tipState, err := chain.NewDBStore(bdb, network, genesisBlock) if err != nil { - return nil, err + return fmt.Errorf("failed to create chain store: %w", err) } cm := chain.NewManager(dbstore, tipState) - l, err := net.Listen("tcp", addr) + syncerListener, err := net.Listen("tcp", cfg.Syncer.Address) + if err != nil { + return fmt.Errorf("failed to listen on %q: %w", cfg.Syncer.Address, err) + } + defer syncerListener.Close() + + httpListener, err := net.Listen("tcp", cfg.HTTP.Address) if err != nil { - return nil, err - } - syncerAddr := l.Addr().String() - if useUPNP { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if d, err := upnp.Discover(ctx); err != nil { - log.Println("WARN: couldn't discover UPnP device:", err) + return fmt.Errorf("failed to listen on %q: %w", cfg.HTTP.Address, err) + } + defer httpListener.Close() + + syncerAddr := syncerListener.Addr().String() + if cfg.Syncer.EnableUPnP { + _, portStr, _ := net.SplitHostPort(cfg.Syncer.Address) + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return fmt.Errorf("failed to parse syncer port: %w", err) + } + + ip, err := setupUPNP(context.Background(), uint16(port), log) + if err != nil { + log.Warn("failed to set up UPnP", zap.Error(err)) } else { - _, portStr, _ := net.SplitHostPort(addr) - port, _ := strconv.Atoi(portStr) - if !d.IsForwarded(uint16(port), "TCP") { - if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { - log.Println("WARN: couldn't forward port:", err) - } else { - log.Println("p2p: Forwarded port", port) - } - } - if ip, err := d.ExternalIP(); err != nil { - log.Println("WARN: couldn't determine external IP:", err) - } else { - log.Println("p2p: External IP is", ip) - syncerAddr = net.JoinHostPort(ip, portStr) - } + syncerAddr = net.JoinHostPort(ip, portStr) } } + // peers will reject us if our hostname is empty or unspecified, so use loopback host, port, _ := net.SplitHostPort(syncerAddr) if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { syncerAddr = net.JoinHostPort("127.0.0.1", port) } - ps, err := syncerutil.NewJSONPeerStore(filepath.Join(dir, "peers.json")) + store, err := sqlite.OpenDatabase(filepath.Join(cfg.Directory, "walletd.sqlite3"), log.Named("sqlite3")) if err != nil { - log.Fatal(err) + return fmt.Errorf("failed to open wallet database: %w", err) } - for _, peer := range bootstrapPeers { - ps.AddPeer(peer) + defer store.Close() + + if cfg.Syncer.Bootstrap { + for _, peer := range bootstrapPeers { + if err := store.AddPeer(peer); err != nil { + return fmt.Errorf("failed to add bootstrap peer %q: %w", peer, err) + } + } + for _, peer := range cfg.Syncer.Peers { + if err := store.AddPeer(peer); err != nil { + return fmt.Errorf("failed to add peer %q: %w", peer, err) + } + } } + + ps, err := sqlite.NewPeerStore(store) + if err != nil { + return fmt.Errorf("failed to create peer store: %w", err) + } + header := gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: syncerAddr, } - s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Default())) - wm, err := walletutil.NewJSONWalletManager(dir, cm) + s := syncer.New(syncerListener, cm, ps, header, syncer.WithLogger(log.Named("syncer"))) + defer s.Close() + go s.Run() + + wm, err := wallet.NewManager(cm, store, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(cfg.Index.Mode), wallet.WithSyncBatchSize(cfg.Index.BatchSize)) if err != nil { - return nil, err - } - - return &node{ - cm: cm, - s: s, - wm: wm, - Start: func() func() { - ch := make(chan struct{}) - go func() { - s.Run() - close(ch) - }() - return func() { - l.Close() - <-ch - db.Close() + return fmt.Errorf("failed to create wallet manager: %w", err) + } + defer wm.Close() + + apiOpts := []api.ServerOption{ + api.WithLogger(log.Named("api")), + api.WithPublicEndpoints(cfg.HTTP.PublicEndpoints), + api.WithBasicAuth(cfg.HTTP.Password), + } + if enableDebug { + apiOpts = append(apiOpts, api.WithDebug()) + } + if cfg.KeyStore.Enabled { + km, err := keys.NewManager(store, cfg.KeyStore.Secret) + if err != nil { + return fmt.Errorf("failed to create key manager: %w", err) + } + defer km.Close() + + apiOpts = append(apiOpts, api.WithKeyManager(km)) + } + api := api.NewServer(cm, s, wm, apiOpts...) + web := walletd.Handler() + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api") { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") + api.ServeHTTP(w, r) + return } - }, - }, nil + web.ServeHTTP(w, r) + }), + ReadTimeout: 10 * time.Second, + } + defer server.Close() + go server.Serve(httpListener) + + log.Info("node started", zap.String("network", network.Name), zap.Stringer("syncer", syncerListener.Addr()), zap.Stringer("http", httpListener.Addr()), zap.String("version", build.Version()), zap.String("commit", build.Commit())) + <-ctx.Done() + log.Info("shutting down") + return nil } diff --git a/cmd/walletd/web.go b/cmd/walletd/web.go deleted file mode 100644 index c6de95c..0000000 --- a/cmd/walletd/web.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "net" - "net/http" - "strings" - - "go.sia.tech/jape" - "go.sia.tech/walletd/api" - "go.sia.tech/web/walletd" -) - -func startWeb(l net.Listener, node *node, password string) error { - renter := api.NewServer(node.cm, node.s, node.wm) - api := jape.BasicAuth(password)(renter) - web := walletd.Handler() - return http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api") { - r.URL.Path = strings.TrimPrefix(r.URL.Path, "/api") - api.ServeHTTP(w, r) - return - } - web.ServeHTTP(w, r) - })) -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..e3d5a75 --- /dev/null +++ b/config/config.go @@ -0,0 +1,102 @@ +package config + +import ( + "bytes" + "fmt" + "os" + + "go.sia.tech/walletd/wallet" + "gopkg.in/yaml.v3" +) + +type ( + // HTTP contains the configuration for the HTTP server. + HTTP struct { + Address string `yaml:"address,omitempty"` + Password string `yaml:"password,omitempty"` + PublicEndpoints bool `yaml:"publicEndpoints,omitempty"` + } + + // Syncer contains the configuration for the consensus set syncer. + Syncer struct { + Address string `yaml:"address,omitempty"` + Bootstrap bool `yaml:"bootstrap,omitempty"` + EnableUPnP bool `yaml:"enableUPnP,omitempty"` + Peers []string `yaml:"peers,omitempty"` + } + + // KeyStore contains the configuration for the key store. + KeyStore struct { + Enabled bool `yaml:"enabled,omitempty"` + Secret string `yaml:"secret,omitempty"` + } + + // Consensus contains the configuration for the consensus set. + Consensus struct { + Network string `yaml:"network,omitempty"` + } + + // Index contains the configuration for the blockchain indexer + Index struct { + Mode wallet.IndexMode `yaml:"mode,omitempty"` + BatchSize int `yaml:"batchSize,omitempty"` + } + + // LogFile configures the file output of the logger. + LogFile struct { + Enabled bool `yaml:"enabled,omitempty"` + Level string `yaml:"level,omitempty"` // override the file log level + Format string `yaml:"format,omitempty"` + // Path is the path of the log file. + Path string `yaml:"path,omitempty"` + } + + // StdOut configures the standard output of the logger. + StdOut struct { + Level string `yaml:"level,omitempty"` // override the stdout log level + Enabled bool `yaml:"enabled,omitempty"` + Format string `yaml:"format,omitempty"` + EnableANSI bool `yaml:"enableANSI,omitempty"` //nolint:tagliatelle + } + + // Log contains the configuration for the logger. + Log struct { + Level string `yaml:"level,omitempty"` // global log level + StdOut StdOut `yaml:"stdout,omitempty"` + File LogFile `yaml:"file,omitempty"` + } + + // Config contains the configuration for the host. + Config struct { + Name string `yaml:"name,omitempty"` + Directory string `yaml:"directory,omitempty"` + AutoOpenWebUI bool `yaml:"autoOpenWebUI,omitempty"` + + HTTP HTTP `yaml:"http,omitempty"` + Consensus Consensus `yaml:"consensus,omitempty"` + Syncer Syncer `yaml:"syncer,omitempty"` + Log Log `yaml:"log,omitempty"` + Index Index `yaml:"index,omitempty"` + KeyStore KeyStore `yaml:"keystore,omitempty"` + } +) + +// LoadFile loads the configuration from the provided file path. +// If the file does not exist, an error is returned. +// If the file exists but cannot be decoded, the function will attempt +// to upgrade the config file. +func LoadFile(fp string, cfg *Config) error { + buf, err := os.ReadFile(fp) + if err != nil { + return fmt.Errorf("failed to read config file: %w", err) + } + + r := bytes.NewReader(buf) + dec := yaml.NewDecoder(r) + dec.KnownFields(true) + + if err := dec.Decode(cfg); err != nil { + return fmt.Errorf("failed to decode config file: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index 5855b34..6edf081 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,42 @@ -module go.sia.tech/walletd +module go.sia.tech/walletd // v2.0.0 -go 1.18 +go 1.23.1 + +toolchain go1.23.2 require ( - go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 - go.sia.tech/jape v0.9.0 - go.sia.tech/web/walletd v0.14.0 - golang.org/x/term v0.6.0 - lukechampine.com/frand v1.4.2 + github.com/mattn/go-sqlite3 v1.14.24 + go.sia.tech/core v0.10.3 + go.sia.tech/coreutils v0.12.0 + go.sia.tech/jape v0.12.1 + go.sia.tech/web/walletd v0.29.0 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.34.0 + golang.org/x/term v0.29.0 + gopkg.in/yaml.v3 v3.0.1 + lukechampine.com/flagg v1.1.1 + lukechampine.com/frand v1.5.1 lukechampine.com/upnp v0.3.0 ) require ( - github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - go.sia.tech/mux v1.2.0 // indirect - go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect - golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/tools v0.7.0 // indirect + github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.50.0 // indirect + github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + go.etcd.io/bbolt v1.4.0 // indirect + go.sia.tech/mux v1.4.0 // indirect + go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect + go.uber.org/mock v0.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 6db5c74..ea94fe8 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,97 @@ -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY= -github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 h1:2nKOKa99g9h9m3hL5UortAbmnwuwXhDcTHIhzmqBae8= -go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= -go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= -go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= -go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= -go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= -go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= -go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/walletd v0.14.0 h1:GcNDv5HrLoMPKB8LcGIMKEoare+zUTkvRUkYFmq5KTE= -go.sia.tech/web/walletd v0.14.0/go.mod h1:OHFWEbjLCR5I06E05GA98HIAdacTM5Ag7sL9ubcvgKw= -golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 h1:NvGWuYG8dkDHFSKksI1P9faiVJ9rayE6l0+ouWVIDs8= -golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo= +github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= +github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= +go.sia.tech/core v0.10.3 h1:fNt5Dkqxr+Q9vz3jQcdZqFPNA0LgKD6ZtpxQvkTZxns= +go.sia.tech/core v0.10.3/go.mod h1:JvW51XfqDljOjv7kagSxQJJ00hzYmgRIkA76Krb3p3U= +go.sia.tech/coreutils v0.12.0 h1:DOPFHveZeIxDATv9MW8G3kVEkOMOSfiSiN3jxcQ7Pik= +go.sia.tech/coreutils v0.12.0/go.mod h1:XBMg1imCUr5eYjaEemP9GZXvoOjYNvGTvmR/QHObAJg= +go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= +go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= +go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo= +go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI= +go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk= +go.sia.tech/web/walletd v0.29.0 h1:JHJj6TlQozKGcUqUyL0YXR0I+Poe1kjgcGA5v1/9tjA= +go.sia.tech/web/walletd v0.29.0/go.mod h1:VkWPLolV88EeAlGzTxSktwQRQ5+MZdkWan0N4d5aCZ8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.34.0 h1:+/C6tk6rf/+t5DhUketUbD1aNGqiSX3j15Z6xuIDlBA= +golang.org/x/crypto v0.34.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= -lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/flagg v1.1.1 h1:jB5oL4D5zSUrzm5og6dDEi5pnrTF1poKfC7KE1lLsqc= +lukechampine.com/flagg v1.1.1/go.mod h1:a9ZuZu5LSPXELWSJrabRD00ort+lDXSOQu34xWgEoDI= +lukechampine.com/frand v1.5.1 h1:fg0eRtdmGFIxhP5zQJzM1lFDbD6CUfu/f+7WgAZd5/w= +lukechampine.com/frand v1.5.1/go.mod h1:4VstaWc2plN4Mjr10chUD46RAVGWhpkZ5Nja8+Azp0Q= lukechampine.com/upnp v0.3.0 h1:UVCD6eD6fmJmwak6DVE3vGN+L46Fk8edTcC6XYCb6C4= lukechampine.com/upnp v0.3.0/go.mod h1:sOuF+fGSDKjpUm6QI0mfb82ScRrhj8bsqsD78O5nK1k= diff --git a/internal/syncerutil/store.go b/internal/syncerutil/store.go deleted file mode 100644 index a486291..0000000 --- a/internal/syncerutil/store.go +++ /dev/null @@ -1,202 +0,0 @@ -package syncerutil - -import ( - "encoding/json" - "net" - "os" - "sync" - "time" - - "go.sia.tech/walletd/syncer" -) - -type peerBan struct { - Expiry time.Time `json:"expiry"` - Reason string `json:"reason"` -} - -// EphemeralPeerStore implements PeerStore with an in-memory map. -type EphemeralPeerStore struct { - peers map[string]syncer.PeerInfo - bans map[string]peerBan - mu sync.Mutex -} - -func (eps *EphemeralPeerStore) banned(peer string) bool { - host, _, err := net.SplitHostPort(peer) - if err != nil { - return false // shouldn't happen - } - for _, s := range []string{ - peer, // 1.2.3.4:5678 - syncer.Subnet(host + "/32"), // 1.2.3.4:* - syncer.Subnet(host + "/24"), // 1.2.3.* - syncer.Subnet(host + "/16"), // 1.2.* - syncer.Subnet(host + "/8"), // 1.* - } { - if b, ok := eps.bans[s]; ok { - if time.Until(b.Expiry) <= 0 { - delete(eps.bans, s) - } else { - return true - } - } - } - return false -} - -// AddPeer implements PeerStore. -func (eps *EphemeralPeerStore) AddPeer(peer string) { - eps.mu.Lock() - defer eps.mu.Unlock() - if _, ok := eps.peers[peer]; !ok { - eps.peers[peer] = syncer.PeerInfo{FirstSeen: time.Now()} - } -} - -// Peers implements PeerStore. -func (eps *EphemeralPeerStore) Peers() []string { - eps.mu.Lock() - defer eps.mu.Unlock() - var peers []string - for p := range eps.peers { - if !eps.banned(p) { - peers = append(peers, p) - } - } - return peers -} - -// UpdatePeerInfo implements PeerStore. -func (eps *EphemeralPeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - eps.mu.Lock() - defer eps.mu.Unlock() - info, ok := eps.peers[peer] - if !ok { - return - } - fn(&info) - eps.peers[peer] = info -} - -// PeerInfo implements PeerStore. -func (eps *EphemeralPeerStore) PeerInfo(peer string) (syncer.PeerInfo, bool) { - eps.mu.Lock() - defer eps.mu.Unlock() - info, ok := eps.peers[peer] - return info, ok -} - -// Ban implements PeerStore. -func (eps *EphemeralPeerStore) Ban(peer string, duration time.Duration, reason string) { - eps.mu.Lock() - defer eps.mu.Unlock() - // canonicalize - if _, ipnet, err := net.ParseCIDR(peer); err == nil { - peer = ipnet.String() - } - eps.bans[peer] = peerBan{Expiry: time.Now().Add(duration), Reason: reason} -} - -// Banned implements PeerStore. -func (eps *EphemeralPeerStore) Banned(peer string) bool { - eps.mu.Lock() - defer eps.mu.Unlock() - return eps.banned(peer) -} - -// NewEphemeralPeerStore initializes an EphemeralPeerStore. -func NewEphemeralPeerStore() *EphemeralPeerStore { - return &EphemeralPeerStore{ - peers: make(map[string]syncer.PeerInfo), - bans: make(map[string]peerBan), - } -} - -type jsonPersist struct { - Peers map[string]syncer.PeerInfo `json:"peers"` - Bans map[string]peerBan `json:"bans"` -} - -// JSONPeerStore implements PeerStore with a JSON file on disk. -type JSONPeerStore struct { - *EphemeralPeerStore - path string - lastSave time.Time -} - -func (jps *JSONPeerStore) load() error { - f, err := os.Open(jps.path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p jsonPersist - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - jps.EphemeralPeerStore.peers = p.Peers - jps.EphemeralPeerStore.bans = p.Bans - return nil -} - -func (jps *JSONPeerStore) save() error { - jps.EphemeralPeerStore.mu.Lock() - defer jps.EphemeralPeerStore.mu.Unlock() - if time.Since(jps.lastSave) < 5*time.Second { - return nil - } - defer func() { jps.lastSave = time.Now() }() - p := jsonPersist{ - Peers: jps.EphemeralPeerStore.peers, - Bans: jps.EphemeralPeerStore.bans, - } - js, err := json.MarshalIndent(p, "", " ") - if err != nil { - return err - } - f, err := os.OpenFile(jps.path+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(jps.path+"_tmp", jps.path); err != nil { - return err - } - return nil -} - -// AddPeer implements PeerStore. -func (jps *JSONPeerStore) AddPeer(peer string) { - jps.EphemeralPeerStore.AddPeer(peer) - jps.save() -} - -// UpdatePeerInfo implements PeerStore. -func (jps *JSONPeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) { - jps.EphemeralPeerStore.UpdatePeerInfo(peer, fn) - jps.save() -} - -// Ban implements PeerStore. -func (jps *JSONPeerStore) Ban(peer string, duration time.Duration, reason string) { - jps.EphemeralPeerStore.Ban(peer, duration, reason) - jps.save() -} - -// NewJSONPeerStore returns a JSONPeerStore backed by the specified file. -func NewJSONPeerStore(path string) (*JSONPeerStore, error) { - jps := &JSONPeerStore{ - EphemeralPeerStore: NewEphemeralPeerStore(), - path: path, - } - return jps, jps.load() -} diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 0000000..839c154 --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,102 @@ +package testutil + +import ( + "net" + "path/filepath" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/persist/sqlite" + "go.uber.org/zap" +) + +type ( + // A ConsensusNode is a test harness for starting a bare-bones consensus node. + ConsensusNode struct { + Store *sqlite.Store + Chain *chain.Manager + Syncer *syncer.Syncer + } +) + +// WaitForSync waits for the store to sync to the current tip of the chain manager. +func (cn *ConsensusNode) WaitForSync(tb testing.TB) { + tb.Helper() + + for i := 0; i < 1000; i++ { + index, err := cn.Store.LastCommittedIndex() + if err != nil { + tb.Fatal(err) + } else if index == cn.Chain.Tip() { + return + } + time.Sleep(10 * time.Millisecond) + } + tb.Fatal("timeout waiting for sync") +} + +// MineBlocks mines n blocks, sending the rewards to addr. +func (cn *ConsensusNode) MineBlocks(tb testing.TB, addr types.Address, n int) { + tb.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(tb, cn.Chain, addr, 1) + cn.WaitForSync(tb) + } +} + +// NewConsensusNode creates a new ConsensusNode. +func NewConsensusNode(tb testing.TB, n *consensus.Network, genesis types.Block, log *zap.Logger) *ConsensusNode { + l, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { l.Close() }) + + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesis) + if err != nil { + tb.Fatal(err) + } + cm := chain.NewManager(dbstore, tipState) + + store, err := sqlite.OpenDatabase(filepath.Join(tb.TempDir(), "walletd.sqlite"), log.Named("sqlite3")) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { store.Close() }) + + peerStore, err := sqlite.NewPeerStore(store) + if err != nil { + tb.Fatal(err) + } + + s := syncer.New(l, cm, peerStore, gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l.Addr().String(), + }) + tb.Cleanup(func() { s.Close() }) + go s.Run() + + return &ConsensusNode{ + Store: store, + Chain: cm, + Syncer: s, + } +} + +// V1Network returns a test network and genesis block. +func V1Network() (*consensus.Network, types.Block) { + return testutil.Network() +} + +// V2Network returns a test network and genesis block with early V2 hardforks +func V2Network() (*consensus.Network, types.Block) { + return testutil.V2Network() +} diff --git a/internal/threadgroup/threadgroup.go b/internal/threadgroup/threadgroup.go new file mode 100644 index 0000000..d418264 --- /dev/null +++ b/internal/threadgroup/threadgroup.go @@ -0,0 +1,110 @@ +// Package threadgroup exposes a ThreadGroup object which can be used to +// facilitate clean shutdown. A ThreadGroup is similar to a sync.WaitGroup, +// but with two important additions: The ability to detect when shutdown has +// been initiated, and protections against adding more threads after shutdown +// has completed. +// +// ThreadGroup was designed with the following shutdown sequence in mind: +// +// 1. Call Stop, signaling that shutdown has begun. After Stop is called, no +// new goroutines should be created. +// +// 2. Wait for Stop to return. When Stop returns, all goroutines should have +// returned. +// +// 3. Free any resources used by the goroutines. +package threadgroup + +import ( + "context" + "errors" + "sync" +) + +type ( + // A ThreadGroup is a sync.WaitGroup with additional functionality for + // facilitating clean shutdown. + ThreadGroup struct { + mu sync.Mutex + wg sync.WaitGroup + closed chan struct{} + } +) + +// ErrClosed is returned when the threadgroup has already been stopped +var ErrClosed = errors.New("threadgroup closed") + +// Done returns a channel that will be closed when the threadgroup is stopped +func (tg *ThreadGroup) Done() <-chan struct{} { + return tg.closed +} + +// Add adds a new thread to the group, done must be called to signal that the +// thread is done. Returns ErrClosed if the threadgroup is already closed. +func (tg *ThreadGroup) Add() (func(), error) { + tg.mu.Lock() + defer tg.mu.Unlock() + select { + case <-tg.closed: + return nil, ErrClosed + default: + } + tg.wg.Add(1) + return func() { tg.wg.Done() }, nil +} + +// WithContext returns a copy of the parent context. The returned context will +// be cancelled if the parent context is cancelled or if the threadgroup is +// stopped. +func (tg *ThreadGroup) WithContext(parent context.Context) (context.Context, context.CancelFunc) { + // wrap the parent context in a cancellable context + ctx, cancel := context.WithCancel(parent) + // start a goroutine to wait for either the parent context being cancelled + // or the threagroup being stopped + go func() { + select { + case <-ctx.Done(): + case <-tg.closed: + } + cancel() // threadgroup is stopping or context cancelled, cancel the context + }() + return ctx, cancel +} + +// AddWithContext adds a new thread to the group and returns a copy of the parent +// context. It is a convenience function combining Add and WithContext. +func (tg *ThreadGroup) AddWithContext(parent context.Context) (context.Context, context.CancelFunc, error) { + // try to add to the group + done, err := tg.Add() + if err != nil { + return nil, nil, err + } + + ctx, cancel := tg.WithContext(parent) + var once sync.Once + return ctx, func() { + cancel() + // it must be safe to call cancel multiple times, but it is not safe to + // call done multiple times since it's decrementing the waitgroup + once.Do(done) + }, nil +} + +// Stop stops accepting new threads and waits for all existing threads to close +func (tg *ThreadGroup) Stop() { + tg.mu.Lock() + select { + case <-tg.closed: + default: + close(tg.closed) + } + tg.mu.Unlock() + tg.wg.Wait() +} + +// New creates a new threadgroup +func New() *ThreadGroup { + return &ThreadGroup{ + closed: make(chan struct{}), + } +} diff --git a/internal/threadgroup/threadgroup_test.go b/internal/threadgroup/threadgroup_test.go new file mode 100644 index 0000000..27e57e3 --- /dev/null +++ b/internal/threadgroup/threadgroup_test.go @@ -0,0 +1,89 @@ +package threadgroup + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestThreadgroup(t *testing.T) { + tg := New() + + for i := 0; i < 10; i++ { + done, err := tg.Add() + if err != nil { + t.Fatal(err) + } + time.AfterFunc(100*time.Millisecond, done) + } + start := time.Now() + tg.Stop() + if time.Since(start) < 100*time.Millisecond { + t.Fatal("expected stop to wait for all threads to complete") + } + + _, err := tg.Add() + if !errors.Is(err, ErrClosed) { + t.Fatalf("expected ErrClosed, got %v", err) + } +} + +func TestThreadgroupContext(t *testing.T) { + tg := New() + + t.Run("context cancel", func(t *testing.T) { + ctx, cancel, err := tg.AddWithContext(context.Background()) + if err != nil { + t.Fatal(err) + } + defer cancel() + + time.AfterFunc(100*time.Millisecond, cancel) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.Canceled) { + t.Fatalf("expected Canceled, got %v", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("expected context to be cancelled") + } + }) + + t.Run("parent cancel", func(t *testing.T) { + parentCtx, parentCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer parentCancel() + + ctx, cancel, err := tg.AddWithContext(parentCtx) + if err != nil { + t.Fatal(err) + } + defer cancel() + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("expected DeadlineExceeded, got %v", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("expected context to be cancelled") + } + }) + + t.Run("stop", func(t *testing.T) { + for i := 0; i < 10; i++ { + _, cancel, err := tg.AddWithContext(context.Background()) + if err != nil { + t.Fatal(err) + } + time.AfterFunc(100*time.Millisecond, cancel) + } + + start := time.Now() + tg.Stop() + if time.Since(start) < 100*time.Millisecond { + t.Fatal("expected threadgroup to wait until all threads complete") + } + }) +} diff --git a/internal/walletutil/manager.go b/internal/walletutil/manager.go deleted file mode 100644 index 77b8d1f..0000000 --- a/internal/walletutil/manager.go +++ /dev/null @@ -1,400 +0,0 @@ -package walletutil - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "sync" - - "go.sia.tech/core/chain" - "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" -) - -var errNoWallet = errors.New("wallet does not exist") - -type ChainManager interface { - AddSubscriber(s chain.Subscriber, tip types.ChainIndex) error - RemoveSubscriber(s chain.Subscriber) - BestIndex(height uint64) (types.ChainIndex, bool) -} - -type managedEphemeralWallet struct { - w *EphemeralStore - info json.RawMessage - subscribed bool -} - -// An EphemeralWalletManager manages multiple ephemeral wallet stores. -type EphemeralWalletManager struct { - cm ChainManager - mu sync.Mutex - wallets map[string]*managedEphemeralWallet -} - -// AddWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) AddWallet(name string, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - if _, ok := wm.wallets[name]; ok { - return errors.New("wallet already exists") - } - store := NewEphemeralStore() - wm.wallets[name] = &managedEphemeralWallet{store, info, false} - return nil -} - -// DeleteWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) DeleteWallet(name string) error { - wm.mu.Lock() - defer wm.mu.Unlock() - delete(wm.wallets, name) - return nil -} - -// Wallets implements api.WalletManager. -func (wm *EphemeralWalletManager) Wallets() map[string]json.RawMessage { - wm.mu.Lock() - defer wm.mu.Unlock() - ws := make(map[string]json.RawMessage, len(wm.wallets)) - for name, w := range wm.wallets { - ws[name] = w.info - } - return ws -} - -// AddAddress implements api.WalletManager. -func (wm *EphemeralWalletManager) AddAddress(name string, addr types.Address, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.AddAddress(addr, info) -} - -// RemoveAddress implements api.WalletManager. -func (wm *EphemeralWalletManager) RemoveAddress(name string, addr types.Address) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.RemoveAddress(addr) -} - -// Addresses implements api.WalletManager. -func (wm *EphemeralWalletManager) Addresses(name string) (map[types.Address]json.RawMessage, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Addresses() -} - -// Events implements api.WalletManager. -func (wm *EphemeralWalletManager) Events(name string, offset, limit int) ([]wallet.Event, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Events(offset, limit) -} - -// Annotate implements api.WalletManager. -func (wm *EphemeralWalletManager) Annotate(name string, txns []types.Transaction) ([]wallet.PoolTransaction, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Annotate(txns), nil -} - -// UnspentOutputs implements api.WalletManager. -func (wm *EphemeralWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, nil, errNoWallet - } - return mw.w.UnspentOutputs() -} - -// SubscribeWallet implements api.WalletManager. -func (wm *EphemeralWalletManager) SubscribeWallet(name string, startHeight uint64) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } else if mw.subscribed { - return errors.New("already subscribed") - } - // AddSubscriber applies each block *after* index, but we want to *include* - // the block at startHeight, so subtract one. - // - // NOTE: if subscribing from height 0, we must pass an empty index in order - // to receive the genesis block. - var index types.ChainIndex - if startHeight > 0 { - if index, ok = wm.cm.BestIndex(startHeight - 1); !ok { - return errors.New("invalid height") - } - } - if err := wm.cm.AddSubscriber(mw.w, index); err != nil { - return err - } - mw.subscribed = true - return nil -} - -// NewEphemeralWalletManager returns a new EphemeralWalletManager. -func NewEphemeralWalletManager(cm ChainManager) *EphemeralWalletManager { - return &EphemeralWalletManager{ - cm: cm, - wallets: make(map[string]*managedEphemeralWallet), - } -} - -type managedJSONWallet struct { - w *JSONStore - info json.RawMessage - subscribed bool -} - -type managerPersistData struct { - Wallets []managerPersistWallet `json:"wallets"` -} - -type managerPersistWallet struct { - Name string `json:"name"` - Info json.RawMessage `json:"info"` - Subscribed bool `json:"subscribed"` -} - -// A JSONWalletManager manages multiple JSON wallet stores. -type JSONWalletManager struct { - dir string - cm ChainManager - mu sync.Mutex - wallets map[string]*managedJSONWallet -} - -func (wm *JSONWalletManager) save() error { - var p managerPersistData - for name, mw := range wm.wallets { - p.Wallets = append(p.Wallets, managerPersistWallet{name, mw.info, mw.subscribed}) - } - js, err := json.MarshalIndent(p, "", " ") - if err != nil { - return err - } - dst := filepath.Join(wm.dir, "wallets.json") - f, err := os.OpenFile(dst+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(dst+"_tmp", dst); err != nil { - return err - } - return nil -} - -func (wm *JSONWalletManager) load() error { - dst := filepath.Join(wm.dir, "wallets.json") - f, err := os.Open(dst) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p managerPersistData - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - for _, pw := range p.Wallets { - wm.wallets[pw.Name] = &managedJSONWallet{nil, pw.Info, pw.Subscribed} - } - return nil -} - -// AddWallet implements api.WalletManager. -func (wm *JSONWalletManager) AddWallet(name string, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - if mw, ok := wm.wallets[name]; ok { - // update existing wallet - mw.info = info - return wm.save() - } - store, _, err := NewJSONStore(filepath.Join(wm.dir, "wallets", name+".json")) - if err != nil { - return err - } - wm.wallets[name] = &managedJSONWallet{store, info, false} - return wm.save() -} - -// DeleteWallet implements api.WalletManager. -func (wm *JSONWalletManager) DeleteWallet(name string) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil - } - wm.cm.RemoveSubscriber(mw.w) - delete(wm.wallets, name) - return os.RemoveAll(filepath.Join(wm.dir, "wallets", name+".json")) -} - -// Wallets implements api.WalletManager. -func (wm *JSONWalletManager) Wallets() map[string]json.RawMessage { - wm.mu.Lock() - defer wm.mu.Unlock() - ws := make(map[string]json.RawMessage, len(wm.wallets)) - for name, w := range wm.wallets { - ws[name] = w.info - } - return ws -} - -// AddAddress implements api.WalletManager. -func (wm *JSONWalletManager) AddAddress(name string, addr types.Address, info json.RawMessage) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.AddAddress(addr, info) -} - -// RemoveAddress implements api.WalletManager. -func (wm *JSONWalletManager) RemoveAddress(name string, addr types.Address) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } - return mw.w.RemoveAddress(addr) -} - -// Addresses implements api.WalletManager. -func (wm *JSONWalletManager) Addresses(name string) (map[types.Address]json.RawMessage, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Addresses() -} - -// Events implements api.WalletManager. -func (wm *JSONWalletManager) Events(name string, offset, limit int) ([]wallet.Event, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Events(offset, limit) -} - -// Annotate implements api.WalletManager. -func (wm *JSONWalletManager) Annotate(name string, txns []types.Transaction) ([]wallet.PoolTransaction, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, errNoWallet - } - return mw.w.Annotate(txns), nil -} - -// UnspentOutputs implements api.WalletManager. -func (wm *JSONWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return nil, nil, errNoWallet - } - return mw.w.UnspentOutputs() -} - -// SubscribeWallet implements api.WalletManager. -func (wm *JSONWalletManager) SubscribeWallet(name string, startHeight uint64) error { - wm.mu.Lock() - defer wm.mu.Unlock() - mw, ok := wm.wallets[name] - if !ok { - return errNoWallet - } else if mw.subscribed { - return errors.New("already subscribed") - } - // AddSubscriber applies each block *after* index, but we want to *include* - // the block at startHeight, so subtract one. - // - // NOTE: if subscribing from height 0, we must pass an empty index in order - // to receive the genesis block. - var index types.ChainIndex - if startHeight > 0 { - if index, ok = wm.cm.BestIndex(startHeight - 1); !ok { - return errors.New("invalid height") - } - } - if err := wm.cm.AddSubscriber(mw.w, index); err != nil { - return err - } - mw.subscribed = true - return wm.save() -} - -// NewJSONWalletManager returns a wallet manager that stores wallets in the -// specified directory. -func NewJSONWalletManager(dir string, cm ChainManager) (*JSONWalletManager, error) { - wm := &JSONWalletManager{ - dir: dir, - cm: cm, - wallets: make(map[string]*managedJSONWallet), - } - if err := os.MkdirAll(filepath.Join(dir, "wallets"), 0700); err != nil { - return nil, err - } else if err := wm.load(); err != nil { - return nil, err - } - for name, mw := range wm.wallets { - store, tip, err := NewJSONStore(filepath.Join(dir, "wallets", name+".json")) - if err != nil { - return nil, err - } - if mw.subscribed { - if err := cm.AddSubscriber(store, tip); err != nil { - return nil, err - } - } - mw.w = store - } - return wm, nil -} diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go deleted file mode 100644 index 0340464..0000000 --- a/internal/walletutil/store.go +++ /dev/null @@ -1,403 +0,0 @@ -package walletutil - -import ( - "encoding/json" - "fmt" - "os" - "sync" - - "go.sia.tech/core/chain" - "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" -) - -// An EphemeralStore stores wallet state in memory. -type EphemeralStore struct { - tip types.ChainIndex - addrs map[types.Address]json.RawMessage - sces map[types.SiacoinOutputID]types.SiacoinElement - sfes map[types.SiafundOutputID]types.SiafundElement - events []wallet.Event - mu sync.Mutex -} - -func (s *EphemeralStore) ownsAddress(addr types.Address) bool { - _, ok := s.addrs[addr] - return ok -} - -// Events implements api.Wallet. -func (s *EphemeralStore) Events(offset, limit int) (events []wallet.Event, err error) { - s.mu.Lock() - defer s.mu.Unlock() - if limit == -1 { - limit = len(s.events) - } - if offset > len(s.events) { - offset = len(s.events) - } - if offset+limit > len(s.events) { - limit = len(s.events) - offset - } - // reverse - es := make([]wallet.Event, limit) - for i := range es { - es[i] = s.events[len(s.events)-offset-i-1] - } - return es, nil -} - -// Annotate implements api.Wallet. -func (s *EphemeralStore) Annotate(txns []types.Transaction) (ptxns []wallet.PoolTransaction) { - s.mu.Lock() - defer s.mu.Unlock() - for _, txn := range txns { - ptxn := wallet.Annotate(txn, s.ownsAddress) - if ptxn.Type != "unrelated" { - ptxns = append(ptxns, ptxn) - } - } - return -} - -// UnspentOutputs implements api.Wallet. -func (s *EphemeralStore) UnspentOutputs() (sces []types.SiacoinElement, sfes []types.SiafundElement, err error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, sco := range s.sces { - sces = append(sces, sco) - } - for _, sfo := range s.sfes { - sfes = append(sfes, sfo) - } - return -} - -// Addresses implements api.Wallet. -func (s *EphemeralStore) Addresses() (map[types.Address]json.RawMessage, error) { - s.mu.Lock() - defer s.mu.Unlock() - addrs := make(map[types.Address]json.RawMessage, len(s.addrs)) - for addr, info := range s.addrs { - addrs[addr] = info - } - return addrs, nil -} - -// AddAddress implements api.Wallet. -func (s *EphemeralStore) AddAddress(addr types.Address, info json.RawMessage) error { - s.mu.Lock() - defer s.mu.Unlock() - s.addrs[addr] = info - return nil -} - -// RemoveAddress implements api.Wallet. -func (s *EphemeralStore) RemoveAddress(addr types.Address) error { - s.mu.Lock() - defer s.mu.Unlock() - if _, ok := s.addrs[addr]; !ok { - return nil - } - delete(s.addrs, addr) - - // filter outputs - for scoid, sce := range s.sces { - if sce.SiacoinOutput.Address == addr { - delete(s.sces, scoid) - } - } - for sfoid, sfe := range s.sfes { - if sfe.SiafundOutput.Address == addr { - delete(s.sfes, sfoid) - } - } - - // filter events - relevantContract := func(fc types.FileContract) bool { - for _, sco := range fc.ValidProofOutputs { - if s.ownsAddress(sco.Address) { - return true - } - } - for _, sco := range fc.MissedProofOutputs { - if s.ownsAddress(sco.Address) { - return true - } - } - return false - } - relevantV2Contract := func(fc types.V2FileContract) bool { - return s.ownsAddress(fc.RenterOutput.Address) || s.ownsAddress(fc.HostOutput.Address) - } - relevantEvent := func(e wallet.Event) bool { - switch e := e.Val.(type) { - case *wallet.EventTransaction: - for _, sce := range e.SiacoinInputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - for _, sce := range e.SiacoinOutputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - for _, sfe := range e.SiafundInputs { - if s.ownsAddress(sfe.SiafundElement.SiafundOutput.Address) || - s.ownsAddress(sfe.ClaimElement.SiacoinOutput.Address) { - return true - } - } - for _, sfe := range e.SiafundOutputs { - if s.ownsAddress(sfe.SiafundOutput.Address) { - return true - } - } - for _, fc := range e.FileContracts { - if relevantContract(fc.FileContract.FileContract) || (fc.Revision != nil && relevantContract(*fc.Revision)) { - return true - } - } - for _, fc := range e.V2FileContracts { - if relevantV2Contract(fc.FileContract.V2FileContract) || (fc.Revision != nil && relevantV2Contract(*fc.Revision)) { - return true - } - if fc.Resolution != nil { - switch r := fc.Resolution.(type) { - case *types.V2FileContractFinalization: - if relevantV2Contract(types.V2FileContract(*r)) { - return true - } - case *types.V2FileContractRenewal: - if relevantV2Contract(r.FinalRevision) || relevantV2Contract(r.InitialRevision) { - return true - } - } - } - } - return false - case *wallet.EventMinerPayout: - return s.ownsAddress(e.SiacoinOutput.SiacoinOutput.Address) - case *wallet.EventMissedFileContract: - for _, sce := range e.MissedOutputs { - if s.ownsAddress(sce.SiacoinOutput.Address) { - return true - } - } - return false - default: - panic(fmt.Sprintf("unhandled event type %T", e)) - } - } - - rem := s.events[:0] - for _, e := range s.events { - if relevantEvent(e) { - rem = append(rem, e) - } - } - s.events = rem - return nil -} - -// ProcessChainApplyUpdate implements chain.Subscriber. -func (s *EphemeralStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, _ bool) error { - s.mu.Lock() - defer s.mu.Unlock() - - events := wallet.AppliedEvents(cau.State, cau.Block, cau, s.ownsAddress) - s.events = append(s.events, events...) - - // add/remove outputs - cau.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if s.ownsAddress(sce.SiacoinOutput.Address) { - if spent { - delete(s.sces, types.SiacoinOutputID(sce.ID)) - } else { - sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) - s.sces[types.SiacoinOutputID(sce.ID)] = sce - } - } - }) - cau.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if s.ownsAddress(sfe.SiafundOutput.Address) { - if spent { - delete(s.sfes, types.SiafundOutputID(sfe.ID)) - } else { - sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) - s.sfes[types.SiafundOutputID(sfe.ID)] = sfe - } - } - }) - - // update proofs - for id, sce := range s.sces { - cau.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cau.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - - s.tip = cau.State.Index - return nil -} - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - s.mu.Lock() - defer s.mu.Unlock() - - // terribly inefficient, but not a big deal because reverts are infrequent - numEvents := len(wallet.AppliedEvents(cru.State, cru.Block, cru, s.ownsAddress)) - s.events = s.events[:len(s.events)-numEvents] - - cru.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if s.ownsAddress(sce.SiacoinOutput.Address) { - if !spent { - delete(s.sces, types.SiacoinOutputID(sce.ID)) - } else { - sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) - s.sces[types.SiacoinOutputID(sce.ID)] = sce - } - } - }) - cru.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if s.ownsAddress(sfe.SiafundOutput.Address) { - if !spent { - delete(s.sfes, types.SiafundOutputID(sfe.ID)) - } else { - sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) - s.sfes[types.SiafundOutputID(sfe.ID)] = sfe - } - } - }) - - // update proofs - for id, sce := range s.sces { - cru.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cru.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - - s.tip = cru.State.Index - return nil -} - -// NewEphemeralStore returns a new EphemeralStore. -func NewEphemeralStore() *EphemeralStore { - return &EphemeralStore{ - addrs: make(map[types.Address]json.RawMessage), - sces: make(map[types.SiacoinOutputID]types.SiacoinElement), - sfes: make(map[types.SiafundOutputID]types.SiafundElement), - } -} - -// A JSONStore stores wallet state in memory, backed by a JSON file. -type JSONStore struct { - *EphemeralStore - path string -} - -type persistData struct { - Tip types.ChainIndex - Addresses map[types.Address]json.RawMessage - SiacoinElements map[types.SiacoinOutputID]types.SiacoinElement - SiafundElements map[types.SiafundOutputID]types.SiafundElement - Events []wallet.Event -} - -func (s *JSONStore) save() error { - js, err := json.MarshalIndent(persistData{ - Tip: s.tip, - Addresses: s.addrs, - SiacoinElements: s.sces, - SiafundElements: s.sfes, - Events: s.events, - }, "", " ") - if err != nil { - return err - } - - f, err := os.OpenFile(s.path+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - defer f.Close() - if _, err = f.Write(js); err != nil { - return err - } else if f.Sync(); err != nil { - return err - } else if f.Close(); err != nil { - return err - } else if err := os.Rename(s.path+"_tmp", s.path); err != nil { - return err - } - return nil -} - -func (s *JSONStore) load() error { - f, err := os.Open(s.path) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - defer f.Close() - var p persistData - if err := json.NewDecoder(f).Decode(&p); err != nil { - return err - } - s.tip = p.Tip - s.addrs = p.Addresses - s.sces = p.SiacoinElements - s.sfes = p.SiafundElements - s.events = p.Events - return nil -} - -// ProcessChainApplyUpdate implements chain.Subscriber. -func (s *JSONStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, mayCommit bool) error { - s.EphemeralStore.ProcessChainApplyUpdate(cau, mayCommit) - if mayCommit { - return s.save() - } - return nil -} - -// ProcessChainRevertUpdate implements chain.Subscriber. -func (s *JSONStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error { - s.EphemeralStore.ProcessChainRevertUpdate(cru) - return nil -} - -// AddAddress implements api.Wallet. -func (s *JSONStore) AddAddress(addr types.Address, info json.RawMessage) error { - if err := s.EphemeralStore.AddAddress(addr, info); err != nil { - return err - } - return s.save() -} - -// RemoveAddress implements api.Wallet. -func (s *JSONStore) RemoveAddress(addr types.Address) error { - if err := s.EphemeralStore.RemoveAddress(addr); err != nil { - return err - } - return s.save() -} - -// NewJSONStore returns a new JSONStore. -func NewJSONStore(path string) (*JSONStore, types.ChainIndex, error) { - s := &JSONStore{ - EphemeralStore: NewEphemeralStore(), - path: path, - } - err := s.load() - return s, s.tip, err -} diff --git a/keys/manager.go b/keys/manager.go new file mode 100644 index 0000000..62f061a --- /dev/null +++ b/keys/manager.go @@ -0,0 +1,174 @@ +package keys + +import ( + "crypto/cipher" + "crypto/ed25519" + "errors" + "fmt" + "strings" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/internal/threadgroup" + "golang.org/x/crypto/argon2" + "golang.org/x/crypto/chacha20poly1305" + "lukechampine.com/frand" +) + +var ( + // ErrInvalidSize is returned when a key has an invalid size. + ErrInvalidSize = errors.New("invalid key size") + // ErrNotFound is returned when a signing key is not found. + ErrNotFound = errors.New("not found") + // ErrSaltSet is returned when the key salt is already set. + ErrSaltSet = errors.New("salt already set") + // ErrIncorrectSecret is returned when the secret is incorrect. + ErrIncorrectSecret = errors.New("incorrect secret") +) + +type ( + // A Store saves and loads ed25519 signing keys. + Store interface { + // GetSigningKey returns the encrypted signing key with the given public key. + // If the key is not found, it returns [ErrNotFound]. The key must be + // decrypted before being used. + GetSigningKey(types.PublicKey) ([]byte, error) + // AddSigningKey adds a signing key to the store. If the key already + // exists, nil is returned. The key must be encrypted before being + // stored. + AddSigningKey(pk types.PublicKey, buf []byte) error + // DeleteSigningKey deletes the signing key with the given public key. + // If the key does not exist, it returns [ErrNotFound]. + DeleteSigningKey(types.PublicKey) error + + // KeySalt returns the salt used to derive the key encryption + // key. If no salt has been set, KeySalt should return (nil, nil). + GetKeySalt() ([]byte, error) + + // SetKeySalt sets the salt used to derive the key encryption key. + // If a salt has already been set, [keys.ErrSaltSet] is returned. + SetKeySalt([]byte) error + + // GetBytesForVerify returns random encrypted bytes for verifying + // the encryption key. + GetBytesForVerify() ([]byte, error) + } + + // A Manager is a key-value store for ed25519 signing keys. + Manager struct { + tg *threadgroup.ThreadGroup + + aead cipher.AEAD + store Store + } +) + +// Add adds a key to the manager. If the key is not the correct +// size, it returns [ErrInvalidSize]. +func (m *Manager) Add(sk types.PrivateKey) error { + if len(sk) != ed25519.PrivateKeySize { + return ErrInvalidSize + } + + done, err := m.tg.Add() + if err != nil { + return err + } + defer done() + + n := m.aead.NonceSize() + buf := make([]byte, m.aead.NonceSize(), n+len(sk)+m.aead.Overhead()) + frand.Read(buf) + encrypted := m.aead.Seal(buf, buf, sk, nil) + return m.store.AddSigningKey(sk.PublicKey(), encrypted) +} + +// Sign returns the signature for a hash. If the key is not +// found, it returns [ErrNotFound]. +func (m *Manager) Sign(key types.PublicKey, hash types.Hash256) (types.Signature, error) { + done, err := m.tg.Add() + if err != nil { + return types.Signature{}, err + } + defer done() + + buf, err := m.store.GetSigningKey(key) + if err != nil { + return types.Signature{}, err + } + defer clear(buf) + + sk := make(types.PrivateKey, 0, ed25519.PrivateKeySize) + defer clear(sk) + sk, err = m.aead.Open(sk, buf[:m.aead.NonceSize()], buf[m.aead.NonceSize():], nil) + if err != nil { + return types.Signature{}, fmt.Errorf("failed to decrypt key: %w", err) + } + + if len(sk) != ed25519.PrivateKeySize { + return types.Signature{}, ErrInvalidSize + } + return types.PrivateKey(sk).SignHash(hash), nil +} + +// Delete removes a key from the manager. +func (m *Manager) Delete(key types.PublicKey) error { + done, err := m.tg.Add() + if err != nil { + return err + } + defer done() + return m.store.DeleteSigningKey(key) +} + +// Close closes the manager. +func (m *Manager) Close() error { + m.tg.Stop() + return nil +} + +// NewManager creates a new key manager. If the store contains +// encrypted keys, the secret must match the secret used to encrypt +// the existing keys. If the secret is incorrect, NewManager returns +// [ErrIncorrectSecret]. +// +// Keys are encrypted using ChaCha20-Poly1305 with a key derived from +// the secret using Argon2ID. +func NewManager(store Store, secret string) (*Manager, error) { + salt, err := store.GetKeySalt() + if err != nil { + return nil, fmt.Errorf("failed to get key salt: %w", err) + } else if len(salt) == 0 { + salt = frand.Bytes(32) + if err := store.SetKeySalt(salt); err != nil { + return nil, fmt.Errorf("failed to set key salt: %w", err) + } + } + + encryptionKey := argon2.IDKey([]byte(secret), salt, 3, 64*1024, 4, 32) + aead, err := chacha20poly1305.NewX(encryptionKey) + if err != nil { + return nil, fmt.Errorf("failed to create AEAD: %w", err) + } + + buf, err := store.GetBytesForVerify() + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, fmt.Errorf("failed to get bytes for verify: %w", err) + } else if err == nil { + defer clear(buf) + + decrypted, err := aead.Open(nil, buf[:aead.NonceSize()], buf[aead.NonceSize():], nil) + if err != nil { + if strings.Contains(err.Error(), "message authentication failed") { + return nil, ErrIncorrectSecret + } + return nil, fmt.Errorf("failed to verify encryption key: %w", err) + } + defer clear(decrypted) + } + + return &Manager{ + aead: aead, + store: store, + tg: threadgroup.New(), + }, nil +} diff --git a/keys/manager_test.go b/keys/manager_test.go new file mode 100644 index 0000000..d049a02 --- /dev/null +++ b/keys/manager_test.go @@ -0,0 +1,82 @@ +package keys_test + +import ( + "errors" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" + "go.sia.tech/walletd/persist/sqlite" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +func TestKeyManager(t *testing.T) { + store, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zap.NewNop()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + m, err := keys.NewManager(store, "foo") + if err != nil { + t.Fatal(err) + } + defer m.Close() + + sk := types.GeneratePrivateKey() + + if err := m.Add(sk); err != nil { + t.Fatal(err) + } + + // try to add it again + if err := m.Add(sk); err != nil { + t.Fatal(err) + } + + hash := types.Hash256(frand.Entropy256()) + + sig, err := m.Sign(sk.PublicKey(), hash) + if err != nil { + t.Fatal(err) + } else if !sk.PublicKey().VerifyHash(hash, sig) { + t.Fatal("signature failed to verify") + } + + // try to sign with an unknown key + _, err = m.Sign(types.GeneratePrivateKey().PublicKey(), hash) + if !errors.Is(err, keys.ErrNotFound) { + t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) + } + + if err := m.Close(); err != nil { + t.Fatal(err) + } + + _, err = keys.NewManager(store, "foobar") + if !errors.Is(err, keys.ErrIncorrectSecret) { + t.Fatalf("expected %v, got %v", keys.ErrIncorrectSecret, err) + } + + m, err = keys.NewManager(store, "foo") + if err != nil { + t.Fatal(err) + } + defer m.Close() + + sig, err = m.Sign(sk.PublicKey(), hash) + if err != nil { + t.Fatal(err) + } else if !sk.PublicKey().VerifyHash(hash, sig) { + t.Fatal("signature failed to verify") + } + + // delete the key + if err := m.Delete(sk.PublicKey()); err != nil { + t.Fatal(err) + } else if _, err := m.Sign(sk.PublicKey(), hash); !errors.Is(err, keys.ErrNotFound) { + t.Fatalf("expected %v, got %v", keys.ErrNotFound, err) + } +} diff --git a/knope.toml b/knope.toml new file mode 100644 index 0000000..1034ad0 --- /dev/null +++ b/knope.toml @@ -0,0 +1,56 @@ +[package] +changelog = "CHANGELOG.md" +versioned_files = ["go.mod"] +ignore_go_major_versioning = true + +[[workflows]] +name = "document-change" + +[[workflows.steps]] +type = "CreateChangeFile" + +[[workflows]] +name = "prepare-release" + +[[workflows.steps]] +type = "Command" +command = "git switch -c release" + +[[workflows.steps]] +type = "PrepareRelease" +ignore_conventional_commits = true + +[[workflows.steps]] +type = "Command" +command = "git commit -m \"chore: prepare release $version\"" +variables = { "$version" = "Version" } + +[[workflows.steps]] +type = "Command" +command = "git push --force --set-upstream origin release" + +[workflows.steps.variables] +"$version" = "Version" + +[[workflows.steps]] +type = "CreatePullRequest" +base = "master" + +[workflows.steps.title] +template = "chore: prepare release $version" +variables = { "$version" = "Version" } + +[workflows.steps.body] +template = "This PR was created automatically. Merging it will finalize the changelog for $version\n\n$changelog" +variables = { "$changelog" = "ChangelogEntry", "$version" = "Version" } + +# Do not enable releases, just changelogs for now. +# [[workflows]] +# name = "release" +# +# [[workflows.steps]] +# type = "Release" + +[github] +owner = "SiaFoundation" +repo = "walletd" diff --git a/persist/sqlite/addresses.go b/persist/sqlite/addresses.go new file mode 100644 index 0000000..b5c9056 --- /dev/null +++ b/persist/sqlite/addresses.go @@ -0,0 +1,297 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// AddressBalance returns the balance of a single address. +func (s *Store) AddressBalance(address types.Address) (balance wallet.Balance, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1` + err := tx.QueryRow(query, encode(address)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) + if errors.Is(err, sql.ErrNoRows) { + balance = wallet.Balance{} + return nil + } + return err + }) + return +} + +func getAddressEvents(tx *txn, address types.Address, offset, limit int) (eventIDs []int64, err error) { + const query = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +WHERE sa.sia_address = $1 +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT $2 OFFSET $3;` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + eventIDs = append(eventIDs, id) + } + return eventIDs, rows.Err() +} + +// AddressEvents returns the events of a single address. +func (s *Store) AddressEvents(address types.Address, offset, limit int) (events []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + dbIDs, err := getAddressEvents(tx, address, offset, limit) + if err != nil { + return err + } + + events, err = getEventsByID(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } + + for i := range events { + events[i].Relevant = []types.Address{address} + } + return nil + }) + return +} + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (s *Store) AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address=$1 AND se.maturity_height <= $2 AND se.spent_index_id IS NULL + LIMIT $3 OFFSET $4` + + rows, err := tx.Query(query, encode(address), index.Height, limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siacoin, err := scanSiacoinElement(rows) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + + siacoins = append(siacoins, siacoin) + } + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siacoins)) + for i, se := range siacoins { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siacoins[i].StateElement.MerkleProof = proof + } + } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + return nil + }) + return +} + +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (s *Store) AddressSiafundOutputs(address types.Address, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE sa.sia_address = $1 AND se.spent_index_id IS NULL + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, encode(address), limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siafund, err := scanSiafundElement(rows) + if err != nil { + return fmt.Errorf("failed to scan siafund element: %w", err) + } + siafunds = append(siafunds, siafund) + } + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siafund elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siafunds)) + for i, se := range siafunds { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siafunds[i].StateElement.MerkleProof = proof + } + } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + return nil + }) + return +} + +// AnnotateV1Events annotates a list of unconfirmed transactions with +// relevant addresses and siacoin/siafund elements. +func (s *Store) AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + siacoinElementStmt, err := tx.Prepare(`SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin statement: %w", err) + } + defer siacoinElementStmt.Close() + + siacoinElementCache := make(map[types.SiacoinOutputID]types.SiacoinElement) + fetchSiacoinElement := func(id types.SiacoinOutputID) (types.SiacoinElement, error) { + if se, ok := siacoinElementCache[id]; ok { + return se, nil + } + + se, err := scanSiacoinElement(siacoinElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiacoinElement{}, fmt.Errorf("failed to fetch siacoin element: %w", err) + } + siacoinElementCache[id] = se + return se, nil + } + + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siafund statement: %w", err) + } + defer siafundElementStmt.Close() + + siafundElementCache := make(map[types.SiafundOutputID]types.SiafundElement) + fetchSiafundElement := func(id types.SiafundOutputID) (types.SiafundElement, error) { + if se, ok := siafundElementCache[id]; ok { + return se, nil + } + + se, err := scanSiafundElement(siafundElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiafundElement{}, fmt.Errorf("failed to fetch siafund element: %w", err) + } + siafundElementCache[id] = se + return se, nil + } + + addEvent := func(id types.Hash256, data wallet.EventData) { + annotated = append(annotated, wallet.Event{ + ID: id, + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height, + Type: wallet.EventTypeV1Transaction, + Data: data, + }) + } + + for _, txn := range v1 { + var relevant bool + ev := wallet.EventV1Transaction{ + Transaction: txn, + } + + for _, input := range txn.SiacoinInputs { + // fetch the siacoin element + sce, err := fetchSiacoinElement(input.ParentID) + if errors.Is(err, sql.ErrNoRows) { + continue // ignore elements that are not found + } else if err != nil { + return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + } + ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) + relevant = true + } + + for i, output := range txn.SiacoinOutputs { + sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(i), + StateElement: types.StateElement{ + LeafIndex: types.UnassignedLeafIndex, + }, + SiacoinOutput: output, + } + siacoinElementCache[sce.ID] = sce + relevant = true + } + + for _, input := range txn.SiafundInputs { + // fetch the siafund element + sfe, err := fetchSiafundElement(input.ParentID) + if errors.Is(err, sql.ErrNoRows) { + continue // ignore elements that are not found + } else if err != nil { + return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + } + ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) + relevant = true + } + + for i, output := range txn.SiafundOutputs { + sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(i), + StateElement: types.StateElement{ + LeafIndex: types.UnassignedLeafIndex, + }, + SiafundOutput: output, + } + siafundElementCache[sfe.ID] = sfe + relevant = true + } + + if !relevant { + continue + } + + addEvent(types.Hash256(txn.ID()), ev) + } + return nil + }) + return +} diff --git a/persist/sqlite/consensus.go b/persist/sqlite/consensus.go new file mode 100644 index 0000000..cf1dcce --- /dev/null +++ b/persist/sqlite/consensus.go @@ -0,0 +1,1436 @@ +package sqlite + +import ( + "bytes" + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" +) + +type updateTx struct { + indexMode wallet.IndexMode + + tx *txn + relevantAddresses map[types.Address]bool +} + +type addressRef struct { + ID int64 + Balance wallet.Balance +} + +type stateElement struct { + ID types.Hash256 + types.StateElement +} + +func (ut *updateTx) UpdateStateElementProofs(update wallet.ProofUpdater) error { + if ut.indexMode == wallet.IndexModeFull { + panic("UpdateStateElementProofs called in full index mode") + } + + se, err := getSiacoinStateElements(ut.tx) + if err != nil { + return fmt.Errorf("failed to get siacoin state elements: %w", err) + } + for i := range se { + update.UpdateElementProof(&se[i].StateElement) + } + if err := updateSiacoinStateElements(ut.tx, se); err != nil { + return fmt.Errorf("failed to update siacoin state elements: %w", err) + } + + sfe, err := getSiafundStateElements(ut.tx) + if err != nil { + return fmt.Errorf("failed to get siafund state elements: %w", err) + } + for i := range sfe { + update.UpdateElementProof(&sfe[i].StateElement) + } + if err := updateSiafundStateElements(ut.tx, sfe); err != nil { + return fmt.Errorf("failed to update siafund state elements: %w", err) + } + return nil +} + +func (ut *updateTx) UpdateStateTree(changes []wallet.TreeNodeUpdate) error { + if ut.indexMode != wallet.IndexModeFull { + panic("UpdateStateTree called in personal index mode") + } + + stmt, err := ut.tx.Prepare(`INSERT INTO state_tree (row, column, value) VALUES ($1, $2, $3) ON CONFLICT (row, column) DO UPDATE SET value=EXCLUDED.value`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, change := range changes { + _, err := stmt.Exec(change.Row, change.Column, encode(change.Hash)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + } + return nil +} + +func (ut *updateTx) AddressRelevant(addr types.Address) (bool, error) { + if ut.indexMode == wallet.IndexModeFull { + return true, nil + } + + if relevant, ok := ut.relevantAddresses[addr]; ok { + return relevant, nil + } + + var id int64 + err := ut.tx.QueryRow(`SELECT id FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + ut.relevantAddresses[addr] = false + return false, nil + } else if err != nil { + return false, fmt.Errorf("failed to query address: %w", err) + } + ut.relevantAddresses[addr] = true + return ut.relevantAddresses[addr], nil +} + +func (ut *updateTx) AddressBalance(addr types.Address) (balance wallet.Balance, err error) { + err = ut.tx.QueryRow(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE sia_address=$1`, encode(addr)).Scan(decode(&balance.Siacoins), decode(&balance.ImmatureSiacoins), &balance.Siafunds) + return +} + +func (ut *updateTx) ApplyIndex(index types.ChainIndex, state wallet.AppliedState) error { + tx := ut.tx + log := tx.log.Named("ApplyIndex").With(zap.Stringer("blockID", index.ID), zap.Uint64("height", index.Height)) + + if err := revertOrphans(tx, index, log.Named("revertOrphans")); err != nil { + return fmt.Errorf("failed to revert orphans: %w", err) + } + + if err := applyMatureSiacoinBalance(tx, index, log.Named("applyMatureSiacoinBalance")); err != nil { + return fmt.Errorf("failed to apply mature siacoin balance: %w", err) + } + + var indexID int64 + if err := tx.QueryRow(`INSERT INTO chain_indices (block_id, height) VALUES ($1, $2) ON CONFLICT (block_id) DO UPDATE SET height=height RETURNING id`, encode(index.ID), index.Height).Scan(&indexID); err != nil { + return fmt.Errorf("failed to insert chain index: %w", err) + } + + if err := addEvents(tx, state.Events, indexID); err != nil { + return fmt.Errorf("failed to add events: %w", err) + } + + if err := spendSiacoinElements(tx, state.SpentSiacoinElements, indexID); err != nil { + return fmt.Errorf("failed to spend siacoin elements: %w", err) + } else if err := addSiacoinElements(tx, state.CreatedSiacoinElements, indexID, ut.indexMode, log.Named("addSiacoinElements")); err != nil { + return fmt.Errorf("failed to add siacoin elements: %w", err) + } + + if err := spendSiafundElements(tx, state.SpentSiafundElements, indexID); err != nil { + return fmt.Errorf("failed to spend siafund elements: %w", err) + } else if err := addSiafundElements(tx, state.CreatedSiafundElements, indexID, ut.indexMode, log.Named("addSiafundElements")); err != nil { + return fmt.Errorf("failed to add siafund elements: %w", err) + } + + return nil +} + +func (ut *updateTx) RevertIndex(index types.ChainIndex, state wallet.RevertedState) error { + tx := ut.tx + + if err := revertSpentSiacoinElements(tx, state.UnspentSiacoinElements); err != nil { + return fmt.Errorf("failed to revert spent siacoin elements: %w", err) + } else if err := removeSiacoinElements(tx, state.DeletedSiacoinElements); err != nil { + return fmt.Errorf("failed to remove siacoin elements: %w", err) + } + + if err := revertSpentSiafundElements(tx, state.UnspentSiafundElements); err != nil { + return fmt.Errorf("failed to revert spent siafund elements: %w", err) + } else if err := removeSiafundElements(tx, state.DeletedSiafundElements); err != nil { + return fmt.Errorf("failed to remove siafund elements: %w", err) + } + + if err := revertEvents(tx, index); err != nil { + return fmt.Errorf("failed to revert events: %w", err) + } else if err := revertMatureSiacoinBalance(tx, index); err != nil { + return fmt.Errorf("failed to revert mature siacoin balance: %w", err) + } + return nil +} + +// UpdateChainState implements chain.Subscriber +func (s *Store) UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error { + if len(applied) == 0 && len(reverted) == 0 { + return nil + } + + log := s.log.Named("UpdateChainState").With(zap.Int("revertedUpdates", len(reverted)), zap.Int("appliedUpdates", len(applied))) + return s.transaction(func(tx *txn) error { + utx := &updateTx{ + indexMode: s.indexMode, + + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } + + if err := wallet.UpdateChainState(utx, reverted, applied, s.indexMode, log); err != nil { + return err + } + + var state consensus.State + switch { + case len(applied) > 0: + state = applied[len(applied)-1].State + case len(reverted) > 0: + state = reverted[len(reverted)-1].State + } + + if err := setGlobalState(tx, state.Index, state.Elements.NumLeaves); err != nil { + return fmt.Errorf("failed to set last committed index: %w", err) + } + + // skip pruning if there are no applied updates + if len(applied) == 0 { + return nil + } + + if state.Index.Height > spentElementRetentionBlocks { + pruneHeight := state.Index.Height - spentElementRetentionBlocks + + siacoins, err := pruneSpentSiacoinElements(tx, pruneHeight) + if err != nil { + return fmt.Errorf("failed to cleanup siacoin elements: %w", err) + } + + siafunds, err := pruneSpentSiafundElements(tx, pruneHeight) + if err != nil { + return fmt.Errorf("failed to cleanup siafund elements: %w", err) + } + log.Debug("pruned elements", zap.Int64("siacoins", siacoins), zap.Int64("siafunds", siafunds), zap.Uint64("pruneHeight", pruneHeight)) + } + return nil + }) +} + +// LastCommittedIndex returns the last chain index that was committed. +func (s *Store) LastCommittedIndex() (index types.ChainIndex, err error) { + err = s.db.QueryRow(`SELECT last_indexed_height, last_indexed_id FROM global_settings`).Scan(&index.Height, decode(&index.ID)) + return +} + +// ResetLastIndex resets the last indexed tip to trigger a full rescan. +func (s *Store) ResetLastIndex() error { + _, err := s.db.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1`, encode(types.BlockID{})) + return err +} + +// IndexMode returns the current index mode. +func (s *Store) IndexMode() (wallet.IndexMode, error) { + var mode wallet.IndexMode + err := s.db.QueryRow(`SELECT index_mode FROM global_settings`).Scan(&mode) + return mode, err +} + +// SetIndexMode sets the index mode. If the index mode is already set, this +// function will return an error. +func (s *Store) SetIndexMode(mode wallet.IndexMode) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`UPDATE global_settings SET index_mode=$1 WHERE index_mode IS NULL`, mode) + if err != nil { + return fmt.Errorf("failed to set index mode: %w", err) + } + + // check that the index mode was set + var existingMode wallet.IndexMode + err = tx.QueryRow(`SELECT index_mode FROM global_settings`).Scan(&existingMode) + if err != nil { + return fmt.Errorf("failed to query index mode: %w", err) + } else if existingMode != mode { + return fmt.Errorf("cannot change index mode from %v to %v", existingMode, mode) + } + s.indexMode = mode // this is a bit annoying + return nil + }) +} + +// ResetChainState deletes all blockchain state from the database. +func (s *Store) ResetChainState() error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`UPDATE sia_addresses SET siacoin_balance=$1, siafund_balance=0, immature_siacoin_balance=$1`, encode(types.ZeroCurrency)) + if err != nil { + return fmt.Errorf("failed to reset sia addresses: %w", err) + } + + _, err = tx.Exec(`DELETE FROM siacoin_elements`) + if err != nil { + return fmt.Errorf("failed to delete siacoin elements: %w", err) + } + + _, err = tx.Exec(`DELETE FROM siafund_elements`) + if err != nil { + return fmt.Errorf("failed to delete siafund elements: %w", err) + } + + _, err = tx.Exec(`DELETE FROM state_tree`) + if err != nil { + return fmt.Errorf("failed to delete state tree: %w", err) + } + + _, err = tx.Exec(`DELETE FROM event_addresses`) + if err != nil { + return fmt.Errorf("failed to delete event addresses: %w", err) + } + + _, err = tx.Exec(`DELETE FROM events`) + if err != nil { + return fmt.Errorf("failed to delete events: %w", err) + } + + _, err = tx.Exec(`DELETE FROM chain_indices`) + if err != nil { + return fmt.Errorf("failed to delete chain indices: %w", err) + } + + _, err = tx.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1, element_num_leaves=0`, encode(types.BlockID{})) + if err != nil { + return fmt.Errorf("failed to reset global settings: %w", err) + } + return nil + }) +} + +func getSiacoinStateElements(tx *txn) ([]stateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siacoin_elements` + rows, err := tx.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var elements []stateElement + for rows.Next() { + var se stateElement + if err := rows.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + elements = append(elements, se) + } + return elements, rows.Err() +} + +func getSiafundStateElements(tx *txn) ([]stateElement, error) { + const query = `SELECT id, leaf_index, merkle_proof FROM siafund_elements` + rows, err := tx.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() + + var elements []stateElement + for rows.Next() { + var se stateElement + if err := rows.Scan(decode(&se.ID), &se.LeafIndex, decode(&se.MerkleProof)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + elements = append(elements, se) + } + return elements, rows.Err() +} + +func updateSiafundStateElements(tx *txn, elements []stateElement) error { + if len(elements) == 0 { + return nil + } + const query = `UPDATE siafund_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3` + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, se := range elements { + res, err := stmt.Exec(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func updateSiacoinStateElements(tx *txn, elements []stateElement) error { + if len(elements) == 0 { + return nil + } + const query = `UPDATE siacoin_elements SET merkle_proof=$1, leaf_index=$2 WHERE id=$3` + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + for _, se := range elements { + res, err := stmt.Exec(encode(se.MerkleProof), se.LeafIndex, encode(se.ID)) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func scanAddress(s scanner) (ab addressRef, err error) { + err = s.Scan(&ab.ID, decode(&ab.Balance.Siacoins), decode(&ab.Balance.ImmatureSiacoins), &ab.Balance.Siafunds) + return +} + +func applyMatureSiacoinBalance(tx *txn, index types.ChainIndex, log *zap.Logger) error { + log = log.With(zap.Uint64("maturityHeight", index.Height)) + const query = `SELECT id, address_id, siacoin_value +FROM siacoin_elements +WHERE maturity_height=$1 AND matured=false AND spent_index_id IS NULL` + rows, err := tx.Query(query, index.Height) + if err != nil { + return fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var matured []types.SiacoinOutputID + balanceDelta := make(map[int64]types.Currency) + for rows.Next() { + var outputID types.SiacoinOutputID + var addressID int64 + var value types.Currency + + if err := rows.Scan(decode(&outputID), &addressID, decode(&value)); err != nil { + return fmt.Errorf("failed to scan siacoin balance: %w", err) + } + balanceDelta[addressID] = balanceDelta[addressID].Add(value) + matured = append(matured, outputID) + log.Debug("matured siacoin output", zap.Stringer("outputID", outputID), zap.Int64("addressID", addressID), zap.Stringer("value", value)) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) + } + + updateMaturedStmt, err := tx.Prepare(`UPDATE siacoin_elements SET matured=true WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateMaturedStmt.Close() + + getAddressBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getAddressBalanceStmt.Close() + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addressID, delta := range balanceDelta { + var balance, immatureBalance types.Currency + err := getAddressBalanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) + if err != nil { + return fmt.Errorf("failed to get address balance: %w", err) + } + balance = balance.Add(delta) + immatureBalance = immatureBalance.Sub(delta) + + res, err := updateAddressBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + + for _, id := range matured { + res, err := updateMaturedStmt.Exec(encode(id)) + if err != nil { + return fmt.Errorf("failed to update matured: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func revertMatureSiacoinBalance(tx *txn, index types.ChainIndex) error { + const query = `SELECT se.id, se.address_id, se.siacoin_value + FROM siacoin_elements se + WHERE maturity_height=$1 AND matured=true AND spent_index_id IS NULL` + rows, err := tx.Query(query, index.Height) + if err != nil { + return fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + var matured []types.SiacoinOutputID + balanceDelta := make(map[int64]types.Currency) + for rows.Next() { + var outputID types.SiacoinOutputID + var addressID int64 + var value types.Currency + + if err := rows.Scan(decode(&outputID), &addressID, decode(&value)); err != nil { + return fmt.Errorf("failed to scan siacoin balance: %w", err) + } + balanceDelta[addressID] = balanceDelta[addressID].Add(value) + matured = append(matured, outputID) + } + + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to scan siacoin elements: %w", err) + } + + updateMaturedStmt, err := tx.Prepare(`UPDATE siacoin_elements SET matured=false WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateMaturedStmt.Close() + + getAddressBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getAddressBalanceStmt.Close() + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addressID, delta := range balanceDelta { + var balance, immatureBalance types.Currency + err := getAddressBalanceStmt.QueryRow(addressID).Scan(decode(&balance), decode(&immatureBalance)) + if err != nil { + return fmt.Errorf("failed to get address balance: %w", err) + } + + balance = balance.Sub(delta) + immatureBalance = immatureBalance.Add(delta) + + res, err := updateAddressBalanceStmt.Exec(encode(balance), encode(immatureBalance), addressID) + if err != nil { + return fmt.Errorf("failed to update address balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + + for _, id := range matured { + res, err := updateMaturedStmt.Exec(encode(id)) + if err != nil { + return fmt.Errorf("failed to update matured: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func addSiacoinElements(tx *txn, elements []types.SiacoinElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siacoin_elements WHERE id=$1)`) + if err != nil { + return fmt.Errorf("failed to prepare exists statement: %w", err) + } + defer existsStmt.Close() + + // ignore elements already in the database. + insertStmt, err := tx.Prepare(`INSERT INTO siacoin_elements (id, siacoin_value, merkle_proof, leaf_index, maturity_height, address_id, matured, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) + if err != nil { + return fmt.Errorf("failed to prepare insert statement: %w", err) + } + defer insertStmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var exists bool + err = existsStmt.QueryRow(encode(se.ID)).Scan(&exists) + if err != nil { + return fmt.Errorf("failed to check if siacoin element exists: %w", err) + } + + // in full index mode, Merkle proofs are stored in the state tree table + // rather than per element. + if indexMode == wallet.IndexModeFull { + se.StateElement.MerkleProof = nil + } + + _, err = insertStmt.Exec(encode(se.ID), encode(se.SiacoinOutput.Value), encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, se.MaturityHeight, addrRef.ID, se.MaturityHeight == 0, indexID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } + // skip balance update if the element already exists + if exists { + log.Debug("updated siacoin element", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value)) + continue + } + + balance := balanceChanges[addrRef.ID] + if se.MaturityHeight == 0 { + balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) + log.Debug("added siacoin output", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value)) + } else { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(se.SiacoinOutput.Value) + log.Debug("added immature siacoin output", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiacoinOutput.Address), zap.Stringer("value", se.SiacoinOutput.Value), zap.Uint64("maturityHeight", se.MaturityHeight)) + } + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), encode(balance.ImmatureSiacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func removeSiacoinElements(tx *txn, elements []types.SiacoinElement) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + stmt, err := tx.Prepare(`DELETE FROM siacoin_elements WHERE id=$1 RETURNING id, matured`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + var matured bool + err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy), &matured) + if err != nil { + return fmt.Errorf("failed to delete element %q: %w", se.ID, err) + } + + balance := balanceChanges[addrRef.ID] + if matured { + balance.Siacoins = balance.Siacoins.Sub(se.SiacoinOutput.Value) + } else { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Sub(se.SiacoinOutput.Value) + } + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), encode(balance.ImmatureSiacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func revertSpentSiacoinElements(tx *txn, elements []types.SiacoinElement) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siacoins = balance.Siacoins.Add(se.SiacoinOutput.Value) + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func spendSiacoinElements(tx *txn, elements []wallet.SpentSiacoinElement, indexID int64) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + getEventIDStmt, err := tx.Prepare(`SELECT id FROM events WHERE event_id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getEventIDStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siacoin_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiacoinOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var eventDBID int64 + if err := getEventIDStmt.QueryRow(encode(se.EventID)).Scan(&eventDBID); err != nil { + return fmt.Errorf("failed to get event ID: %w", err) + } + + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, eventDBID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siacoins = balance.Siacoins.Sub(se.SiacoinOutput.Value) + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(encode(balance.Siacoins), addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func addSiafundElements(tx *txn, elements []types.SiafundElement, indexID int64, indexMode wallet.IndexMode, log *zap.Logger) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + existsStmt, err := tx.Prepare(`SELECT EXISTS(SELECT 1 FROM siafund_elements WHERE id=$1)`) + if err != nil { + return fmt.Errorf("failed to prepare exists statement: %w", err) + } + defer existsStmt.Close() + + insertStmt, err := tx.Prepare(`INSERT INTO siafund_elements (id, siafund_value, merkle_proof, leaf_index, claim_start, address_id, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET leaf_index=EXCLUDED.leaf_index, merkle_proof=EXCLUDED.merkle_proof`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer insertStmt.Close() + + balanceChanges := make(map[int64]uint64) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiafundOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance.Siafunds + } + + var exists bool + if err := existsStmt.QueryRow(encode(se.ID)).Scan(&exists); err != nil { + return fmt.Errorf("failed to check if siafund element exists: %w", err) + } + + // in full index mode, Merkle proofs are stored in the state tree table + // rather than per element. + if indexMode == wallet.IndexModeFull { + se.StateElement.MerkleProof = nil + } + + _, err = insertStmt.Exec(encode(se.ID), se.SiafundOutput.Value, encode(se.StateElement.MerkleProof), se.StateElement.LeafIndex, encode(se.ClaimStart), addrRef.ID, indexID) + if err != nil { + return fmt.Errorf("failed to execute statement: %w", err) + } else if exists { + // skip balance update if the element already exists + log.Debug("updated siafund element", zap.Stringer("id", se.ID), zap.Stringer("address", se.SiafundOutput.Address), zap.Uint64("value", se.SiafundOutput.Value)) + continue + } + balanceChanges[addrRef.ID] += se.SiafundOutput.Value + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func removeSiafundElements(tx *txn, elements []types.SiafundElement) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + stmt, err := tx.Prepare(`DELETE FROM siafund_elements WHERE id=$1 RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]uint64) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiafundOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance.Siafunds + } + + var dummy types.Hash256 + err = stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)) + if err != nil { + return fmt.Errorf("failed to delete element %q: %w", se.ID, err) + } + + if balanceChanges[addrRef.ID] < se.SiafundOutput.Value { + panic("siafund balance cannot be negative") + } + balanceChanges[addrRef.ID] -= se.SiafundOutput.Value + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$2`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func spendSiafundElements(tx *txn, elements []wallet.SpentSiafundElement, indexID int64) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + getEventIDStmt, err := tx.Prepare(`SELECT id FROM events WHERE event_id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer getEventIDStmt.Close() + + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=$1, spent_event_id=$2 WHERE id=$3 AND spent_index_id IS NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiafundOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var eventDBID int64 + if err := getEventIDStmt.QueryRow(encode(se.EventID)).Scan(&eventDBID); err != nil { + return fmt.Errorf("failed to get event ID: %w", err) + } + + var dummy types.Hash256 + if err := stmt.QueryRow(indexID, eventDBID, encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + if balance.Siafunds < se.SiafundOutput.Value { + panic("siafund balance cannot be negative") + } + balance.Siafunds -= se.SiafundOutput.Value + + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance.Siafunds, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func revertSpentSiafundElements(tx *txn, elements []types.SiafundElement) error { + if len(elements) == 0 { + return nil + } + + addressRefStmt, done, err := addressRefStmt(tx) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer done() + + stmt, err := tx.Prepare(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id=$1 AND spent_index_id IS NOT NULL RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + balanceChanges := make(map[int64]wallet.Balance) + for _, se := range elements { + addrRef, err := addressRefStmt(se.SiafundOutput.Address) + if err != nil { + return fmt.Errorf("failed to query address: %w", err) + } else if _, ok := balanceChanges[addrRef.ID]; !ok { + balanceChanges[addrRef.ID] = addrRef.Balance + } + + var dummy types.Hash256 + if err := stmt.QueryRow(encode(se.ID)).Scan(decode(&dummy)); err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } else if errors.Is(err, sql.ErrNoRows) { + continue // skip if the element does not exist + } + + balance := balanceChanges[addrRef.ID] + balance.Siafunds += se.SiafundOutput.Value + balanceChanges[addrRef.ID] = balance + } + + if len(balanceChanges) == 0 { + return nil + } + + updateAddressBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siafund_balance=$1 WHERE id=$3`) + if err != nil { + return fmt.Errorf("failed to prepare update balance statement: %w", err) + } + defer updateAddressBalanceStmt.Close() + + for addrID, balance := range balanceChanges { + res, err := updateAddressBalanceStmt.Exec(balance.Siafunds, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + return nil +} + +func addEvents(tx *txn, events []wallet.Event, indexID int64) error { + if len(events) == 0 { + return nil + } + + insertEventStmt, err := tx.Prepare(`INSERT INTO events (event_id, maturity_height, date_created, event_type, event_data, chain_index_id) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (event_id) DO NOTHING RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare event statement: %w", err) + } + defer insertEventStmt.Close() + + addrStmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $2, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id`) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + relevantAddrStmt, err := tx.Prepare(`INSERT INTO event_addresses (event_id, address_id, event_maturity_height) VALUES ($1, $2, $3) ON CONFLICT (event_id, address_id) DO NOTHING`) + if err != nil { + return fmt.Errorf("failed to prepare relevant address statement: %w", err) + } + defer relevantAddrStmt.Close() + + var buf bytes.Buffer + enc := types.NewEncoder(&buf) + for _, event := range events { + buf.Reset() + ev, ok := event.Data.(types.EncoderTo) + if !ok { + panic("event data does not implement types.EncoderTo") // developer error + } + ev.EncodeTo(enc) + enc.Flush() + + var eventID int64 + err = insertEventStmt.QueryRow(encode(event.ID), event.MaturityHeight, encode(event.Timestamp), event.Type, buf.Bytes(), indexID).Scan(&eventID) + if errors.Is(err, sql.ErrNoRows) { + continue // skip if the event already exists + } else if err != nil { + return fmt.Errorf("failed to add event: %w", err) + } + + used := make(map[types.Address]bool) + for _, addr := range event.Relevant { + if used[addr] { + continue + } + + var addressID int64 + err = addrStmt.QueryRow(encode(addr), encode(types.ZeroCurrency)).Scan(&addressID) + if err != nil { + return fmt.Errorf("failed to get address: %w", err) + } + + _, err = relevantAddrStmt.Exec(eventID, addressID, event.MaturityHeight) + if err != nil { + return fmt.Errorf("failed to add relevant address: %w", err) + } + + used[addr] = true + } + } + return nil +} + +// RevertEvents reverts any events that were added by the index +func revertEvents(tx *txn, index types.ChainIndex) error { + const query = `DELETE FROM events WHERE chain_index_id IN (SELECT id FROM chain_indices WHERE block_id=$1 AND height=$2)` + _, err := tx.Exec(query, encode(index.ID), index.Height) + return err +} + +func revertSpentOrphanedSiacoinElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]wallet.Balance, error) { + rows, err := tx.Query(`UPDATE siacoin_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id IN (SELECT se.id FROM siacoin_elements se +INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING address_id, siacoin_value`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]wallet.Balance) + for rows.Next() { + var addrID int64 + var value types.Currency + + if err := rows.Scan(&addrID, decode(&value)); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + + balance := balances[addrID] + balance.Siacoins = balance.Siacoins.Add(value) + balances[addrID] = balance + log.Debug("reverting spent orphaned siacoin element", zap.Stringer("value", value)) + } + return balances, rows.Err() +} + +func deleteOrphanedSiacoinElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]wallet.Balance, error) { + rows, err := tx.Query(`DELETE FROM siacoin_elements WHERE id IN (SELECT se.id FROM siacoin_elements se +INNER JOIN chain_indices ci ON (ci.id=se.chain_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siacoin_value, matured, spent_index_id IS NOT NULL`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query siacoin elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]wallet.Balance) + for rows.Next() { + var outputID types.SiacoinOutputID + var addrID int64 + var value types.Currency + var matured bool + var spent bool + + if err := rows.Scan(decode(&outputID), &addrID, decode(&value), &matured, &spent); err != nil { + return nil, fmt.Errorf("failed to scan siacoin element: %w", err) + } + + balance := balances[addrID] + if !matured { + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(value) + } else if !spent { + balance.Siacoins = balance.Siacoins.Add(value) + } + balances[addrID] = balance + log.Debug("deleting orphaned siacoin element", zap.Stringer("id", outputID), zap.Stringer("value", value), zap.Bool("matured", matured), zap.Bool("spent", spent)) + } + return balances, rows.Err() +} + +func revertSpentOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { + rows, err := tx.Query(`UPDATE siafund_elements SET spent_index_id=NULL, spent_event_id=NULL WHERE id IN (SELECT se.id FROM siafund_elements se +INNER JOIN chain_indices ci ON (ci.id=se.spent_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siafund_value`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]uint64) + for rows.Next() { + var outputID types.SiafundOutputID + var addrID int64 + var value uint64 + + if err := rows.Scan(decode(&outputID), &addrID, value); err != nil { + return nil, fmt.Errorf("failed to scan siafund element: %w", err) + } + + balance := balances[addrID] + balance += value + balances[addrID] = balance + log.Debug("reverting spent orphaned siafund element", zap.Stringer("id", outputID), zap.Uint64("value", value)) + } + return balances, rows.Err() +} + +func deleteOrphanedSiafundElements(tx *txn, index types.ChainIndex, log *zap.Logger) (map[int64]uint64, error) { + rows, err := tx.Query(`DELETE FROM siafund_elements WHERE id IN (SELECT se.id FROM siafund_elements se +INNER JOIN chain_indices ci ON (ci.id=se.chain_index_id) +WHERE ci.height=$1 AND ci.block_id<>$2) +RETURNING id, address_id, siafund_value, spent_index_id IS NOT NULL`, index.Height, encode(index.ID)) + if err != nil { + return nil, fmt.Errorf("failed to query siafund elements: %w", err) + } + defer rows.Close() + + balances := make(map[int64]uint64) + for rows.Next() { + var outputID types.SiafundOutputID + var addrID int64 + var value uint64 + var spent bool + + if err := rows.Scan(decode(&outputID), &addrID, &value, &spent); err != nil { + return nil, fmt.Errorf("failed to scan siafund element: %w", err) + } + balances[addrID] += value + log.Debug("deleting orphaned siafund element", zap.Stringer("id", outputID), zap.Uint64("value", value), zap.Bool("spent", spent)) + } + return balances, rows.Err() +} + +func deleteOrphanedEvents(tx *txn, index types.ChainIndex) error { + _, err := tx.Exec(`DELETE FROM events WHERE id IN (SELECT ev.id FROM events ev +INNER JOIN chain_indices ci ON (ev.chain_index_id=ci.id) +WHERE ci.height=$1 AND ci.block_id<>$2);`, index.Height, encode(index.ID)) + return err +} + +// revertOrphans reverts any chain indices that were orphaned by the given index +func revertOrphans(tx *txn, index types.ChainIndex, log *zap.Logger) error { + // fetch orphaned siacoin balances + deletedSiacoins, err := deleteOrphanedSiacoinElements(tx, index, log.Named("deleteOrphanedSiacoinElements")) + if err != nil { + return fmt.Errorf("failed to get orphaned siacoin elements: %w", err) + } + + // fetch orphaned siafund balances + deletedSiafunds, err := deleteOrphanedSiafundElements(tx, index, log.Named("deleteOrphanedSiafundElements")) + if err != nil { + return fmt.Errorf("failed to get orphaned siafund elements: %w", err) + } + + unspentSiacoins, err := revertSpentOrphanedSiacoinElements(tx, index, log.Named("revertSpentOrphanedSiacoinElements")) + if err != nil { + return fmt.Errorf("failed to revert spent orphaned siacoin elements: %w", err) + } + + unspentSiafunds, err := revertSpentOrphanedSiafundElements(tx, index, log.Named("revertSpentOrphanedSiafundElements")) + if err != nil { + return fmt.Errorf("failed to revert spent orphaned siafund elements: %w", err) + } + + // get the addrIDs of all affected addresses + addrIDs := make(map[int64]bool) + for id := range deletedSiacoins { + addrIDs[id] = true + } + for id := range deletedSiafunds { + addrIDs[id] = true + } + for id := range unspentSiacoins { + addrIDs[id] = true + } + for id := range unspentSiafunds { + addrIDs[id] = true + } + + getBalanceStmt, err := tx.Prepare(`SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses WHERE id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare balance statement: %w", err) + } + defer getBalanceStmt.Close() + + updateBalanceStmt, err := tx.Prepare(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$2, siafund_balance=$3 WHERE id=$4`) + if err != nil { + return fmt.Errorf("failed to prepare update statement: %w", err) + } + defer updateBalanceStmt.Close() + + for addrID := range addrIDs { + var existing wallet.Balance + err := getBalanceStmt.QueryRow(addrID).Scan(decode(&existing.Siacoins), decode(&existing.ImmatureSiacoins), &existing.Siafunds) + if err != nil { + return fmt.Errorf("failed to get balance: %w", err) + } + + existing.Siacoins = existing.Siacoins.Sub(deletedSiacoins[addrID].Siacoins) + existing.ImmatureSiacoins = existing.ImmatureSiacoins.Sub(deletedSiacoins[addrID].ImmatureSiacoins) + if existing.Siafunds < deletedSiafunds[addrID] { + panic("siafund balance cannot be negative") + } + existing.Siafunds -= deletedSiafunds[addrID] + + existing.Siacoins = existing.Siacoins.Add(unspentSiacoins[addrID].Siacoins) + existing.Siafunds += unspentSiafunds[addrID] + + res, err := updateBalanceStmt.Exec(encode(existing.Siacoins), encode(existing.ImmatureSiacoins), existing.Siafunds, addrID) + if err != nil { + return fmt.Errorf("failed to update balance: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } else if n != 1 { + return fmt.Errorf("expected 1 row affected, got %v", n) + } + } + + if err := deleteOrphanedEvents(tx, index); err != nil { + return fmt.Errorf("failed to delete orphaned events: %w", err) + } + + _, err = tx.Exec(`DELETE FROM chain_indices WHERE height=$1 AND block_id<>$2`, index.Height, encode(index.ID)) + return err +} + +func pruneSpentSiacoinElements(tx *txn, height uint64) (removed int64, err error) { + const query = `DELETE FROM siacoin_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` + res, err := tx.Exec(query, height) + if err != nil { + return 0, fmt.Errorf("failed to query siacoin elements: %w", err) + } + return res.RowsAffected() +} + +func pruneSpentSiafundElements(tx *txn, height uint64) (removed int64, err error) { + const query = `DELETE FROM siafund_elements WHERE spent_index_id IN (SELECT id FROM chain_indices WHERE height <= $1)` + res, err := tx.Exec(query, height) + if err != nil { + return 0, fmt.Errorf("failed to query siacoin elements: %w", err) + } + return res.RowsAffected() +} + +func setGlobalState(tx *txn, index types.ChainIndex, numLeaves uint64) error { + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_height=$1, last_indexed_id=$2, element_num_leaves=$3`, index.Height, encode(index.ID), numLeaves) + return err +} + +func addressRefStmt(tx *txn) (func(types.Address) (addressRef, error), func() error, error) { + stmt, err := tx.Prepare(`INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) VALUES ($1, $2, $3, $4) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address RETURNING id, siacoin_balance, immature_siacoin_balance, siafund_balance`) + if err != nil { + return nil, nil, fmt.Errorf("failed to prepare address statement: %w", err) + } + // the on conflict is effectively a no-op, but enables us to return the id of the existing address + return func(addr types.Address) (addressRef, error) { + ref, err := scanAddress(stmt.QueryRow(encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency), 0)) + if err != nil { + return addressRef{}, fmt.Errorf("failed to get address %q: %w", addr, err) + } + return ref, nil + }, stmt.Close, nil +} diff --git a/persist/sqlite/consensus_test.go b/persist/sqlite/consensus_test.go new file mode 100644 index 0000000..8480192 --- /dev/null +++ b/persist/sqlite/consensus_test.go @@ -0,0 +1,315 @@ +package sqlite + +import ( + "path/filepath" + "testing" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap/zaptest" +) + +func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + } + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func syncDB(tb testing.TB, store *Store, cm *chain.Manager) { + index, err := store.LastCommittedIndex() + if err != nil { + tb.Fatalf("failed to get last committed index: %v", err) + } + for index != cm.Tip() { + crus, caus, err := cm.UpdatesSince(index, 1000) + if err != nil { + tb.Fatalf("failed to subscribe to chain manager: %v", err) + } else if err := store.UpdateChainState(crus, caus); err != nil { + tb.Fatalf("failed to update chain state: %v", err) + } + + switch { + case len(caus) > 0: + index = caus[len(caus)-1].State.Index + case len(crus) > 0: + index = crus[len(crus)-1].State.Index + } + } +} + +func TestPruneSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + // create a wallet + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + // mine a block to the wallet + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + assertBalance := func(siacoin, immature types.Currency) { + t.Helper() + + b, err := db.WalletBalance(w.ID) + if err != nil { + t.Fatalf("failed to get wallet balance: %v", err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } + } + + assertUTXOs := func(spent int, unspent int) { + t.Helper() + + var n int + err := db.db.QueryRow(`SELECT COUNT(*) FROM siacoin_elements WHERE spent_index_id IS NOT NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count spent siacoin elements: %v", err) + } else if n != spent { + t.Fatalf("expected %v spent siacoin elements, got %v", spent, n) + } + + err = db.db.QueryRow(`SELECT COUNT(*) FROM siacoin_elements WHERE spent_index_id IS NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count unspent siacoin elements: %v", err) + } else if n != unspent { + t.Fatalf("expected %v unspent siacoin elements, got %v", unspent, n) + } + } + + assertBalance(types.ZeroCurrency, expectedPayout) + assertUTXOs(0, 1) + + // mine until the payout matures + for i := 0; i < int(maturityHeight); i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + syncDB(t, db, cm) + assertBalance(expectedPayout, types.ZeroCurrency) + assertUTXOs(0, 1) + + // spend the utxo + utxos, _, err := db.WalletSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatalf("failed to get wallet siacoin outputs: %v", err) + } + + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Value: utxos[0].SiacoinOutput.Value, Address: types.VoidAddress}, + }, + } + + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + PublicKeyIndex: 0, + Timelock: 0, + Signature: sig[:], + }) + + // mine a block with the transaction + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + // the utxo should now have 0 balance and 1 spent element + assertBalance(types.ZeroCurrency, types.ZeroCurrency) + assertUTXOs(1, 0) + + // mine until the element is pruned + for i := 0; i < spentElementRetentionBlocks-1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(1, 0) // check that the element is not pruned early + } + + // trigger the pruning + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(0, 0) +} + +func TestPruneSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafund airdrop to the wallet + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + // create a wallet + w, err := db.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + syncDB(t, db, cm) + + assertBalance := func(siafunds uint64) { + t.Helper() + + b, err := db.WalletBalance(w.ID) + if err != nil { + t.Fatalf("failed to get wallet balance: %v", err) + } else if b.Siafunds != siafunds { + t.Fatalf("expected siafund balance %v, got %v", siafunds, b.ImmatureSiacoins) + } + } + + assertUTXOs := func(spent int, unspent int) { + t.Helper() + + var n int + err := db.db.QueryRow(`SELECT COUNT(*) FROM siafund_elements WHERE spent_index_id IS NOT NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count spent siacoin elements: %v", err) + } else if n != spent { + t.Fatalf("expected %v spent siacoin elements, got %v", spent, n) + } + + err = db.db.QueryRow(`SELECT COUNT(*) FROM siafund_elements WHERE spent_index_id IS NULL`).Scan(&n) + if err != nil { + t.Fatalf("failed to count unspent siacoin elements: %v", err) + } else if n != unspent { + t.Fatalf("expected %v unspent siacoin elements, got %v", unspent, n) + } + } + + assertBalance(cm.TipState().SiafundCount()) + assertUTXOs(0, 1) + + // spend the utxo + utxos, _, err := db.WalletSiafundOutputs(w.ID, 0, 100) + if err != nil { + t.Fatalf("failed to get wallet siacoin outputs: %v", err) + } + + txn := types.Transaction{ + SiafundInputs: []types.SiafundInput{{ + ParentID: types.SiafundOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }}, + SiafundOutputs: []types.SiafundOutput{ + {Value: utxos[0].SiafundOutput.Value, Address: types.VoidAddress}, + }, + } + + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxos[0].ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + PublicKeyIndex: 0, + Timelock: 0, + Signature: sig[:], + }) + + // mine a block with the transaction + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + + // the utxo should now have 0 balance and 1 spent element + assertBalance(0) + assertUTXOs(1, 0) + + // mine until the element is pruned + for i := 0; i < spentElementRetentionBlocks-1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) // check that the element is not pruned early + assertUTXOs(1, 0) + } + + // the spent element should now be pruned + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + syncDB(t, db, cm) + assertUTXOs(0, 0) +} diff --git a/persist/sqlite/consts_default.go b/persist/sqlite/consts_default.go new file mode 100644 index 0000000..19f84e5 --- /dev/null +++ b/persist/sqlite/consts_default.go @@ -0,0 +1,14 @@ +//go:build !testing + +package sqlite + +import "time" + +const ( + busyTimeout = 10000 // 10 seconds + maxRetryAttempts = 30 // 30 attempts + factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds + maxBackoff = 15 * time.Second + + spentElementRetentionBlocks = 144 // 1 day +) diff --git a/persist/sqlite/consts_testing.go b/persist/sqlite/consts_testing.go new file mode 100644 index 0000000..e4ade1e --- /dev/null +++ b/persist/sqlite/consts_testing.go @@ -0,0 +1,14 @@ +//go:build testing + +package sqlite + +import "time" + +const ( + busyTimeout = 100 // 100ms + maxRetryAttempts = 10 // 10 attempts + factor = 2.0 // factor ^ retryAttempts = backoff time in milliseconds + maxBackoff = 15 * time.Second + + spentElementRetentionBlocks = 36 +) diff --git a/persist/sqlite/encoding.go b/persist/sqlite/encoding.go new file mode 100644 index 0000000..48b9e25 --- /dev/null +++ b/persist/sqlite/encoding.go @@ -0,0 +1,95 @@ +package sqlite + +import ( + "bytes" + "database/sql" + "encoding/binary" + "errors" + "fmt" + "time" + + "go.sia.tech/core/types" +) + +func encode(obj any) any { + switch obj := obj.(type) { + case types.Currency: + // Currency is encoded as two 64-bit big-endian integers for sorting + buf := make([]byte, 16) + binary.BigEndian.PutUint64(buf, obj.Hi) + binary.BigEndian.PutUint64(buf[8:], obj.Lo) + return buf + case []types.Hash256: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + types.EncodeSlice(e, obj) + e.Flush() + return buf.Bytes() + case types.EncoderTo: + var buf bytes.Buffer + e := types.NewEncoder(&buf) + obj.EncodeTo(e) + e.Flush() + return buf.Bytes() + case uint64: + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, obj) + return b + case time.Time: + return obj.Unix() + default: + panic(fmt.Sprintf("dbEncode: unsupported type %T", obj)) + } +} + +type decodable struct { + v any +} + +// Scan implements the sql.Scanner interface. +func (d *decodable) Scan(src any) error { + if src == nil { + return errors.New("cannot scan nil into decodable") + } + + switch src := src.(type) { + case []byte: + switch v := d.v.(type) { + case *types.Currency: + if len(src) != 16 { + return fmt.Errorf("cannot scan %d bytes into Currency", len(src)) + } + v.Hi = binary.BigEndian.Uint64(src) + v.Lo = binary.BigEndian.Uint64(src[8:]) + case types.DecoderFrom: + dec := types.NewBufDecoder(src) + v.DecodeFrom(dec) + return dec.Err() + case *uint64: + *v = binary.LittleEndian.Uint64(src) + case *[]types.Hash256: + dec := types.NewBufDecoder(src) + types.DecodeSlice(dec, v) + return dec.Err() + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } + return nil + case int64: + switch v := d.v.(type) { + case *uint64: + *v = uint64(src) + case *time.Time: + *v = time.Unix(src, 0).UTC() + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } + return nil + default: + return fmt.Errorf("cannot scan %T to %T", src, d.v) + } +} + +func decode(obj any) sql.Scanner { + return &decodable{obj} +} diff --git a/persist/sqlite/events.go b/persist/sqlite/events.go new file mode 100644 index 0000000..a6b3754 --- /dev/null +++ b/persist/sqlite/events.go @@ -0,0 +1,144 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// Events returns the events with the given event IDs. If an event is not found, +// it is skipped. +func (s *Store) Events(eventIDs []types.Hash256) (events []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + var scanHeight uint64 + err := tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return fmt.Errorf("failed to get last indexed height: %w", err) + } + + // sqlite doesn't have easy support for IN clauses, use a statement since + // the number of event IDs is likely to be small instead of dynamically + // building the query + const query = `SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + ev.event_type, + ev.event_data +FROM events ev +INNER JOIN event_addresses ea ON (ev.id = ea.event_id) +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) +WHERE ev.event_id = $1` + + stmt, err := tx.Prepare(query) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]wallet.Event, 0, len(eventIDs)) + for _, id := range eventIDs { + event, _, err := scanEvent(stmt.QueryRow(encode(id)), scanHeight) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return fmt.Errorf("failed to query transaction %q: %w", id, err) + } + events = append(events, event) + } + return nil + }) + return +} + +func decodeEventData[T wallet.EventPayout | + wallet.EventV1Transaction | + wallet.EventV2Transaction | + wallet.EventV1ContractResolution | + wallet.EventV2ContractResolution, TP interface { + *T + types.DecoderFrom +}](dec *types.Decoder) T { + v := new(T) + TP(v).DecodeFrom(dec) + return *v +} + +func getEventsByID(tx *txn, eventIDs []int64) (events []wallet.Event, err error) { + var scanHeight uint64 + err = tx.QueryRow(`SELECT COALESCE(last_indexed_height, 0) FROM global_settings`).Scan(&scanHeight) + if err != nil { + return nil, fmt.Errorf("failed to get last indexed height: %w", err) + } + + stmt, err := tx.Prepare(`SELECT + ev.id, + ev.event_id, + ev.maturity_height, + ev.date_created, + ci.height, + ci.block_id, + ev.event_type, + ev.event_data +FROM events ev +INNER JOIN event_addresses ea ON (ev.id = ea.event_id) +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN chain_indices ci ON (ev.chain_index_id = ci.id) +WHERE ev.id=$1`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + events = make([]wallet.Event, 0, len(eventIDs)) + for i, id := range eventIDs { + event, _, err := scanEvent(stmt.QueryRow(id), scanHeight) + if errors.Is(err, sql.ErrNoRows) { + continue + } else if err != nil { + return nil, fmt.Errorf("failed to query event %d: %w", i, err) + } + events = append(events, event) + } + return +} + +func scanEvent(s scanner, scanHeight uint64) (ev wallet.Event, eventID int64, err error) { + var eventBuf []byte + err = s.Scan(&eventID, decode(&ev.ID), &ev.MaturityHeight, decode(&ev.Timestamp), &ev.Index.Height, decode(&ev.Index.ID), &ev.Type, &eventBuf) + if err != nil { + return + } + + if scanHeight >= ev.Index.Height { + ev.Confirmations = 1 + scanHeight - ev.Index.Height + } + + dec := types.NewBufDecoder(eventBuf) + switch ev.Type { + case wallet.EventTypeV1Transaction: + ev.Data = decodeEventData[wallet.EventV1Transaction](dec) + case wallet.EventTypeV2Transaction: + ev.Data = decodeEventData[wallet.EventV2Transaction](dec) + case wallet.EventTypeV1ContractResolution: + ev.Data = decodeEventData[wallet.EventV1ContractResolution](dec) + case wallet.EventTypeV2ContractResolution: + ev.Data = decodeEventData[wallet.EventV2ContractResolution](dec) + case wallet.EventTypeSiafundClaim, wallet.EventTypeMinerPayout, wallet.EventTypeFoundationSubsidy: + ev.Data = decodeEventData[wallet.EventPayout](dec) + default: + return wallet.Event{}, 0, fmt.Errorf("unknown event type: %q", ev.Type) + } + if err := dec.Err(); err != nil { + return wallet.Event{}, 0, fmt.Errorf("failed to decode event data: %w", err) + } + + return +} diff --git a/persist/sqlite/events_test.go b/persist/sqlite/events_test.go new file mode 100644 index 0000000..1ef067b --- /dev/null +++ b/persist/sqlite/events_test.go @@ -0,0 +1,102 @@ +package sqlite + +import ( + "fmt" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +func runBenchmarkWalletEvents(b *testing.B, name string, addresses, eventsPerAddress int) { + b.Run(name, func(b *testing.B) { + db, err := OpenDatabase(filepath.Join(b.TempDir(), "walletd.sqlite3"), zap.NewNop()) + if err != nil { + b.Fatal(err) + } + defer db.Close() + + w, err := db.AddWallet(wallet.Wallet{ + Name: "test", + }) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < addresses; i++ { + addr := types.Address(frand.Entropy256()) + if err := db.AddWalletAddress(w.ID, wallet.Address{Address: addr}); err != nil { + b.Fatal(err) + } + + err := db.transaction(func(tx *txn) error { + utx := &updateTx{ + indexMode: wallet.IndexModeFull, + tx: tx, + relevantAddresses: make(map[types.Address]bool), + } + + events := make([]wallet.Event, eventsPerAddress) + for i := range events { + events[i] = wallet.Event{ + ID: types.Hash256(frand.Entropy256()), + MaturityHeight: uint64(i + 1), + Relevant: []types.Address{addr}, + Type: wallet.EventTypeV1Transaction, + Data: wallet.EventV1Transaction{}, + } + } + + return utx.ApplyIndex(types.ChainIndex{ + Height: uint64(i + 1), + ID: types.BlockID(frand.Entropy256()), + }, wallet.AppliedState{ + Events: events, + }) + }) + if err != nil { + b.Fatal(err) + } + } + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + expectedEvents := eventsPerAddress * addresses + if expectedEvents > 100 { + expectedEvents = 100 + } + + events, err := db.WalletEvents(w.ID, 0, 100) + if err != nil { + b.Fatal(err) + } else if len(events) != expectedEvents { + b.Fatalf("expected %d events, got %d", expectedEvents, len(events)) + } + } + }) +} + +func BenchmarkWalletEvents(b *testing.B) { + benchmarks := []struct { + addresses int + eventsPerAddress int + }{ + {1, 1}, + {1, 10}, + {1, 1000}, + {10, 1}, + {10, 1000}, + {10, 100000}, + {1000000, 0}, + {1000000, 1}, + {1000000, 10}, + } + for _, bm := range benchmarks { + totalTransactions := bm.addresses * bm.eventsPerAddress + runBenchmarkWalletEvents(b, fmt.Sprintf("wallet with %d addresses and %d transactions", bm.addresses, totalTransactions), bm.addresses, bm.eventsPerAddress) + } +} diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go new file mode 100644 index 0000000..95f39a5 --- /dev/null +++ b/persist/sqlite/init.go @@ -0,0 +1,103 @@ +package sqlite + +import ( + "database/sql" + _ "embed" // for init.sql + "errors" + "time" + + "fmt" + + "go.sia.tech/core/types" + "go.uber.org/zap" +) + +// init queries are run when the database is first created. +// +//go:embed init.sql +var initDatabase string + +func initializeSettings(tx *txn, target int64) error { + _, err := tx.Exec(`INSERT INTO global_settings (id, db_version, last_indexed_height, last_indexed_id, element_num_leaves) VALUES (0, ?, 0, ?, 0)`, target, encode(types.BlockID{})) + return err +} + +func (s *Store) initNewDatabase(target int64) error { + return s.transaction(func(tx *txn) error { + if _, err := tx.Exec(initDatabase); err != nil { + return fmt.Errorf("failed to initialize database: %w", err) + } else if err := initializeSettings(tx, target); err != nil { + return fmt.Errorf("failed to initialize settings: %w", err) + } + return nil + }) +} + +func (s *Store) upgradeDatabase(current, target int64) error { + log := s.log.Named("migrations").With(zap.Int64("target", target)) + for ; current < target; current++ { + version := current + 1 // initial schema is version 1, migration 0 is version 2, etc. + log := log.With(zap.Int64("version", version)) + start := time.Now() + fn := migrations[current-1] + err := s.transaction(func(tx *txn) error { + if _, err := tx.Exec("PRAGMA defer_foreign_keys=ON"); err != nil { + return fmt.Errorf("failed to enable foreign key deferral: %w", err) + } else if err := fn(tx, log); err != nil { + return err + } else if err := foreignKeyCheck(tx, log); err != nil { + return fmt.Errorf("failed foreign key check: %w", err) + } + return setDBVersion(tx, version) + }) + if err != nil { + return fmt.Errorf("migration %d failed: %w", version, err) + } + log.Info("migration complete", zap.Duration("elapsed", time.Since(start))) + } + return nil +} + +func (s *Store) init() error { + // calculate the expected final database version + target := int64(len(migrations) + 1) + + version := getDBVersion(s.db) + switch { + case version == 0: + return s.initNewDatabase(target) + case version < target: + return s.upgradeDatabase(version, target) + case version > target: + return fmt.Errorf("database version %v is newer than expected %v. database downgrades are not supported", version, target) + } + // nothing to do + return nil +} + +func foreignKeyCheck(txn *txn, log *zap.Logger) error { + rows, err := txn.Query("PRAGMA foreign_key_check") + if err != nil { + return fmt.Errorf("failed to run foreign key check: %w", err) + } + defer rows.Close() + var hasErrors bool + for rows.Next() { + var table string + var rowid sql.NullInt64 + var fkTable string + var fkRowid sql.NullInt64 + + if err := rows.Scan(&table, &rowid, &fkTable, &fkRowid); err != nil { + return fmt.Errorf("failed to scan foreign key check result: %w", err) + } + hasErrors = true + log.Error("foreign key constraint violated", zap.String("table", table), zap.Int64("rowid", rowid.Int64), zap.String("fkTable", fkTable), zap.Int64("fkRowid", fkRowid.Int64)) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("failed to iterate foreign key check results: %w", err) + } else if hasErrors { + return errors.New("foreign key constraint violated") + } + return nil +} diff --git a/persist/sqlite/init.sql b/persist/sqlite/init.sql new file mode 100644 index 0000000..d3afb4c --- /dev/null +++ b/persist/sqlite/init.sql @@ -0,0 +1,127 @@ +CREATE TABLE chain_indices ( + id INTEGER PRIMARY KEY, + block_id BLOB UNIQUE NOT NULL, + height INTEGER UNIQUE NOT NULL +); +CREATE INDEX chain_indices_height_idx ON chain_indices (block_id, height); + +CREATE TABLE sia_addresses ( + id INTEGER PRIMARY KEY, + sia_address BLOB UNIQUE NOT NULL, + siacoin_balance BLOB NOT NULL, + immature_siacoin_balance BLOB NOT NULL, + siafund_balance INTEGER NOT NULL +); + +CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, -- stored as int64 for easier querying + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL, -- tracks whether the value has been added to the address balance + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id), -- soft delete + spent_event_id INTEGER REFERENCES events (id) -- atomic swap tracking +); +CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_spent_event_id_idx ON siacoin_elements (spent_event_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id); + +CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id), -- soft delete + spent_event_id INTEGER REFERENCES events (id) -- atomic swap tracking +); +CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_spent_event_id_idx ON siafund_elements (spent_event_id); +CREATE INDEX siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id); + +CREATE TABLE state_tree ( + row INTEGER, + column INTEGER, + value BLOB NOT NULL, + PRIMARY KEY (row, column) +); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + event_id BLOB UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, + date_created INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_data BLOB NOT NULL +); +CREATE INDEX events_chain_index_id_idx ON events (chain_index_id); +CREATE INDEX events_maturity_height_id_idx ON events (maturity_height DESC, id DESC); + +CREATE TABLE event_addresses ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + event_maturity_height INTEGER NOT NULL, -- flattened from events to improve query performance + PRIMARY KEY (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_event_maturity_height_event_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); + +CREATE TABLE wallets ( + id INTEGER PRIMARY KEY, + friendly_name TEXT NOT NULL, + description TEXT NOT NULL, + date_created INTEGER NOT NULL, + last_updated INTEGER NOT NULL, + extra_data BLOB +); + +CREATE TABLE wallet_addresses ( + wallet_id INTEGER NOT NULL REFERENCES wallets (id), + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + description TEXT NOT NULL, + spend_policy BLOB, + extra_data BLOB, + UNIQUE (wallet_id, address_id) +); +CREATE INDEX wallet_addresses_wallet_id_idx ON wallet_addresses (wallet_id); +CREATE INDEX wallet_addresses_address_id_idx ON wallet_addresses (address_id); +CREATE INDEX wallet_addresses_wallet_id_address_id_idx ON wallet_addresses (wallet_id, address_id); + +CREATE TABLE syncer_peers ( + peer_address TEXT PRIMARY KEY NOT NULL, + first_seen INTEGER NOT NULL +); + +CREATE TABLE syncer_bans ( + net_cidr TEXT PRIMARY KEY NOT NULL, + expiration INTEGER NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX syncer_bans_expiration_index_idx ON syncer_bans (expiration); + +CREATE TABLE signing_keys ( + public_key BLOB PRIMARY KEY, + private_key BLOB UNIQUE NOT NULL +); + +CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + index_mode INTEGER, -- the mode of the data store + last_indexed_height INTEGER NOT NULL, -- the height of the last chain index that was processed + last_indexed_id BLOB NOT NULL, -- the block ID of the last chain index that was processed + element_num_leaves INTEGER NOT NULL, -- the number of leaves in the state tree + key_salt BLOB -- the salt used for deriving keys +); diff --git a/persist/sqlite/keys.go b/persist/sqlite/keys.go new file mode 100644 index 0000000..6de47b6 --- /dev/null +++ b/persist/sqlite/keys.go @@ -0,0 +1,80 @@ +package sqlite + +import ( + "database/sql" + "errors" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" +) + +// AddSigningKey adds a signing key to the store. If the key already exists, it +// is not added again. +func (s *Store) AddSigningKey(pk types.PublicKey, buf []byte) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec("INSERT INTO signing_keys (public_key, private_key) VALUES (?, ?) ON CONFLICT (public_key) DO NOTHING", encode(pk), buf) + return err + }) +} + +// GetSigningKey returns the private key corresponding to the given public key. +// If the key is not found, it returns [keys.ErrNotFound]. +func (s *Store) GetSigningKey(pk types.PublicKey) (buf []byte, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT private_key FROM signing_keys WHERE public_key = ?", encode(pk)).Scan(&buf) + if errors.Is(err, sql.ErrNoRows) { + return keys.ErrNotFound + } else if err != nil { + return err + } + return nil + }) + return +} + +// DeleteSigningKey deletes the signing key with the given public key. If the key +// does not exist, it returns nil. +func (s *Store) DeleteSigningKey(pk types.PublicKey) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec("DELETE FROM signing_keys WHERE public_key = ?", encode(pk)) + return err + }) +} + +// GetKeySalt returns the salt used to derive the key encryption +// key. If no salt has been set, GetKeySalt returns (nil, nil). +func (s *Store) GetKeySalt() (salt []byte, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT key_salt FROM global_settings").Scan(&salt) + return err + }) + return +} + +// SetKeySalt sets the salt used to derive the key encryption key. +// If a salt has already been set, [keys.ErrSaltSet] is returned. +func (s *Store) SetKeySalt(salt []byte) error { + return s.transaction(func(tx *txn) error { + res, err := tx.Exec("UPDATE global_settings SET key_salt = ? WHERE key_salt IS NULL", salt) + if err != nil { + return err + } else if n, _ := res.RowsAffected(); n == 0 { + return keys.ErrSaltSet + } + return nil + }) +} + +// GetBytesForVerify returns random encrypted bytes for verifying +// the encryption key. If there are no keys in the store, it returns +// [keys.ErrNotFound]. +func (s *Store) GetBytesForVerify() (buf []byte, err error) { + err = s.transaction(func(tx *txn) error { + err := s.db.QueryRow("SELECT private_key FROM signing_keys LIMIT 1").Scan(&buf) + if errors.Is(err, sql.ErrNoRows) { + return keys.ErrNotFound + } + return err + }) + return +} diff --git a/persist/sqlite/keys_test.go b/persist/sqlite/keys_test.go new file mode 100644 index 0000000..bf38570 --- /dev/null +++ b/persist/sqlite/keys_test.go @@ -0,0 +1,76 @@ +package sqlite + +import ( + "bytes" + "errors" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/keys" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +func TestSigningKeys(t *testing.T) { + store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + sk := types.GeneratePrivateKey() + + _, err = store.GetSigningKey(sk.PublicKey()) + if !errors.Is(err, keys.ErrNotFound) { + t.Fatal(err) + } + + expected := frand.Bytes(64) // mock encrypted key + if err = store.AddSigningKey(sk.PublicKey(), expected); err != nil { + t.Fatal(err) + } + + buf, err := store.GetSigningKey(sk.PublicKey()) + if err != nil { + t.Fatal(err) + } else if !bytes.Equal(expected, buf) { + t.Fatal("keys don't match") + } +} + +func TestSalt(t *testing.T) { + store, err := OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), zaptest.NewLogger(t)) + if err != nil { + t.Fatal(err) + } + defer store.Close() + + assertSalt := func(t *testing.T, expected []byte) { + t.Helper() + s, err := store.GetKeySalt() + if err != nil { + t.Fatal(err) + } else if expected == nil && s != nil { + t.Fatal("expected nil salt") // bytes.Equal([]byte{}, nil) == true + } else if !bytes.Equal(s, expected) { + t.Fatal("salts don't match") + } + } + + // check salt is initially nil + assertSalt(t, nil) + + expected := frand.Bytes(32) + if err = store.SetKeySalt(expected); err != nil { + t.Fatal(err) + } + assertSalt(t, expected) + + if err = store.SetKeySalt(frand.Bytes(32)); !errors.Is(err, keys.ErrSaltSet) { + t.Fatalf("expected %v, got %v", keys.ErrSaltSet, err) + } + + // check salt was not changed + assertSalt(t, expected) +} diff --git a/persist/sqlite/migrations.go b/persist/sqlite/migrations.go new file mode 100644 index 0000000..77a747e --- /dev/null +++ b/persist/sqlite/migrations.go @@ -0,0 +1,212 @@ +package sqlite + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.uber.org/zap" +) + +func migrateVersion8(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`CREATE TABLE signing_keys ( + public_key BLOB PRIMARY KEY, + private_key BLOB UNIQUE NOT NULL +); +ALTER TABLE global_settings ADD COLUMN key_salt BLOB;`) + return err +} + +// migrateVersion7 adds spent_event_id columns to siacoin_elements and +// siafund_elements to track the event that spent the element. +func migrateVersion7(tx *txn, _ *zap.Logger) error { + const query = `ALTER TABLE siacoin_elements ADD COLUMN spent_event_id INTEGER REFERENCES events (id); +CREATE INDEX siacoin_elements_spent_event_id_idx ON siacoin_elements (spent_event_id); +ALTER TABLE siafund_elements ADD COLUMN spent_event_id INTEGER REFERENCES events (id); +CREATE INDEX siafund_elements_spent_event_id_idx ON siafund_elements (spent_event_id);` + _, err := tx.Exec(query) + return err +} + +// migrateVersion6 flattens the maturity height from events into event_addresses +// to improve query performance. +func migrateVersion6(tx *txn, _ *zap.Logger) error { + const query = ` +CREATE TABLE event_addresses_new ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + event_maturity_height INTEGER NOT NULL, -- flattened from events to improve query performance + PRIMARY KEY (event_id, address_id) +); +INSERT INTO event_addresses_new (event_id, address_id, event_maturity_height) SELECT ea.event_id, ea.address_id, ev.maturity_height FROM event_addresses ea INNER JOIN events ev ON ea.event_id = ev.id; + +DROP TABLE event_addresses; + +ALTER TABLE event_addresses_new RENAME TO event_addresses; +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX event_addresses_event_id_address_id_event_maturity_height_event_id_idx ON event_addresses (address_id, event_maturity_height DESC, event_id DESC); +` + _, err := tx.Exec(query) + return err +} + +// migrateVersion5 resets the database to trigger a full resync to switch +// events from JSON to Sia encoding +func migrateVersion5(tx *txn, _ *zap.Logger) error { + if _, err := tx.Exec(`DELETE FROM siacoin_elements;`); err != nil { + return fmt.Errorf("failed to delete siacoin_elements: %w", err) + } else if _, err := tx.Exec(`DELETE FROM siafund_elements;`); err != nil { + return fmt.Errorf("failed to delete siafund_elements: %w", err) + } else if _, err := tx.Exec(`DELETE FROM state_tree;`); err != nil { + return fmt.Errorf("failed to delete state_tree: %w", err) + } else if _, err := tx.Exec(`DELETE FROM event_addresses;`); err != nil { + return fmt.Errorf("failed to delete event_addresses: %w", err) + } else if _, err := tx.Exec(`DELETE FROM events;`); err != nil { + return fmt.Errorf("failed to delete events: %w", err) + } else if _, err := tx.Exec(`DELETE FROM chain_indices;`); err != nil { + return fmt.Errorf("failed to delete chain_indices: %w", err) + } else if _, err := tx.Exec(`DROP TABLE siacoin_elements;`); err != nil { + return fmt.Errorf("failed to drop siacoin_elements: %w", err) + } else if _, err := tx.Exec(`DROP TABLE siafund_elements;`); err != nil { + return fmt.Errorf("failed to drop siafund_elements: %w", err) + } + + _, err := tx.Exec(`UPDATE global_settings SET last_indexed_height=0, last_indexed_id=$1, element_num_leaves=0`, encode(types.ChainIndex{})) + if err != nil { + return fmt.Errorf("failed to reset global_settings: %w", err) + } + + _, err = tx.Exec(`UPDATE sia_addresses SET siacoin_balance=$1, immature_siacoin_balance=$1, siafund_balance=0;`, encode(types.ZeroCurrency)) + if err != nil { + return fmt.Errorf("failed to reset sia_addresses: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id);`) + if err != nil { + return fmt.Errorf("failed to create siacoin_elements: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER UNIQUE NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id);`) + if err != nil { + return fmt.Errorf("failed to create siafund_elements: %w", err) + } + return nil +} + +// migrateVersion4 splits the height and ID of the last indexed tip into two +// separate columns for easier querying. +func migrateVersion4(tx *txn, _ *zap.Logger) error { + var dbVersion int + var indexMode int + var elementNumLeaves uint64 + var index types.ChainIndex + err := tx.QueryRow(`SELECT db_version, index_mode, element_num_leaves, last_indexed_tip FROM global_settings`).Scan(&dbVersion, &indexMode, &elementNumLeaves, decode(&index)) + if err != nil { + return fmt.Errorf("failed to get last indexed tip: %w", err) + } else if _, err := tx.Exec(`DROP TABLE global_settings`); err != nil { + return fmt.Errorf("failed to drop global_settings: %w", err) + } + + _, err = tx.Exec(`CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + index_mode INTEGER, -- the mode of the data store + last_indexed_height INTEGER NOT NULL, -- the height of the last chain index that was processed + last_indexed_id BLOB NOT NULL, -- the block ID of the last chain index that was processed + element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree +);`) + if err != nil { + return fmt.Errorf("failed to create global_settings: %w", err) + } + + _, err = tx.Exec(`INSERT INTO global_settings (id, db_version, index_mode, last_indexed_height, last_indexed_id, element_num_leaves) VALUES (0, ?, ?, ?, ?, ?)`, dbVersion, indexMode, index.Height, encode(index.ID), elementNumLeaves) + return err +} + +// migrateVersion3 adds additional indices to event_addresses and wallet_addresses +// to improve query performance. +func migrateVersion3(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`CREATE INDEX event_addresses_event_id_address_id_idx ON event_addresses (event_id, address_id); +CREATE INDEX wallet_addresses_wallet_id_address_id_idx ON wallet_addresses (wallet_id, address_id);`) + return err +} + +// migrateVersion2 recreates indices and speeds up event queries +func migrateVersion2(tx *txn, _ *zap.Logger) error { + _, err := tx.Exec(`DROP INDEX IF EXISTS chain_indices_height; +DROP INDEX IF EXISTS siacoin_elements_address_id; +DROP INDEX IF EXISTS siacoin_elements_maturity_height_matured; +DROP INDEX IF EXISTS siacoin_elements_chain_index_id; +DROP INDEX IF EXISTS siacoin_elements_spent_index_id; +DROP INDEX IF EXISTS siacoin_elements_address_id_spent_index_id; +DROP INDEX IF EXISTS siafund_elements_address_id; +DROP INDEX IF EXISTS siafund_elements_chain_index_id; +DROP INDEX IF EXISTS siafund_elements_spent_index_id; +DROP INDEX IF EXISTS siafund_elements_address_id_spent_index_id; +DROP INDEX IF EXISTS events_chain_index_id; +DROP INDEX IF EXISTS event_addresses_event_id_idx; +DROP INDEX IF EXISTS event_addresses_address_id_idx; +DROP INDEX IF EXISTS wallet_addresses_wallet_id; +DROP INDEX IF EXISTS wallet_addresses_address_id; +DROP INDEX IF EXISTS syncer_bans_expiration_index; + +CREATE INDEX IF NOT EXISTS chain_indices_height_idx ON chain_indices (block_id, height); +CREATE INDEX IF NOT EXISTS siacoin_elements_address_id_idx ON siacoin_elements (address_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_maturity_height_matured_idx ON siacoin_elements (maturity_height, matured); +CREATE INDEX IF NOT EXISTS siacoin_elements_chain_index_id_idx ON siacoin_elements (chain_index_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_spent_index_id_idx ON siacoin_elements (spent_index_id); +CREATE INDEX IF NOT EXISTS siacoin_elements_address_id_spent_index_id_idx ON siacoin_elements(address_id, spent_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_address_id_idx ON siafund_elements (address_id); +CREATE INDEX IF NOT EXISTS siafund_elements_chain_index_id_idx ON siafund_elements (chain_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_spent_index_id_idx ON siafund_elements (spent_index_id); +CREATE INDEX IF NOT EXISTS siafund_elements_address_id_spent_index_id_idx ON siafund_elements(address_id, spent_index_id); +CREATE INDEX IF NOT EXISTS events_chain_index_id_idx ON events (chain_index_id); +CREATE INDEX IF NOT EXISTS events_maturity_height_id_idx ON events (maturity_height DESC, id DESC); +CREATE INDEX IF NOT EXISTS event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX IF NOT EXISTS event_addresses_address_id_idx ON event_addresses (address_id); +CREATE INDEX IF NOT EXISTS wallet_addresses_wallet_id_idx ON wallet_addresses (wallet_id); +CREATE INDEX IF NOT EXISTS wallet_addresses_address_id_idx ON wallet_addresses (address_id); +CREATE INDEX IF NOT EXISTS syncer_bans_expiration_index_idx ON syncer_bans (expiration);`) + return err +} + +// migrations is a list of functions that are run to migrate the database from +// one version to the next. Migrations are used to update existing databases to +// match the schema in init.sql. +var migrations = []func(tx *txn, log *zap.Logger) error{ + migrateVersion2, + migrateVersion3, + migrateVersion4, + migrateVersion5, + migrateVersion6, + migrateVersion7, + migrateVersion8, +} diff --git a/persist/sqlite/migrations_test.go b/persist/sqlite/migrations_test.go new file mode 100644 index 0000000..369e05e --- /dev/null +++ b/persist/sqlite/migrations_test.go @@ -0,0 +1,323 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + "go.sia.tech/core/types" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +// nolint:misspell +const initialSchema = `CREATE TABLE chain_indices ( + id INTEGER PRIMARY KEY, + block_id BLOB UNIQUE NOT NULL, + height INTEGER UNIQUE NOT NULL +); +CREATE INDEX chain_indices_height ON chain_indices (block_id, height); + +CREATE TABLE sia_addresses ( + id INTEGER PRIMARY KEY, + sia_address BLOB UNIQUE NOT NULL, + siacoin_balance BLOB NOT NULL, + immature_siacoin_balance BLOB NOT NULL, + siafund_balance INTEGER NOT NULL +); + +CREATE TABLE siacoin_elements ( + id BLOB PRIMARY KEY, + siacoin_value BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + maturity_height INTEGER NOT NULL, /* stored as int64 for easier querying */ + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + matured BOOLEAN NOT NULL, /* tracks whether the value has been added to the address balance */ + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siacoin_elements_address_id ON siacoin_elements (address_id); +CREATE INDEX siacoin_elements_maturity_height_matured ON siacoin_elements (maturity_height, matured); +CREATE INDEX siacoin_elements_chain_index_id ON siacoin_elements (chain_index_id); +CREATE INDEX siacoin_elements_spent_index_id ON siacoin_elements (spent_index_id); +CREATE INDEX siacoin_elements_address_id_spent_index_id ON siacoin_elements(address_id, spent_index_id); + +CREATE TABLE siafund_elements ( + id BLOB PRIMARY KEY, + claim_start BLOB NOT NULL, + merkle_proof BLOB NOT NULL, + leaf_index INTEGER NOT NULL, + siafund_value INTEGER NOT NULL, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + spent_index_id INTEGER REFERENCES chain_indices (id) /* soft delete */ +); +CREATE INDEX siafund_elements_address_id ON siafund_elements (address_id); +CREATE INDEX siafund_elements_chain_index_id ON siafund_elements (chain_index_id); +CREATE INDEX siafund_elements_spent_index_id ON siafund_elements (spent_index_id); +CREATE INDEX siafund_elements_address_id_spent_index_id ON siafund_elements(address_id, spent_index_id); + +CREATE TABLE state_tree ( + row INTEGER, + column INTEGER, + value BLOB NOT NULL, + PRIMARY KEY (row, column) +); + +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + chain_index_id INTEGER NOT NULL REFERENCES chain_indices (id), + event_id BLOB UNIQUE NOT NULL, + maturity_height INTEGER NOT NULL, + date_created INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_data BLOB NOT NULL +); +CREATE INDEX events_chain_index_id ON events (chain_index_id); + +CREATE TABLE event_addresses ( + event_id INTEGER NOT NULL REFERENCES events (id) ON DELETE CASCADE, + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + PRIMARY KEY (event_id, address_id) +); +CREATE INDEX event_addresses_event_id_idx ON event_addresses (event_id); +CREATE INDEX event_addresses_address_id_idx ON event_addresses (address_id); + +CREATE TABLE wallets ( + id INTEGER PRIMARY KEY, + friendly_name TEXT NOT NULL, + description TEXT NOT NULL, + date_created INTEGER NOT NULL, + last_updated INTEGER NOT NULL, + extra_data BLOB +); + +CREATE TABLE wallet_addresses ( + wallet_id INTEGER NOT NULL REFERENCES wallets (id), + address_id INTEGER NOT NULL REFERENCES sia_addresses (id), + description TEXT NOT NULL, + spend_policy BLOB, + extra_data BLOB, + UNIQUE (wallet_id, address_id) +); +CREATE INDEX wallet_addresses_wallet_id ON wallet_addresses (wallet_id); +CREATE INDEX wallet_addresses_address_id ON wallet_addresses (address_id); + +CREATE TABLE syncer_peers ( + peer_address TEXT PRIMARY KEY NOT NULL, + first_seen INTEGER NOT NULL +); + +CREATE TABLE syncer_bans ( + net_cidr TEXT PRIMARY KEY NOT NULL, + expiration INTEGER NOT NULL, + reason TEXT NOT NULL +); +CREATE INDEX syncer_bans_expiration_index ON syncer_bans (expiration); + +CREATE TABLE global_settings ( + id INTEGER PRIMARY KEY NOT NULL DEFAULT 0 CHECK (id = 0), -- enforce a single row + db_version INTEGER NOT NULL, -- used for migrations + index_mode INTEGER, -- the mode of the data store + last_indexed_tip BLOB NOT NULL, -- the last chain index that was processed + element_num_leaves INTEGER NOT NULL -- the number of leaves in the state tree +);` + +func TestMigrationConsistency(t *testing.T) { + fp := filepath.Join(t.TempDir(), "walletd.sqlite3") + db, err := sql.Open("sqlite3", sqliteFilepath(fp)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if _, err := db.Exec(initialSchema); err != nil { + t.Fatal(err) + } + + // initialize the settings table + _, err = db.Exec(`INSERT INTO global_settings (id, db_version, index_mode, element_num_leaves, last_indexed_tip) VALUES (0, 1, 0, 0, ?)`, encode(types.ChainIndex{})) + if err != nil { + t.Fatal(err) + } + + if err := db.Close(); err != nil { + t.Fatal(err) + } + + expectedVersion := int64(len(migrations) + 1) + log := zaptest.NewLogger(t) + store, err := OpenDatabase(fp, log) + if err != nil { + t.Fatal(err) + } + defer store.Close() + v := getDBVersion(store.db) + if v != expectedVersion { + t.Fatalf("expected version %d, got %d", expectedVersion, v) + } else if err := store.Close(); err != nil { + t.Fatal(err) + } + + // ensure the database does not change version when opened again + store, err = OpenDatabase(fp, log) + if err != nil { + t.Fatal(err) + } + defer store.Close() + v = getDBVersion(store.db) + if v != expectedVersion { + t.Fatalf("expected version %d, got %d", expectedVersion, v) + } + + fp2 := filepath.Join(t.TempDir(), "walletd.sqlite3") + baseline, err := OpenDatabase(fp2, zap.NewNop()) + if err != nil { + t.Fatal(err) + } + defer baseline.Close() + + getTableIndices := func(db *sql.DB) (map[string]bool, error) { + const query = `SELECT name, tbl_name, sql FROM sqlite_schema WHERE type='index'` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + indices := make(map[string]bool) + for rows.Next() { + var name, table string + var sqlStr sql.NullString // auto indices have no sql + if err := rows.Scan(&name, &table, &sqlStr); err != nil { + return nil, err + } + indices[fmt.Sprintf("%s.%s.%s", name, table, sqlStr.String)] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return indices, nil + } + + // ensure the migrated database has the same indices as the baseline + baselineIndices, err := getTableIndices(baseline.db) + if err != nil { + t.Fatal(err) + } + + migratedIndices, err := getTableIndices(store.db) + if err != nil { + t.Fatal(err) + } + + for k := range baselineIndices { + if !migratedIndices[k] { + t.Errorf("missing index %s", k) + } + } + + for k := range migratedIndices { + if !baselineIndices[k] { + t.Errorf("unexpected index %s", k) + } + } + + getTables := func(db *sql.DB) (map[string]bool, error) { + const query = `SELECT name FROM sqlite_schema WHERE type='table'` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + tables := make(map[string]bool) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + tables[name] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return tables, nil + } + + // ensure the migrated database has the same tables as the baseline + baselineTables, err := getTables(baseline.db) + if err != nil { + t.Fatal(err) + } + + migratedTables, err := getTables(store.db) + if err != nil { + t.Fatal(err) + } + + for k := range baselineTables { + if !migratedTables[k] { + t.Errorf("missing table %s", k) + } + } + for k := range migratedTables { + if !baselineTables[k] { + t.Errorf("unexpected table %s", k) + } + } + + // ensure each table has the same columns as the baseline + getTableColumns := func(db *sql.DB, table string) (map[string]bool, error) { + query := fmt.Sprintf(`PRAGMA table_info(%s)`, table) // cannot use parameterized query for PRAGMA statements + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + columns := make(map[string]bool) + for rows.Next() { + var cid int + var name, colType string + var defaultValue sql.NullString + var notNull bool + var primaryKey int // composite keys are indices + if err := rows.Scan(&cid, &name, &colType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, err + } + // column ID is ignored since it may not match between the baseline and migrated databases + key := fmt.Sprintf("%s.%s.%s.%t.%d", name, colType, defaultValue.String, notNull, primaryKey) + columns[key] = true + } + if err := rows.Err(); err != nil { + return nil, err + } + return columns, nil + } + + for k := range baselineTables { + baselineColumns, err := getTableColumns(baseline.db, k) + if err != nil { + t.Fatal(err) + } + migratedColumns, err := getTableColumns(store.db, k) + if err != nil { + t.Fatal(err) + } + + for c := range baselineColumns { + if !migratedColumns[c] { + t.Errorf("missing column %s.%s", k, c) + } + } + + for c := range migratedColumns { + if !baselineColumns[c] { + t.Errorf("unexpected column %s.%s", k, c) + } + } + } +} diff --git a/persist/sqlite/peers.go b/persist/sqlite/peers.go new file mode 100644 index 0000000..daca41f --- /dev/null +++ b/persist/sqlite/peers.go @@ -0,0 +1,234 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + "time" + + "go.sia.tech/coreutils/syncer" + "go.uber.org/zap" +) + +// A PeerStore stores information about peers. +type PeerStore struct { + s *Store + + // session-specific peer info is stored in memory to reduce write load + // on the database + mu sync.Mutex + peerInfo map[string]syncer.PeerInfo +} + +// AddPeer adds the given peer to the store. +func (ps *PeerStore) AddPeer(peer string) error { + ps.mu.Lock() + defer ps.mu.Unlock() + ps.peerInfo[peer] = syncer.PeerInfo{ + Address: peer, + FirstSeen: time.Now(), + } + return ps.s.AddPeer(peer) +} + +// Peers returns the addresses of all known peers. +func (ps *PeerStore) Peers() ([]syncer.PeerInfo, error) { + ps.mu.Lock() + defer ps.mu.Unlock() + + // copy the map to a slice + peers := make([]syncer.PeerInfo, 0, len(ps.peerInfo)) + for _, pi := range ps.peerInfo { + peers = append(peers, pi) + } + return peers, nil +} + +// UpdatePeerInfo updates the information for the given peer. +func (ps *PeerStore) UpdatePeerInfo(peer string, fn func(*syncer.PeerInfo)) error { + ps.mu.Lock() + defer ps.mu.Unlock() + if pi, ok := ps.peerInfo[peer]; !ok { + return syncer.ErrPeerNotFound + } else { + fn(&pi) + ps.peerInfo[peer] = pi + } + return nil +} + +// Ban temporarily bans the given peer. +func (ps *PeerStore) Ban(peer string, duration time.Duration, reason string) error { + return ps.s.Ban(peer, duration, reason) +} + +// Banned returns true if the peer is banned. +func (ps *PeerStore) Banned(peer string) (bool, error) { + return ps.s.Banned(peer) +} + +// PeerInfo returns the information for the given peer. +func (ps *PeerStore) PeerInfo(peer string) (syncer.PeerInfo, error) { + ps.mu.Lock() + defer ps.mu.Unlock() + if pi, ok := ps.peerInfo[peer]; ok { + return pi, nil + } + return syncer.PeerInfo{}, syncer.ErrPeerNotFound +} + +// NewPeerStore creates a new peer store using the given store. +func NewPeerStore(s *Store) (syncer.PeerStore, error) { + ps := &PeerStore{s: s, peerInfo: make(map[string]syncer.PeerInfo)} + peers, err := s.Peers() + if err != nil { + return nil, fmt.Errorf("failed to load peers: %w", err) + } + for _, pi := range peers { + ps.peerInfo[pi.Address] = pi + } + return ps, nil +} + +func scanPeerInfo(s scanner) (pi syncer.PeerInfo, err error) { + err = s.Scan(&pi.Address, decode(&pi.FirstSeen)) + return +} + +// AddPeer adds the given peer to the store. +func (s *Store) AddPeer(peer string) error { + return s.transaction(func(tx *txn) error { + const query = `INSERT INTO syncer_peers (peer_address, first_seen) VALUES ($1, $2) ON CONFLICT (peer_address) DO NOTHING` + _, err := tx.Exec(query, peer, encode(time.Now())) + return err + }) +} + +// Peers returns the addresses of all known peers. +func (s *Store) Peers() (peers []syncer.PeerInfo, _ error) { + err := s.transaction(func(tx *txn) error { + const query = `SELECT peer_address, first_seen FROM syncer_peers` + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + peer, err := scanPeerInfo(rows) + if err != nil { + return fmt.Errorf("failed to scan peer info: %w", err) + } + peers = append(peers, peer) + } + return rows.Err() + }) + return peers, err +} + +// normalizePeer normalizes a peer address to a CIDR subnet. +func normalizePeer(peer string) (string, error) { + host, _, err := net.SplitHostPort(peer) + if err != nil { + host = peer + } + if strings.IndexByte(host, '/') != -1 { + _, subnet, err := net.ParseCIDR(host) + if err != nil { + return "", fmt.Errorf("failed to parse CIDR: %w", err) + } + return subnet.String(), nil + } + + ip := net.ParseIP(host) + if ip == nil { + return "", errors.New("invalid IP address") + } + + var maskLen int + if ip.To4() != nil { + maskLen = 32 + } else { + maskLen = 128 + } + + _, normalized, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), maskLen)) + if err != nil { + panic("failed to parse CIDR") + } + return normalized.String(), nil +} + +// Ban temporarily bans one or more IPs. The addr should either be a single +// IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). +func (s *Store) Ban(peer string, duration time.Duration, reason string) error { + address, err := normalizePeer(peer) + if err != nil { + return err + } + return s.transaction(func(tx *txn) error { + const query = `INSERT INTO syncer_bans (net_cidr, expiration, reason) VALUES ($1, $2, $3) ON CONFLICT (net_cidr) DO UPDATE SET expiration=EXCLUDED.expiration, reason=EXCLUDED.reason` + _, err := tx.Exec(query, address, encode(time.Now().Add(duration)), reason) + return err + }) +} + +// Banned returns true if the peer is banned. +func (s *Store) Banned(peer string) (banned bool, _ error) { + // normalize the peer into a CIDR subnet + peer, err := normalizePeer(peer) + if err != nil { + return false, fmt.Errorf("failed to normalize peer: %w", err) + } + + _, subnet, err := net.ParseCIDR(peer) + if err != nil { + return false, fmt.Errorf("failed to parse CIDR: %w", err) + } + + // check all subnets from the given subnet to the max subnet length + var maxMaskLen int + if subnet.IP.To4() != nil { + maxMaskLen = 32 + } else { + maxMaskLen = 128 + } + + checkSubnets := make([]string, 0, maxMaskLen) + for i := maxMaskLen; i > 0; i-- { + _, subnet, err := net.ParseCIDR(subnet.IP.String() + "/" + strconv.Itoa(i)) + if err != nil { + panic("failed to parse CIDR") + } + checkSubnets = append(checkSubnets, subnet.String()) + } + + err = s.transaction(func(tx *txn) error { + checkSubnetStmt, err := tx.Prepare(`SELECT expiration FROM syncer_bans WHERE net_cidr = $1 ORDER BY expiration DESC LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare statement: %w", err) + } + defer checkSubnetStmt.Close() + + for _, subnet := range checkSubnets { + var expiration time.Time + + err := checkSubnetStmt.QueryRow(subnet).Scan(decode(&expiration)) + banned = time.Now().Before(expiration) // will return false for any sql errors, including ErrNoRows + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to check ban status: %w", err) + } else if banned { + s.log.Debug("found ban", zap.String("subnet", subnet), zap.Time("expiration", expiration)) + return nil + } + } + return nil + }) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return false, fmt.Errorf("failed to check ban status: %w", err) + } + return banned, nil +} diff --git a/persist/sqlite/peers_test.go b/persist/sqlite/peers_test.go new file mode 100644 index 0000000..0ef91dc --- /dev/null +++ b/persist/sqlite/peers_test.go @@ -0,0 +1,122 @@ +package sqlite + +import ( + "net" + "path/filepath" + "testing" + "time" + + "go.sia.tech/coreutils/syncer" + "go.uber.org/zap/zaptest" +) + +func TestAddPeer(t *testing.T) { + log := zaptest.NewLogger(t) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + ps, err := NewPeerStore(db) + if err != nil { + t.Fatal(err) + } + + const peer = "1.2.3.4:9981" + + if err := ps.AddPeer(peer); err != nil { + t.Fatal(err) + } + + lastConnect := time.Now().UTC().Truncate(time.Second) // stored as unix milliseconds + syncedBlocks := uint64(15) + syncDuration := 5 * time.Second + + err = ps.UpdatePeerInfo(peer, func(info *syncer.PeerInfo) { + info.LastConnect = lastConnect + info.SyncedBlocks = syncedBlocks + info.SyncDuration = syncDuration + }) + if err != nil { + t.Fatal(err) + } + + info, err := ps.PeerInfo(peer) + if err != nil { + t.Fatal(err) + } + + if !info.LastConnect.Equal(lastConnect) { + t.Errorf("expected LastConnect = %v; got %v", lastConnect, info.LastConnect) + } + if info.SyncedBlocks != syncedBlocks { + t.Errorf("expected SyncedBlocks = %d; got %d", syncedBlocks, info.SyncedBlocks) + } + if info.SyncDuration != 5*time.Second { + t.Errorf("expected SyncDuration = %s; got %s", syncDuration, info.SyncDuration) + } + + peers, err := ps.Peers() + if err != nil { + t.Fatal(err) + } else if len(peers) != 1 { + t.Fatalf("expected 1 peer; got %d", len(peers)) + } else if peerInfo := peers[0]; peerInfo.Address != peer { + t.Errorf("expected peer address = %q; got %q", peer, peerInfo.Address) + } else if peerInfo.LastConnect != lastConnect { + t.Errorf("expected LastConnect = %v; got %v", lastConnect, peerInfo.LastConnect) + } else if peerInfo.SyncedBlocks != syncedBlocks { + t.Errorf("expected SyncedBlocks = %d; got %d", syncedBlocks, peerInfo.SyncedBlocks) + } else if peerInfo.SyncDuration != syncDuration { + t.Errorf("expected SyncDuration = %s; got %s", syncDuration, peerInfo.SyncDuration) + } else if peerInfo.FirstSeen.IsZero() { + t.Errorf("expected FirstSeen to be non-zero; got %v", peerInfo.FirstSeen) + } +} + +func TestBanPeer(t *testing.T) { + log := zaptest.NewLogger(t) + db, err := OpenDatabase(filepath.Join(t.TempDir(), "test.db"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + ps, err := NewPeerStore(db) + if err != nil { + t.Fatal(err) + } + + const peer = "1.2.3.4" + + if banned, err := ps.Banned(peer); err != nil || banned { + t.Fatal("expected peer to not be banned", err) + } + + // ban the peer + ps.Ban(peer, 5*time.Second, "test") + + if banned, err := ps.Banned(peer); err != nil || !banned { + t.Fatal("expected peer to be banned", err) + } + + // wait for the ban to expire + time.Sleep(5 * time.Second) + + if banned, err := ps.Banned(peer); err != nil || banned { + t.Fatal("expected peer to not be banned", err) + } + + // ban a subnet + _, subnet, err := net.ParseCIDR(peer + "/24") + if err != nil { + t.Fatal(err) + } + + t.Log("banning", subnet) + ps.Ban(subnet.String(), time.Second, "test") + if banned, err := ps.Banned(peer); err != nil || !banned { + t.Fatal("expected peer to be banned", err) + } +} diff --git a/persist/sqlite/sql.go b/persist/sqlite/sql.go new file mode 100644 index 0000000..2ea2424 --- /dev/null +++ b/persist/sqlite/sql.go @@ -0,0 +1,190 @@ +package sqlite + +import ( + "context" + "database/sql" + "math/rand" + "time" + + _ "github.com/mattn/go-sqlite3" // import sqlite3 driver + "go.uber.org/zap" +) + +const ( + longQueryDuration = 10 * time.Millisecond + longTxnDuration = time.Second // reduce syncing spam +) + +type ( + // A scanner is an interface that wraps the Scan method of sql.Rows and sql.Row + scanner interface { + Scan(dest ...any) error + } + + // A stmt wraps a *sql.Stmt, logging slow queries. + stmt struct { + *sql.Stmt + query string + + log *zap.Logger + } + + // A txn wraps a *sql.Tx, logging slow queries. + txn struct { + *sql.Tx + log *zap.Logger + } + + // A row wraps a *sql.Row, logging slow queries. + row struct { + *sql.Row + log *zap.Logger + } + + // rows wraps a *sql.Rows, logging slow queries. + rows struct { + *sql.Rows + + log *zap.Logger + } +) + +func (r *rows) Next() bool { + start := time.Now() + next := r.Rows.Next() + if dur := time.Since(start); dur > longQueryDuration { + r.log.Debug("slow next", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return next +} + +func (r *rows) Scan(dest ...any) error { + start := time.Now() + err := r.Rows.Scan(dest...) + if dur := time.Since(start); dur > longQueryDuration { + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return err +} + +func (r *row) Scan(dest ...any) error { + start := time.Now() + err := r.Row.Scan(dest...) + if dur := time.Since(start); dur > longQueryDuration { + r.log.Debug("slow scan", zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return err +} + +func (s *stmt) Exec(args ...any) (sql.Result, error) { + return s.ExecContext(context.Background(), args...) +} + +func (s *stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) { + start := time.Now() + result, err := s.Stmt.ExecContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + s.log.Debug("slow exec", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return result, err +} + +func (s *stmt) Query(args ...any) (*sql.Rows, error) { + return s.QueryContext(context.Background(), args...) +} + +func (s *stmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) { + start := time.Now() + rows, err := s.Stmt.QueryContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + s.log.Debug("slow query", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return rows, err +} + +func (s *stmt) QueryRow(args ...any) *row { + return s.QueryRowContext(context.Background(), args...) +} + +func (s *stmt) QueryRowContext(ctx context.Context, args ...any) *row { + start := time.Now() + r := s.Stmt.QueryRowContext(ctx, args...) + if dur := time.Since(start); dur > longQueryDuration { + s.log.Debug("slow query row", zap.String("query", s.query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &row{r, s.log.Named("row")} +} + +// Exec executes a query without returning any rows. The args are for +// any placeholder parameters in the query. +func (tx *txn) Exec(query string, args ...any) (sql.Result, error) { + start := time.Now() + result, err := tx.Tx.Exec(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + tx.log.Debug("slow exec", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return result, err +} + +// Prepare creates a prepared statement for later queries or executions. +// Multiple queries or executions may be run concurrently from the +// returned statement. The caller must call the statement's Close method +// when the statement is no longer needed. +func (tx *txn) Prepare(query string) (*stmt, error) { + start := time.Now() + s, err := tx.Tx.Prepare(query) + if dur := time.Since(start); dur > longQueryDuration { + tx.log.Debug("slow prepare", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } else if err != nil { + return nil, err + } + return &stmt{ + Stmt: s, + query: query, + log: tx.log.Named("statement"), + }, nil +} + +// Query executes a query that returns rows, typically a SELECT. The +// args are for any placeholder parameters in the query. +func (tx *txn) Query(query string, args ...any) (*rows, error) { + start := time.Now() + r, err := tx.Tx.Query(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + tx.log.Debug("slow query", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &rows{r, tx.log.Named("rows")}, err +} + +// QueryRow executes a query that is expected to return at most one row. +// QueryRow always returns a non-nil value. Errors are deferred until +// Row's Scan method is called. If the query selects no rows, the *Row's +// Scan will return ErrNoRows. Otherwise, the *Row's Scan scans the +// first selected row and discards the rest. +func (tx *txn) QueryRow(query string, args ...any) *row { + start := time.Now() + r := tx.Tx.QueryRow(query, args...) + if dur := time.Since(start); dur > longQueryDuration { + tx.log.Debug("slow query row", zap.String("query", query), zap.Duration("elapsed", dur), zap.Stack("stack")) + } + return &row{r, tx.log.Named("row")} +} + +// getDBVersion returns the current version of the database. +func getDBVersion(db *sql.DB) (version int64) { + // error is ignored -- the database may not have been initialized yet. + db.QueryRow(`SELECT db_version FROM global_settings;`).Scan(&version) + return +} + +// setDBVersion sets the current version of the database. +func setDBVersion(tx *txn, version int64) error { + const query = `UPDATE global_settings SET db_version=$1 RETURNING id;` + var dbID int64 + return tx.QueryRow(query, version).Scan(&dbID) +} + +// jitterSleep sleeps for a random duration between t and t*1.5. +func jitterSleep(t time.Duration) { + time.Sleep(t + time.Duration(rand.Int63n(int64(t/2)))) +} diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go new file mode 100644 index 0000000..1929df5 --- /dev/null +++ b/persist/sqlite/store.go @@ -0,0 +1,126 @@ +package sqlite + +import ( + "database/sql" + "encoding/hex" + "errors" + "fmt" + "math" + "strings" + "time" + + "github.com/mattn/go-sqlite3" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "lukechampine.com/frand" +) + +type ( + // A Store is a persistent store that uses a SQL database as its backend. + Store struct { + indexMode wallet.IndexMode + + db *sql.DB + log *zap.Logger + } +) + +// Close closes the underlying database. +func (s *Store) Close() error { + return s.db.Close() +} + +// transaction executes a function within a database transaction. If the +// function returns an error, the transaction is rolled back. Otherwise, the +// transaction is committed. If the transaction fails due to a busy error, it is +// retried up to 10 times before returning. +func (s *Store) transaction(fn func(*txn) error) error { + var err error + txnID := hex.EncodeToString(frand.Bytes(4)) + log := s.log.Named("transaction").With(zap.String("id", txnID)) + start := time.Now() + attempt := 1 + for ; attempt < maxRetryAttempts; attempt++ { + attemptStart := time.Now() + log := log.With(zap.Int("attempt", attempt)) + err = doTransaction(s.db, log, fn) + if err == nil { + // no error, break out of the loop + return nil + } + + // return immediately if the error is not a busy error + if !strings.Contains(err.Error(), "database is locked") { + break + } + // exponential backoff + sleep := time.Duration(math.Pow(factor, float64(attempt))) * time.Millisecond + if sleep > maxBackoff { + sleep = maxBackoff + } + log.Debug("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) + jitterSleep(sleep) + } + return fmt.Errorf("transaction failed (attempt %d): %w", attempt, err) +} + +func sqliteFilepath(fp string) string { + params := []string{ + fmt.Sprintf("_busy_timeout=%d", busyTimeout), + "_foreign_keys=true", + "_journal_mode=WAL", + "_secure_delete=false", + "_cache_size=-65536", // 64MiB + } + return "file:" + fp + "?" + strings.Join(params, "&") +} + +// doTransaction is a helper function to execute a function within a transaction. If fn returns +// an error, the transaction is rolled back. Otherwise, the transaction is +// committed. +func doTransaction(db *sql.DB, log *zap.Logger, fn func(tx *txn) error) error { + dbtx, err := db.Begin() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + start := time.Now() + defer func() { + if err := dbtx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + log.Error("failed to rollback transaction", zap.Error(err)) + } + // log the transaction if it took longer than txn duration + if time.Since(start) > longTxnDuration { + log.Debug("long transaction", zap.Duration("elapsed", time.Since(start)), zap.Stack("stack"), zap.Bool("failed", err != nil)) + } + }() + + tx := &txn{ + Tx: dbtx, + log: log, + } + if err := fn(tx); err != nil { + return err + } else if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + return nil +} + +// OpenDatabase creates a new SQLite store and initializes the database. If the +// database does not exist, it is created. +func OpenDatabase(fp string, log *zap.Logger) (*Store, error) { + db, err := sql.Open("sqlite3", sqliteFilepath(fp)) + if err != nil { + return nil, err + } + store := &Store{ + db: db, + log: log, + } + if err := store.init(); err != nil { + return nil, err + } + sqliteVersion, _, _ := sqlite3.Version() + log.Debug("database initialized", zap.String("sqliteVersion", sqliteVersion), zap.Int("schemaVersion", len(migrations)+1), zap.String("path", fp)) + return store, nil +} diff --git a/persist/sqlite/utxo.go b/persist/sqlite/utxo.go new file mode 100644 index 0000000..3a8fc56 --- /dev/null +++ b/persist/sqlite/utxo.go @@ -0,0 +1,129 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +// SiacoinElement returns an unspent Siacoin UTXO by its ID. +func (s *Store) SiacoinElement(id types.SiacoinOutputID) (ele types.SiacoinElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address +FROM siacoin_elements se +INNER JOIN sia_addresses sa ON (se.address_id = sa.id) +WHERE se.id=$1 AND spent_index_id IS NULL` + + ele, err = scanSiacoinElement(tx.QueryRow(query, encode(id))) + if err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin element + if s.indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen + } + ele.StateElement.MerkleProof = proof[0] + } + return nil + }) + if errors.Is(err, sql.ErrNoRows) { + err = wallet.ErrNotFound + } + return +} + +// SiafundElement returns an unspent Siafund UTXO by its ID. +func (s *Store) SiafundElement(id types.SiafundOutputID) (ele types.SiafundElement, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address +FROM siafund_elements se +INNER JOIN sia_addresses sa ON (se.address_id = sa.id) +WHERE se.id=$1 AND spent_index_id IS NULL` + + ele, err = scanSiafundElement(tx.QueryRow(query, encode(id))) + if err != nil { + return err + } + + // retrieve the merkle proofs for the siafund element + if s.indexMode == wallet.IndexModeFull { + proof, err := fillElementProofs(tx, []uint64{ele.StateElement.LeafIndex}) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } else if len(proof) != 1 { + panic("expected exactly one proof") // should never happen + } + ele.StateElement.MerkleProof = proof[0] + } + return nil + }) + if errors.Is(err, sql.ErrNoRows) { + err = wallet.ErrNotFound + } + return +} + +// SiacoinElementSpentEvent returns the event that spent a Siacoin UTXO. +func (s *Store) SiacoinElementSpentEvent(id types.SiacoinOutputID) (ev wallet.Event, spent bool, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT spent_event_id FROM siacoin_elements WHERE id=$1` + + var spentEventID sql.NullInt64 + err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } else if err != nil { + return fmt.Errorf("failed to query spent event ID: %w", err) + } else if !spentEventID.Valid { + return nil + } + + spent = true + events, err := getEventsByID(tx, []int64{spentEventID.Int64}) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } else if len(events) != 1 { + panic("expected exactly one event") // should never happen + } + ev = events[0] + return nil + }) + return +} + +// SiafundElementSpentEvent returns the event that spent a Siafund UTXO. +func (s *Store) SiafundElementSpentEvent(id types.SiafundOutputID) (ev wallet.Event, spent bool, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT spent_event_id FROM siafund_elements WHERE id=$1` + + var spentEventID sql.NullInt64 + err = tx.QueryRow(query, encode(id)).Scan(&spentEventID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } else if err != nil { + return fmt.Errorf("failed to query spent event ID: %w", err) + } else if !spentEventID.Valid { + return nil + } + + spent = true + events, err := getEventsByID(tx, []int64{spentEventID.Int64}) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } else if len(events) != 1 { + panic("expected exactly one event") // should never happen + } + ev = events[0] + return nil + }) + + return +} diff --git a/persist/sqlite/wallet.go b/persist/sqlite/wallet.go new file mode 100644 index 0000000..06e2733 --- /dev/null +++ b/persist/sqlite/wallet.go @@ -0,0 +1,741 @@ +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "math/bits" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/walletd/wallet" +) + +func (s *Store) getWalletEventRelevantAddresses(tx *txn, id wallet.ID, eventIDs []int64) (map[int64][]types.Address, error) { + stmt, err := tx.Prepare(`SELECT sa.sia_address +FROM event_addresses ea +INNER JOIN sia_addresses sa ON (ea.address_id = sa.id) +INNER JOIN wallet_addresses wa ON (ea.address_id = wa.address_id) +WHERE wa.wallet_id=? AND ea.event_id=?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + relevant := func(walletID wallet.ID, eventID int64) (addresses []types.Address, err error) { + rows, err := stmt.Query(walletID, eventID) + if err != nil { + return nil, fmt.Errorf("failed to query relevant addresses: %w", err) + } + defer rows.Close() + + for rows.Next() { + var address types.Address + if err := rows.Scan(decode(&address)); err != nil { + return nil, fmt.Errorf("failed to scan relevant address: %w", err) + } + addresses = append(addresses, address) + } + return addresses, rows.Err() + } + + relevantAddresses := make(map[int64][]types.Address) + for _, eventID := range eventIDs { + addresses, err := relevant(id, eventID) + if err != nil { + return nil, err + } + relevantAddresses[eventID] = addresses + } + return relevantAddresses, nil +} + +// WalletEvents returns the events relevant to a wallet, sorted by height descending. +func (s *Store) WalletEvents(id wallet.ID, offset, limit int) (events []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + dbIDs, err := getWalletEvents(tx, id, offset, limit) + if err != nil { + return fmt.Errorf("failed to get wallet events: %w", err) + } + + events, err = getEventsByID(tx, dbIDs) + if err != nil { + return fmt.Errorf("failed to get events by ID: %w", err) + } + + eventRelevantAddresses, err := s.getWalletEventRelevantAddresses(tx, id, dbIDs) + if err != nil { + return fmt.Errorf("failed to get relevant addresses: %w", err) + } + + for i := range events { + events[i].Relevant = eventRelevantAddresses[dbIDs[i]] + } + return nil + }) + return +} + +// AddWallet adds a wallet to the database. +func (s *Store) AddWallet(w wallet.Wallet) (wallet.Wallet, error) { + w.DateCreated = time.Now().Truncate(time.Second) + w.LastUpdated = time.Now().Truncate(time.Second) + + err := s.transaction(func(tx *txn) error { + const query = `INSERT INTO wallets (friendly_name, description, date_created, last_updated, extra_data) VALUES ($1, $2, $3, $4, $5) RETURNING id` + return tx.QueryRow(query, w.Name, w.Description, encode(w.DateCreated), encode(w.LastUpdated), w.Metadata).Scan(&w.ID) + }) + return w, err +} + +// UpdateWallet updates a wallet in the database. +func (s *Store) UpdateWallet(w wallet.Wallet) (wallet.Wallet, error) { + w.LastUpdated = time.Now() + err := s.transaction(func(tx *txn) error { + var dummyID int64 + const query = `UPDATE wallets SET friendly_name=$1, description=$2, last_updated=$3, extra_data=$4 WHERE id=$5 RETURNING id, date_created, last_updated` + err := tx.QueryRow(query, w.Name, w.Description, encode(w.LastUpdated), w.Metadata, w.ID).Scan(&dummyID, decode(&w.DateCreated), decode(&w.LastUpdated)) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } + return err + }) + return w, err +} + +// DeleteWallet deletes a wallet from the database. This does not stop tracking +// addresses that were previously associated with the wallet. +func (s *Store) DeleteWallet(id wallet.ID) error { + return s.transaction(func(tx *txn) error { + _, err := tx.Exec(`DELETE FROM wallet_addresses WHERE wallet_id=$1`, id) + if err != nil { + return fmt.Errorf("failed to delete wallet addresses: %w", err) + } + + var dummyID int64 + err = tx.QueryRow(`DELETE FROM wallets WHERE id=$1 RETURNING id`, id).Scan(&dummyID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } + return err + }) +} + +// Wallets returns a map of wallet names to wallet extra data. +func (s *Store) Wallets() (wallets []wallet.Wallet, err error) { + err = s.transaction(func(tx *txn) error { + const query = `SELECT id, friendly_name, description, date_created, last_updated, extra_data FROM wallets` + + rows, err := tx.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var w wallet.Wallet + if err := rows.Scan(&w.ID, &w.Name, &w.Description, decode(&w.DateCreated), decode(&w.LastUpdated), (*[]byte)(&w.Metadata)); err != nil { + return fmt.Errorf("failed to scan wallet: %w", err) + } + wallets = append(wallets, w) + } + return rows.Err() + }) + return +} + +// AddWalletAddress adds an address to a wallet. +func (s *Store) AddWalletAddress(id wallet.ID, addr wallet.Address) error { + return s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + addressID, err := insertAddress(tx, addr.Address) + if err != nil { + return fmt.Errorf("failed to insert address: %w", err) + } + + var encodedPolicy any + if addr.SpendPolicy != nil { + encodedPolicy = encode(*addr.SpendPolicy) + } + + _, err = tx.Exec(`INSERT INTO wallet_addresses (wallet_id, address_id, description, spend_policy, extra_data) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (wallet_id, address_id) DO UPDATE set description=EXCLUDED.description, spend_policy=EXCLUDED.spend_policy, extra_data=EXCLUDED.extra_data`, id, addressID, addr.Description, encodedPolicy, addr.Metadata) + return err + }) +} + +// RemoveWalletAddress removes an address from a wallet. This does not stop tracking +// the address. +func (s *Store) RemoveWalletAddress(id wallet.ID, address types.Address) error { + return s.transaction(func(tx *txn) error { + const query = `DELETE FROM wallet_addresses WHERE wallet_id=$1 AND address_id=(SELECT id FROM sia_addresses WHERE sia_address=$2) RETURNING address_id` + var dummyID int64 + err := tx.QueryRow(query, id, encode(address)).Scan(&dummyID) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } + return err + }) +} + +// WalletAddress returns an address registered to the wallet. +func (s *Store) WalletAddress(id wallet.ID, address types.Address) (addr wallet.Address, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data +FROM wallet_addresses wa +INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) +WHERE wa.wallet_id=$1 AND sa.sia_address=$2` + + addr, err = scanWalletAddress(tx.QueryRow(query, id, encode(address))) + return err + }) + return +} + +// WalletAddresses returns a slice of addresses registered to the wallet. +func (s *Store) WalletAddresses(id wallet.ID) (addresses []wallet.Address, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + const query = `SELECT sa.sia_address, wa.description, wa.spend_policy, wa.extra_data +FROM wallet_addresses wa +INNER JOIN sia_addresses sa ON (sa.id = wa.address_id) +WHERE wa.wallet_id=$1` + + rows, err := tx.Query(query, id) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + addr, err := scanWalletAddress(rows) + if err != nil { + return fmt.Errorf("failed to scan address: %w", err) + } + addresses = append(addresses, addr) + } + return rows.Err() + }) + return +} + +// WalletSiacoinOutputs returns the unspent siacoin outputs for a wallet. +func (s *Store) WalletSiacoinOutputs(id wallet.ID, offset, limit int) (siacoins []types.SiacoinElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + const query = `SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.spent_index_id IS NULL AND se.maturity_height <= $1 AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$2) + LIMIT $3 OFFSET $4` + + rows, err := tx.Query(query, basis.Height, id, limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siacoin, err := scanSiacoinElement(rows) + if err != nil { + return fmt.Errorf("failed to scan siacoin element: %w", err) + } + + siacoins = append(siacoins, siacoin) + } + + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siacoins)) + for i, se := range siacoins { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siacoins[i].StateElement.MerkleProof = proof + } + } + return nil + }) + return +} + +// WalletSiafundOutputs returns the unspent siafund outputs for a wallet. +func (s *Store) WalletSiafundOutputs(id wallet.ID, offset, limit int) (siafunds []types.SiafundElement, basis types.ChainIndex, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + basis, err = getScanBasis(tx) + if err != nil { + return fmt.Errorf("failed to get basis: %w", err) + } + + const query = `SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.spent_index_id IS NULL AND se.address_id IN (SELECT address_id FROM wallet_addresses WHERE wallet_id=$1) + LIMIT $2 OFFSET $3` + + rows, err := tx.Query(query, id, limit, offset) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + siafund, err := scanSiafundElement(rows) + if err != nil { + return fmt.Errorf("failed to scan siafund element: %w", err) + } + siafunds = append(siafunds, siafund) + } + if err := rows.Err(); err != nil { + return err + } + + // retrieve the merkle proofs for the siacoin elements + if s.indexMode == wallet.IndexModeFull { + indices := make([]uint64, len(siafunds)) + for i, se := range siafunds { + indices[i] = se.StateElement.LeafIndex + } + proofs, err := fillElementProofs(tx, indices) + if err != nil { + return fmt.Errorf("failed to fill element proofs: %w", err) + } + for i, proof := range proofs { + siafunds[i].StateElement.MerkleProof = proof + } + } + return nil + }) + return +} + +// WalletBalance returns the total balance of a wallet. +func (s *Store) WalletBalance(id wallet.ID) (balance wallet.Balance, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + const query = `SELECT siacoin_balance, immature_siacoin_balance, siafund_balance FROM sia_addresses sa + INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) + WHERE wa.wallet_id=$1` + + rows, err := tx.Query(query, id) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var addressSC types.Currency + var addressISC types.Currency + var addressSF uint64 + + if err := rows.Scan(decode(&addressSC), decode(&addressISC), &addressSF); err != nil { + return fmt.Errorf("failed to scan address balance: %w", err) + } + balance.Siacoins = balance.Siacoins.Add(addressSC) + balance.ImmatureSiacoins = balance.ImmatureSiacoins.Add(addressISC) + balance.Siafunds += addressSF + } + return rows.Err() + }) + return +} + +// WalletUnconfirmedEvents annotates a list of unconfirmed transactions with +// relevant addresses and siacoin/siafund elements. +func (s *Store) WalletUnconfirmedEvents(id wallet.ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []wallet.Event, err error) { + err = s.transaction(func(tx *txn) error { + if err := walletExists(tx, id); err != nil { + return err + } + + addrStmt, err := tx.Prepare(`SELECT sa.id FROM sia_addresses sa + INNER JOIN wallet_addresses wa ON (sa.id = wa.address_id) + WHERE wa.wallet_id=$1 AND sa.sia_address=$2 LIMIT 1`) + if err != nil { + return fmt.Errorf("failed to prepare address statement: %w", err) + } + defer addrStmt.Close() + + // note: this would be more performant for small wallets to load all + // addresses into memory. However, for larger wallets (> 10K addresses), + // this is time consuming. Instead, the database is queried for each + // address. Monitor performance and consider changing this in the + // future. From a memory perspective, it would be fine to lazy load all + // addresses into memory. + checkedAddresses := make(map[types.Address]bool) + ownsAddress := func(address types.Address) bool { + if relevant, ok := checkedAddresses[address]; ok { + return relevant + } + + var dbID int64 + err := addrStmt.QueryRow(id, encode(address)).Scan(&dbID) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + panic(err) // database error + } + relevant := err == nil + checkedAddresses[address] = relevant + return relevant + } + + siacoinElementStmt, err := tx.Prepare(`SELECT se.id, se.siacoin_value, se.merkle_proof, se.leaf_index, se.maturity_height, sa.sia_address + FROM siacoin_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siacoin statement: %w", err) + } + defer siacoinElementStmt.Close() + + siacoinElementCache := make(map[types.SiacoinOutputID]types.SiacoinElement) + fetchSiacoinElement := func(id types.SiacoinOutputID) (types.SiacoinElement, error) { + if se, ok := siacoinElementCache[id]; ok { + return se, nil + } + + se, err := scanSiacoinElement(siacoinElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiacoinElement{}, fmt.Errorf("failed to fetch siacoin element: %w", err) + } + siacoinElementCache[id] = se + return se, nil + } + + siafundElementStmt, err := tx.Prepare(`SELECT se.id, se.leaf_index, se.merkle_proof, se.siafund_value, se.claim_start, sa.sia_address + FROM siafund_elements se + INNER JOIN sia_addresses sa ON (se.address_id = sa.id) + WHERE se.id=$1`) + if err != nil { + return fmt.Errorf("failed to prepare siafund statement: %w", err) + } + defer siafundElementStmt.Close() + + siafundElementCache := make(map[types.SiafundOutputID]types.SiafundElement) + fetchSiafundElement := func(id types.SiafundOutputID) (types.SiafundElement, error) { + if se, ok := siafundElementCache[id]; ok { + return se, nil + } + + se, err := scanSiafundElement(siafundElementStmt.QueryRow(encode(id))) + if err != nil { + return types.SiafundElement{}, fmt.Errorf("failed to fetch siafund element: %w", err) + } + siafundElementCache[id] = se + return se, nil + } + + addEvent := func(id types.Hash256, eventType string, data wallet.EventData, relevant []types.Address) { + annotated = append(annotated, wallet.Event{ + ID: id, + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height + 1, + Type: eventType, + Data: data, + Relevant: relevant, + }) + } + + for _, txn := range v1 { + var relevant []types.Address + seen := make(map[types.Address]bool) + ev := wallet.EventV1Transaction{ + Transaction: txn, + } + + for _, input := range txn.SiacoinInputs { + address := input.UnlockConditions.UnlockHash() + if !ownsAddress(address) { + continue + } + + if !seen[address] { + seen[address] = true + relevant = append(relevant, address) + } + + // fetch the siacoin element + sce, err := fetchSiacoinElement(input.ParentID) + if err != nil { + return fmt.Errorf("failed to fetch siacoin element %q: %w", input.ParentID, err) + } + ev.SpentSiacoinElements = append(ev.SpentSiacoinElements, sce) + } + + for i, output := range txn.SiacoinOutputs { + if !ownsAddress(output.Address) { + continue + } + + if !seen[output.Address] { + seen[output.Address] = true + relevant = append(relevant, output.Address) + } + + sce := types.SiacoinElement{ + ID: txn.SiacoinOutputID(i), + StateElement: types.StateElement{ + LeafIndex: types.UnassignedLeafIndex, + }, + SiacoinOutput: output, + } + siacoinElementCache[sce.ID] = sce + } + + for _, input := range txn.SiafundInputs { + address := input.UnlockConditions.UnlockHash() + if !ownsAddress(address) { + continue + } + + if !seen[address] { + seen[address] = true + relevant = append(relevant, address) + } + + // fetch the siafund element + sfe, err := fetchSiafundElement(input.ParentID) + if err != nil { + return fmt.Errorf("failed to fetch siafund element %q: %w", input.ParentID, err) + } + ev.SpentSiafundElements = append(ev.SpentSiafundElements, sfe) + } + + for i, output := range txn.SiafundOutputs { + if !ownsAddress(output.Address) { + continue + } + + if !seen[output.Address] { + seen[output.Address] = true + relevant = append(relevant, output.Address) + } + + sfe := types.SiafundElement{ + ID: txn.SiafundOutputID(i), + StateElement: types.StateElement{ + LeafIndex: types.UnassignedLeafIndex, + }, + SiafundOutput: output, + } + siafundElementCache[sfe.ID] = sfe + } + + if len(relevant) == 0 { + continue + } + addEvent(types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, ev, relevant) + } + + // only need to check if the address is relevant for v2 transactions + // the inputs contain the necessary metadata for calculating value + for _, txn := range v2 { + var relevant []types.Address + seen := make(map[types.Address]bool) + + for _, sci := range txn.SiacoinInputs { + if !ownsAddress(sci.Parent.SiacoinOutput.Address) || seen[sci.Parent.SiacoinOutput.Address] { + continue + } + seen[sci.Parent.SiacoinOutput.Address] = true + relevant = append(relevant, sci.Parent.SiacoinOutput.Address) + } + + for _, sco := range txn.SiacoinOutputs { + if !ownsAddress(sco.Address) || seen[sco.Address] { + continue + } + seen[sco.Address] = true + relevant = append(relevant, sco.Address) + } + + for _, sfi := range txn.SiafundInputs { + if !ownsAddress(sfi.Parent.SiafundOutput.Address) || seen[sfi.Parent.SiafundOutput.Address] { + continue + } + seen[sfi.Parent.SiafundOutput.Address] = true + relevant = append(relevant, sfi.Parent.SiafundOutput.Address) + } + + for _, sfo := range txn.SiafundOutputs { + if !ownsAddress(sfo.Address) || seen[sfo.Address] { + continue + } + seen[sfo.Address] = true + relevant = append(relevant, sfo.Address) + } + + if len(relevant) == 0 { + continue + } + + addEvent(types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, wallet.EventV2Transaction(txn), relevant) + } + return nil + }) + return +} + +func scanSiacoinElement(s scanner) (se types.SiacoinElement, err error) { + err = s.Scan(decode(&se.ID), decode(&se.SiacoinOutput.Value), decode(&se.StateElement.MerkleProof), &se.StateElement.LeafIndex, &se.MaturityHeight, decode(&se.SiacoinOutput.Address)) + return +} + +func scanSiafundElement(s scanner) (se types.SiafundElement, err error) { + err = s.Scan(decode(&se.ID), &se.StateElement.LeafIndex, decode(&se.StateElement.MerkleProof), &se.SiafundOutput.Value, decode(&se.ClaimStart), decode(&se.SiafundOutput.Address)) + return +} + +func insertAddress(tx *txn, addr types.Address) (id int64, err error) { + const query = `INSERT INTO sia_addresses (sia_address, siacoin_balance, immature_siacoin_balance, siafund_balance) +VALUES ($1, $2, $3, 0) ON CONFLICT (sia_address) DO UPDATE SET sia_address=EXCLUDED.sia_address +RETURNING id` + + err = tx.QueryRow(query, encode(addr), encode(types.ZeroCurrency), encode(types.ZeroCurrency)).Scan(&id) + return +} + +func scanWalletAddress(s scanner) (wallet.Address, error) { + var address wallet.Address + var decodedPolicy any + if err := s.Scan(decode(&address.Address), &address.Description, &decodedPolicy, (*[]byte)(&address.Metadata)); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return wallet.Address{}, wallet.ErrNotFound + } + return wallet.Address{}, fmt.Errorf("failed to scan address: %w", err) + } + + if decodedPolicy != nil { + switch v := decodedPolicy.(type) { + case []byte: + dec := types.NewBufDecoder(v) + address.SpendPolicy = new(types.SpendPolicy) + address.SpendPolicy.DecodeFrom(dec) + if err := dec.Err(); err != nil { + return wallet.Address{}, fmt.Errorf("failed to decode spend policy: %w", err) + } + default: + return wallet.Address{}, fmt.Errorf("unexpected spend policy type: %T", decodedPolicy) + } + } + return address, nil +} + +func getScanBasis(tx *txn) (index types.ChainIndex, err error) { + err = tx.QueryRow(`SELECT last_indexed_id, last_indexed_height FROM global_settings`).Scan(decode(&index.ID), &index.Height) + return +} + +func fillElementProofs(tx *txn, indices []uint64) (proofs [][]types.Hash256, _ error) { + if len(indices) == 0 { + return nil, nil + } + + var numLeaves uint64 + if err := tx.QueryRow(`SELECT element_num_leaves FROM global_settings LIMIT 1`).Scan(&numLeaves); err != nil { + return nil, fmt.Errorf("failed to query state tree leaves: %w", err) + } + + stmt, err := tx.Prepare(`SELECT value FROM state_tree WHERE row=? AND column=?`) + if err != nil { + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + data := make(map[uint64]map[uint64]types.Hash256) + for _, leafIndex := range indices { + proof := make([]types.Hash256, bits.Len64(leafIndex^numLeaves)-1) + for j := range proof { + row, col := uint64(j), (leafIndex>>j)^1 + + // check if the hash is already in the cache + if h, ok := data[row][col]; ok { + proof[j] = h + continue + } + + // query the hash from the database + if err := stmt.QueryRow(row, col).Scan(decode(&proof[j])); err != nil { + return nil, fmt.Errorf("failed to query state element (%d,%d): %w", row, col, err) + } + + // cache the hash + if _, ok := data[row]; !ok { + data[row] = make(map[uint64]types.Hash256) + } + data[row][col] = proof[j] + } + proofs = append(proofs, proof) + } + return +} + +func getWalletEvents(tx *txn, id wallet.ID, offset, limit int) (eventIDs []int64, err error) { + const eventsQuery = `SELECT DISTINCT ea.event_id +FROM event_addresses ea +INNER JOIN sia_addresses sa ON ea.address_id = sa.id +INNER JOIN wallet_addresses wa ON sa.id = wa.address_id +WHERE wa.wallet_id = $1 +ORDER BY ea.event_maturity_height DESC, ea.event_id DESC +LIMIT $2 OFFSET $3;` + + rows, err := tx.Query(eventsQuery, id, limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var eventID int64 + if err := rows.Scan(&eventID); err != nil { + return nil, fmt.Errorf("failed to scan event ID: %w", err) + } + eventIDs = append(eventIDs, eventID) + } + if err := rows.Err(); err != nil { + return nil, err + } + return +} + +func walletExists(tx *txn, id wallet.ID) error { + const query = `SELECT 1 FROM wallets WHERE id=$1` + var dummy int + err := tx.QueryRow(query, id).Scan(&dummy) + if errors.Is(err, sql.ErrNoRows) { + return wallet.ErrNotFound + } + return err +} diff --git a/syncer/syncer.go b/syncer/syncer.go deleted file mode 100644 index ec1597b..0000000 --- a/syncer/syncer.go +++ /dev/null @@ -1,922 +0,0 @@ -package syncer - -import ( - "context" - "errors" - "io" - "log" - "net" - "reflect" - "sync" - "time" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" - "go.sia.tech/core/types" - "lukechampine.com/frand" -) - -// A ChainManager manages blockchain state. -type ChainManager interface { - History() ([32]types.BlockID, error) - BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) - Block(id types.BlockID) (types.Block, bool) - SyncCheckpoint(index types.ChainIndex) (types.Block, consensus.State, bool) - AddBlocks(blocks []types.Block) error - Tip() types.ChainIndex - TipState() consensus.State - - PoolTransaction(txid types.TransactionID) (types.Transaction, bool) - AddPoolTransactions(txns []types.Transaction) error - V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) - AddV2PoolTransactions(txns []types.V2Transaction) error - TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) -} - -// PeerInfo contains metadata about a peer. -type PeerInfo struct { - FirstSeen time.Time `json:"firstSeen"` - LastConnect time.Time `json:"lastConnect,omitempty"` - SyncedBlocks uint64 `json:"syncedBlocks,omitempty"` - SyncDuration time.Duration `json:"syncDuration,omitempty"` -} - -// A PeerStore stores peers and bans. -type PeerStore interface { - AddPeer(peer string) - Peers() []string - UpdatePeerInfo(peer string, fn func(*PeerInfo)) - PeerInfo(peer string) (PeerInfo, bool) - - // Ban temporarily bans one or more IPs. The addr should either be a single - // IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. 1.2.3.4/16). - Ban(addr string, duration time.Duration, reason string) - Banned(peer string) bool -} - -// Subnet normalizes the provided CIDR subnet string. -func Subnet(cidr string) string { - ip, ipnet, err := net.ParseCIDR(cidr) - if err != nil { - return "" // shouldn't happen - } - return ip.Mask(ipnet.Mask).String() + cidr -} - -type config struct { - MaxInboundPeers int - MaxOutboundPeers int - MaxInflightRPCs int - ConnectTimeout time.Duration - ShareNodesTimeout time.Duration - SendBlockTimeout time.Duration - SendTransactionsTimeout time.Duration - RelayHeaderTimeout time.Duration - RelayBlockOutlineTimeout time.Duration - RelayTransactionSetTimeout time.Duration - SendBlocksTimeout time.Duration - MaxSendBlocks uint64 - PeerDiscoveryInterval time.Duration - SyncInterval time.Duration - Logger *log.Logger -} - -// An Option modifies a Syncer's configuration. -type Option func(*config) - -// WithMaxInboundPeers sets the maximum number of inbound connections. The -// default is 8. -func WithMaxInboundPeers(n int) Option { - return func(c *config) { c.MaxInboundPeers = n } -} - -// WithMaxOutboundPeers sets the maximum number of outbound connections. The -// default is 8. -func WithMaxOutboundPeers(n int) Option { - return func(c *config) { c.MaxOutboundPeers = n } -} - -// WithMaxInflightRPCs sets the maximum number of concurrent RPCs per peer. The -// default is 3. -func WithMaxInflightRPCs(n int) Option { - return func(c *config) { c.MaxInflightRPCs = n } -} - -// WithConnectTimeout sets the timeout when connecting to a peer. The default is -// 5 seconds. -func WithConnectTimeout(d time.Duration) Option { - return func(c *config) { c.ConnectTimeout = d } -} - -// WithShareNodesTimeout sets the timeout for the ShareNodes RPC. The default is -// 5 seconds. -func WithShareNodesTimeout(d time.Duration) Option { - return func(c *config) { c.ShareNodesTimeout = d } -} - -// WithSendBlockTimeout sets the timeout for the SendBlock RPC. The default is -// 60 seconds. -func WithSendBlockTimeout(d time.Duration) Option { - return func(c *config) { c.SendBlockTimeout = d } -} - -// WithSendBlocksTimeout sets the timeout for the SendBlocks RPC. The default is -// 120 seconds. -func WithSendBlocksTimeout(d time.Duration) Option { - return func(c *config) { c.SendBlocksTimeout = d } -} - -// WithMaxSendBlocks sets the maximum number of blocks requested per SendBlocks -// RPC. The default is 10. -func WithMaxSendBlocks(n uint64) Option { - return func(c *config) { c.MaxSendBlocks = n } -} - -// WithSendTransactionsTimeout sets the timeout for the SendTransactions RPC. -// The default is 60 seconds. -func WithSendTransactionsTimeout(d time.Duration) Option { - return func(c *config) { c.SendTransactionsTimeout = d } -} - -// WithRelayHeaderTimeout sets the timeout for the RelayHeader and RelayV2Header -// RPCs. The default is 5 seconds. -func WithRelayHeaderTimeout(d time.Duration) Option { - return func(c *config) { c.RelayHeaderTimeout = d } -} - -// WithRelayBlockOutlineTimeout sets the timeout for the RelayV2BlockOutline -// RPC. The default is 60 seconds. -func WithRelayBlockOutlineTimeout(d time.Duration) Option { - return func(c *config) { c.RelayBlockOutlineTimeout = d } -} - -// WithRelayTransactionSetTimeout sets the timeout for the RelayTransactionSet -// RPC. The default is 60 seconds. -func WithRelayTransactionSetTimeout(d time.Duration) Option { - return func(c *config) { c.RelayTransactionSetTimeout = d } -} - -// WithPeerDiscoveryInterval sets the frequency at which the syncer attempts to -// discover and connect to new peers. The default is 5 seconds. -func WithPeerDiscoveryInterval(d time.Duration) Option { - return func(c *config) { c.PeerDiscoveryInterval = d } -} - -// WithSyncInterval sets the frequency at which the syncer attempts to sync with -// peers. The default is 5 seconds. -func WithSyncInterval(d time.Duration) Option { - return func(c *config) { c.SyncInterval = d } -} - -// WithLogger sets the logger used by a Syncer. The default is a logger that -// outputs to io.Discard. -func WithLogger(l *log.Logger) Option { - return func(c *config) { c.Logger = l } -} - -// A Syncer synchronizes blockchain data with peers. -type Syncer struct { - l net.Listener - cm ChainManager - pm PeerStore - header gateway.Header - config config - log *log.Logger // redundant, but convenient - - mu sync.Mutex - peers map[string]*gateway.Peer - synced map[string]bool - strikes map[string]int -} - -type rpcHandler struct { - s *Syncer -} - -func (h *rpcHandler) PeersForShare() (peers []string) { - peers = h.s.pm.Peers() - if len(peers) > 10 { - frand.Shuffle(len(peers), reflect.Swapper(peers)) - peers = peers[:10] - } - return peers -} - -func (h *rpcHandler) Block(id types.BlockID) (types.Block, error) { - b, ok := h.s.cm.Block(id) - if !ok { - return types.Block{}, errors.New("block not found") - } - return b, nil -} - -func (h *rpcHandler) BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) { - return h.s.cm.BlocksForHistory(history, max) -} - -func (h *rpcHandler) Transactions(index types.ChainIndex, txnHashes []types.Hash256) (txns []types.Transaction, v2txns []types.V2Transaction, _ error) { - if b, ok := h.s.cm.Block(index.ID); ok { - // get txns from block - want := make(map[types.Hash256]bool) - for _, h := range txnHashes { - want[h] = true - } - for _, txn := range b.Transactions { - if want[txn.FullHash()] { - txns = append(txns, txn) - } - } - for _, txn := range b.V2Transactions() { - if want[txn.FullHash()] { - v2txns = append(v2txns, txn) - } - } - return - } - txns, v2txns = h.s.cm.TransactionsForPartialBlock(txnHashes) - return -} - -func (h *rpcHandler) Checkpoint(index types.ChainIndex) (types.Block, consensus.State, error) { - b, cs, ok := h.s.cm.SyncCheckpoint(index) - if !ok { - return types.Block{}, consensus.State{}, errors.New("checkpoint not found") - } - return b, cs, nil -} - -func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bh.ID()); ok { - return // already seen - } else if _, ok := h.s.cm.Block(bh.ParentID); !ok { - h.s.log.Printf("peer %v relayed a header with unknown parent (%v); triggering a resync", origin, bh.ParentID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } else if cs := h.s.cm.TipState(); bh.ParentID != cs.Index.ID { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } else if bh.ID().CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) - return - } - - // header is valid and attaches to our tip; request + validate full block - if b, err := origin.SendBlock(bh.ID(), h.s.config.SendBlockTimeout); err != nil { - // log-worthy, but not ban-worthy - h.s.log.Printf("couldn't retrieve new block %v after header relay from %v: %v", bh.ID(), origin, err) - return - } else if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { - h.s.ban(origin, err) - return - } - - h.s.relayHeader(bh, origin) // non-blocking -} - -func (h *rpcHandler) RelayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { - // if we've already seen these transactions, don't relay them again - for _, txn := range txns { - if _, ok := h.s.cm.PoolTransaction(txn.ID()); !ok { - goto add - } - } - return - -add: - if err := h.s.cm.AddPoolTransactions(txns); err != nil { - // too risky to ban here (txns are probably just outdated), but at least - // log it if we think we're synced - if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - } - return - } - h.s.relayTransactionSet(txns, origin) // non-blocking -} - -func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bh.Parent.ID); !ok { - h.s.log.Printf("peer %v relayed a v2 header with unknown parent (%v); triggering a resync", origin, bh.Parent.ID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } - cs := h.s.cm.TipState() - bid := bh.ID(cs) - if _, ok := h.s.cm.Block(bid); ok { - // already seen - return - } else if bh.Parent.ID != cs.Index.ID { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) - return - } - - // header is sufficiently valid; relay it - // - // NOTE: The purpose of header announcements is to inform the network as - // quickly as possible that a new block has been found. A proper - // BlockOutline should follow soon after, allowing peers to obtain the - // actual block. As such, we take no action here other than relaying. - h.s.relayV2Header(bh, origin) // non-blocking -} - -func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { - if _, ok := h.s.cm.Block(bo.ParentID); !ok { - h.s.log.Printf("peer %v relayed a header with unknown parent (%v); triggering a resync", origin, bo.ParentID) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } - cs := h.s.cm.TipState() - bid := bo.ID(cs) - if _, ok := h.s.cm.Block(bid); ok { - // already seen - return - } else if bo.ParentID != cs.Index.ID { - // block extends a sidechain, which peer (if honest) believes to be the - // heaviest chain - h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) - h.s.mu.Lock() - h.s.synced[origin.Addr] = false - h.s.mu.Unlock() - return - } else if bid.CmpWork(cs.ChildTarget) < 0 { - h.s.ban(origin, errors.New("peer sent header with insufficient work")) - return - } - - // block has sufficient work and attaches to our tip, but may be missing - // transactions; first, check for them in our txpool; then, if block is - // still incomplete, request remaining transactions from the peer - txns, v2txns := h.s.cm.TransactionsForPartialBlock(bo.Missing()) - b, missing := bo.Complete(cs, txns, v2txns) - if len(missing) > 0 { - index := types.ChainIndex{ID: bid, Height: cs.Index.Height + 1} - txns, v2txns, err := origin.SendTransactions(index, missing, h.s.config.SendTransactionsTimeout) - if err != nil { - // log-worthy, but not ban-worthy - h.s.log.Printf("couldn't retrieve missing transactions of %v after relay from %v: %v", bid, origin, err) - return - } - b, missing = bo.Complete(cs, txns, v2txns) - if len(missing) > 0 { - // inexcusable - h.s.ban(origin, errors.New("peer sent wrong missing transactions for a block it relayed")) - return - } - } - if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { - h.s.ban(origin, err) - return - } - - // when we forward the block, exclude any txns that were in our txpool, - // since they're probably present in our peers' txpools as well - // - // NOTE: crucially, we do NOT exclude any txns we had to request from the - // sending peer, since other peers probably don't have them either - bo.RemoveTransactions(txns, v2txns) - - h.s.relayV2BlockOutline(bo, origin) // non-blocking -} - -func (h *rpcHandler) RelayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { - // if we've already seen these transactions, don't relay them again - for _, txn := range txns { - if _, ok := h.s.cm.V2PoolTransaction(txn.ID()); !ok { - goto add - } - } - return - -add: - if err := h.s.cm.AddV2PoolTransactions(txns); err != nil { - // too risky to ban here (txns are probably just outdated), but at least - // log it if we think we're synced - if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { - h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) - } - return - } - h.s.relayV2TransactionSet(txns, origin) // non-blocking -} - -func (s *Syncer) ban(p *gateway.Peer, err error) { - p.SetErr(errors.New("banned")) - s.pm.Ban(p.ConnAddr, 24*time.Hour, err.Error()) - - host, _, err := net.SplitHostPort(p.ConnAddr) - if err != nil { - return // shouldn't happen - } - // add a strike to each subnet - for subnet, maxStrikes := range map[string]int{ - Subnet(host + "/32"): 2, // 1.2.3.4:* - Subnet(host + "/24"): 8, // 1.2.3.* - Subnet(host + "/16"): 64, // 1.2.* - Subnet(host + "/8"): 512, // 1.* - } { - s.mu.Lock() - ban := (s.strikes[subnet] + 1) >= maxStrikes - if ban { - delete(s.strikes, subnet) - } else { - s.strikes[subnet]++ - } - s.mu.Unlock() - if ban { - s.pm.Ban(subnet, 24*time.Hour, "too many strikes") - } - } -} - -func (s *Syncer) runPeer(p *gateway.Peer) { - s.pm.AddPeer(p.Addr) - s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { - info.LastConnect = time.Now() - }) - s.mu.Lock() - s.peers[p.Addr] = p - s.mu.Unlock() - defer func() { - s.mu.Lock() - delete(s.peers, p.Addr) - s.mu.Unlock() - }() - - h := &rpcHandler{s: s} - inflight := make(chan struct{}, s.config.MaxInflightRPCs) - for { - if p.Err() != nil { - return - } - id, stream, err := p.AcceptRPC() - if err != nil { - p.SetErr(err) - return - } - inflight <- struct{}{} - go func() { - defer stream.Close() - // NOTE: we do not set any deadlines on the stream. If a peer is - // slow, fine; we don't need to worry about resource exhaustion - // unless we have tons of peers. - if err := p.HandleRPC(id, stream, h); err != nil { - s.log.Printf("incoming RPC %v from peer %v failed: %v", id, p, err) - } - <-inflight - }() - } -} - -func (s *Syncer) relayHeader(h gateway.BlockHeader, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin { - continue - } - go p.RelayHeader(h, s.config.RelayHeaderTimeout) - } -} - -func (s *Syncer) relayTransactionSet(txns []types.Transaction, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin { - continue - } - go p.RelayTransactionSet(txns, s.config.RelayTransactionSetTimeout) - } -} - -func (s *Syncer) relayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2Header(bh, s.config.RelayHeaderTimeout) - } -} - -func (s *Syncer) relayV2BlockOutline(pb gateway.V2BlockOutline, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2BlockOutline(pb, s.config.RelayBlockOutlineTimeout) - } -} - -func (s *Syncer) relayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p == origin || !p.SupportsV2() { - continue - } - go p.RelayV2TransactionSet(txns, s.config.RelayTransactionSetTimeout) - } -} - -func (s *Syncer) allowConnect(peer string, inbound bool) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.l == nil { - return errors.New("syncer is shutting down") - } - if s.pm.Banned(peer) { - return errors.New("banned") - } - var in, out int - for _, p := range s.peers { - if p.Inbound { - in++ - } else { - out++ - } - } - // TODO: subnet-based limits - if inbound && in >= s.config.MaxInboundPeers { - return errors.New("too many inbound peers") - } else if !inbound && out >= s.config.MaxOutboundPeers { - return errors.New("too many outbound peers") - } - return nil -} - -func (s *Syncer) alreadyConnected(peer *gateway.Peer) bool { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if p.UniqueID == peer.UniqueID { - return true - } - } - return false -} - -func (s *Syncer) acceptLoop() error { - for { - conn, err := s.l.Accept() - if err != nil { - return err - } - go func() { - defer conn.Close() - if err := s.allowConnect(conn.RemoteAddr().String(), true); err != nil { - s.log.Printf("rejected inbound connection from %v: %v", conn.RemoteAddr(), err) - } else if p, err := gateway.Accept(conn, s.header); err != nil { - s.log.Printf("failed to accept inbound connection from %v: %v", conn.RemoteAddr(), err) - } else if s.alreadyConnected(p) { - s.log.Printf("rejected inbound connection from %v: already connected", conn.RemoteAddr()) - } else { - s.runPeer(p) - } - }() - } -} - -func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { - numOutbound := func() (n int) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if !p.Inbound { - n++ - } - } - return - } - - lastTried := make(map[string]time.Time) - peersForConnect := func() (peers []string) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.pm.Peers() { - // TODO: don't include port in comparison - if _, ok := s.peers[p]; !ok && time.Since(lastTried[p]) > 5*time.Minute { - peers = append(peers, p) - } - } - // TODO: weighted random selection? - frand.Shuffle(len(peers), reflect.Swapper(peers)) - return peers - } - discoverPeers := func() { - // try up to three randomly-chosen peers - var peers []*gateway.Peer - s.mu.Lock() - for _, p := range s.peers { - if peers = append(peers, p); len(peers) >= 3 { - break - } - } - s.mu.Unlock() - for _, p := range peers { - nodes, err := p.ShareNodes(s.config.ShareNodesTimeout) - if err != nil { - continue - } - for _, n := range nodes { - s.pm.AddPeer(n) - } - } - } - - ticker := time.NewTicker(s.config.PeerDiscoveryInterval) - defer ticker.Stop() - sleep := func() bool { - select { - case <-ticker.C: - return true - case <-closeChan: - return false - } - } - closing := func() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.l == nil - } - for fst := true; fst || sleep(); fst = false { - if numOutbound() >= s.config.MaxOutboundPeers { - continue - } - candidates := peersForConnect() - if len(candidates) == 0 { - discoverPeers() - continue - } - for _, p := range candidates { - if numOutbound() >= s.config.MaxOutboundPeers || closing() { - break - } - if _, err := s.Connect(p); err == nil { - s.log.Printf("formed outbound connection to %v", p) - } else { - s.log.Printf("failed to form outbound connection to %v: %v", p, err) - } - lastTried[p] = time.Now() - } - } - return nil -} - -func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { - peersForSync := func() (peers []*gateway.Peer) { - s.mu.Lock() - defer s.mu.Unlock() - for _, p := range s.peers { - if s.synced[p.Addr] { - continue - } - if peers = append(peers, p); len(peers) >= 3 { - break - } - } - return - } - - ticker := time.NewTicker(s.config.SyncInterval) - defer ticker.Stop() - sleep := func() bool { - select { - case <-ticker.C: - return true - case <-closeChan: - return false - } - } - for fst := true; fst || sleep(); fst = false { - for _, p := range peersForSync() { - history, err := s.cm.History() - if err != nil { - return err // generally fatal - } - s.mu.Lock() - s.synced[p.Addr] = true - s.mu.Unlock() - s.log.Printf("starting sync with %v", p) - oldTip := s.cm.Tip() - oldTime := time.Now() - lastPrint := time.Now() - startTime, startHeight := oldTime, oldTip.Height - addBlocks := func(blocks []types.Block) error { - if err := s.cm.AddBlocks(blocks); err != nil { - return err - } - endTime, endHeight := time.Now(), s.cm.Tip().Height - s.pm.UpdatePeerInfo(p.Addr, func(info *PeerInfo) { - info.SyncedBlocks += endHeight - startHeight - info.SyncDuration += endTime.Sub(startTime) - }) - startTime, startHeight = endTime, endHeight - if time.Since(lastPrint) > 30*time.Second { - s.log.Printf("syncing with %v, tip now %v (avg %.2f blocks/s)", p, s.cm.Tip(), float64(s.cm.Tip().Height-oldTip.Height)/endTime.Sub(oldTime).Seconds()) - lastPrint = time.Now() - } - return nil - } - if p.SupportsV2() { - history := history[:] - err = func() error { - for { - blocks, rem, err := p.SendV2Blocks(history, s.config.MaxSendBlocks, s.config.SendBlocksTimeout) - if err != nil { - return err - } else if addBlocks(blocks); err != nil { - return err - } else if rem == 0 { - return nil - } - history = []types.BlockID{blocks[len(blocks)-1].ID()} - } - }() - } else { - err = p.SendBlocks(history, s.config.SendBlocksTimeout, addBlocks) - } - totalBlocks := s.cm.Tip().Height - oldTip.Height - if err != nil { - s.log.Printf("syncing with %v failed after %v blocks: %v", p, totalBlocks, err) - } else if newTip := s.cm.Tip(); newTip != oldTip { - s.log.Printf("finished syncing %v blocks with %v, tip now %v", totalBlocks, p, newTip) - } else { - s.log.Printf("finished syncing with %v, tip unchanged", p) - } - } - } - return nil -} - -// Run spawns goroutines for accepting inbound connections, forming outbound -// connections, and syncing the blockchain from active peers. It blocks until an -// error occurs, upon which all connections are closed and goroutines are -// terminated. To gracefully shutdown a Syncer, close its net.Listener. -func (s *Syncer) Run() error { - errChan := make(chan error) - closeChan := make(chan struct{}) - go func() { errChan <- s.acceptLoop() }() - go func() { errChan <- s.peerLoop(closeChan) }() - go func() { errChan <- s.syncLoop(closeChan) }() - err := <-errChan - - // when one goroutine exits, shutdown and wait for the others - close(closeChan) - s.l.Close() - s.mu.Lock() - s.l = nil - for addr, p := range s.peers { - p.Close() - delete(s.peers, addr) - } - s.mu.Unlock() - <-errChan - <-errChan - if errors.Is(err, net.ErrClosed) { - return nil // graceful shutdown - } - return err -} - -// Connect forms an outbound connection to a peer. -func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { - if err := s.allowConnect(addr, false); err != nil { - return nil, err - } - ctx, cancel := context.WithTimeout(context.Background(), s.config.ConnectTimeout) - defer cancel() - // slightly gross polling hack so that we shutdown quickly - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(100 * time.Millisecond): - s.mu.Lock() - if s.l == nil { - cancel() - } - s.mu.Unlock() - } - } - }() - conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) - if err != nil { - return nil, err - } - conn.SetDeadline(time.Now().Add(s.config.ConnectTimeout)) - defer conn.SetDeadline(time.Time{}) - p, err := gateway.Dial(conn, s.header) - if err != nil { - conn.Close() - return nil, err - } else if s.alreadyConnected(p) { - conn.Close() - return nil, errors.New("already connected") - } - go s.runPeer(p) - - // runPeer does this too, but doing it outside the goroutine prevents a race - s.mu.Lock() - s.peers[p.Addr] = p - s.mu.Unlock() - return p, nil -} - -// BroadcastHeader broadcasts a header to all peers. -func (s *Syncer) BroadcastHeader(h gateway.BlockHeader) { s.relayHeader(h, nil) } - -// BroadcastV2Header broadcasts a v2 header to all peers. -func (s *Syncer) BroadcastV2Header(h gateway.V2BlockHeader) { s.relayV2Header(h, nil) } - -// BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. -func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } - -// BroadcastTransactionSet broadcasts a transaction set to all peers. -func (s *Syncer) BroadcastTransactionSet(txns []types.Transaction) { s.relayTransactionSet(txns, nil) } - -// BroadcastV2TransactionSet broadcasts a v2 transaction set to all peers. -func (s *Syncer) BroadcastV2TransactionSet(txns []types.V2Transaction) { - s.relayV2TransactionSet(txns, nil) -} - -// Peers returns the set of currently-connected peers. -func (s *Syncer) Peers() []*gateway.Peer { - s.mu.Lock() - defer s.mu.Unlock() - var peers []*gateway.Peer - for _, p := range s.peers { - peers = append(peers, p) - } - return peers -} - -// PeerInfo returns metadata about the specified peer. -func (s *Syncer) PeerInfo(peer string) (PeerInfo, bool) { - s.mu.Lock() - defer s.mu.Unlock() - info, ok := s.pm.PeerInfo(peer) - return info, ok -} - -// Addr returns the address of the Syncer. -func (s *Syncer) Addr() string { - return s.l.Addr().String() -} - -// New returns a new Syncer. -func New(l net.Listener, cm ChainManager, pm PeerStore, header gateway.Header, opts ...Option) *Syncer { - config := config{ - MaxInboundPeers: 8, - MaxOutboundPeers: 8, - MaxInflightRPCs: 3, - ConnectTimeout: 5 * time.Second, - ShareNodesTimeout: 5 * time.Second, - SendBlockTimeout: 60 * time.Second, - SendTransactionsTimeout: 60 * time.Second, - RelayHeaderTimeout: 5 * time.Second, - RelayBlockOutlineTimeout: 60 * time.Second, - RelayTransactionSetTimeout: 60 * time.Second, - SendBlocksTimeout: 120 * time.Second, - MaxSendBlocks: 10, - PeerDiscoveryInterval: 5 * time.Second, - SyncInterval: 5 * time.Second, - Logger: log.New(io.Discard, "", 0), - } - for _, opt := range opts { - opt(&config) - } - return &Syncer{ - l: l, - cm: cm, - pm: pm, - header: header, - config: config, - log: config.Logger, - peers: make(map[string]*gateway.Peer), - synced: make(map[string]bool), - } -} diff --git a/wallet/addresses.go b/wallet/addresses.go new file mode 100644 index 0000000..e14dae2 --- /dev/null +++ b/wallet/addresses.go @@ -0,0 +1,120 @@ +package wallet + +import ( + "time" + + "go.sia.tech/core/types" +) + +// AddressBalance returns the balance of a single address. +func (m *Manager) AddressBalance(address types.Address) (balance Balance, err error) { + return m.store.AddressBalance(address) +} + +// AddressSiacoinOutputs returns the unspent siacoin outputs for an address. +func (m *Manager) AddressSiacoinOutputs(address types.Address, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + return m.store.AddressSiacoinOutputs(address, m.chain.Tip(), offset, limit) +} + +// AddressSiafundOutputs returns the unspent siafund outputs for an address. +func (m *Manager) AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + return m.store.AddressSiafundOutputs(address, offset, limit) +} + +// AddressEvents returns the events of a single address. +func (m *Manager) AddressEvents(address types.Address, offset, limit int) (events []Event, err error) { + return m.store.AddressEvents(address, offset, limit) +} + +// AddressUnconfirmedEvents returns the unconfirmed events for a single address. +func (m *Manager) AddressUnconfirmedEvents(address types.Address) ([]Event, error) { + index := m.chain.Tip() + index.Height++ + index.ID = types.BlockID{} + timestamp := time.Now() + + v1, v2 := m.chain.PoolTransactions(), m.chain.V2PoolTransactions() + + relevantV1Txn := func(txn types.Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.UnlockConditions.UnlockHash() == address { + return true + } + } + return false + } + + relevantV1 := v1[:0] + for _, txn := range v1 { + if !relevantV1Txn(txn) { + continue + } + relevantV1 = append(relevantV1, txn) + } + + events, err := m.store.AnnotateV1Events(index, timestamp, relevantV1) + if err != nil { + return nil, err + } + + for i := range events { + events[i].Relevant = []types.Address{address} + } + + relevantV2Txn := func(txn types.V2Transaction) bool { + for _, output := range txn.SiacoinOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiacoinInputs { + if input.Parent.SiacoinOutput.Address == address { + return true + } + } + for _, output := range txn.SiafundOutputs { + if output.Address == address { + return true + } + } + for _, input := range txn.SiafundInputs { + if input.Parent.SiafundOutput.Address == address { + return true + } + } + return false + } + + // Annotate v2 transactions. + for _, txn := range v2 { + if !relevantV2Txn(txn) { + continue + } + + events = append(events, Event{ + ID: types.Hash256(txn.ID()), + Index: index, + Timestamp: timestamp, + MaturityHeight: index.Height, + Type: EventTypeV2Transaction, + Data: EventV2Transaction(txn), + Relevant: []types.Address{address}, + }) + } + return events, nil +} diff --git a/wallet/manager.go b/wallet/manager.go new file mode 100644 index 0000000..182e577 --- /dev/null +++ b/wallet/manager.go @@ -0,0 +1,731 @@ +package wallet + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "sync" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.sia.tech/walletd/internal/threadgroup" + "go.uber.org/zap" +) + +// IndexMode represents the index mode of the wallet manager. The index mode +// determines how the wallet manager stores the consensus state. +// +// IndexModePersonal - The wallet manager scans the blockchain starting at +// genesis. Only state from addresses that are registered with a +// wallet will be stored. If an address is added to a wallet after the +// scan completes, the manager will need to rescan. +// +// IndexModeFull - The wallet manager scans the blockchain starting at genesis +// and stores the state of all addresses. +// +// IndexModeNone - The wallet manager does not scan the blockchain. This is +// useful for multiple nodes sharing the same database. None should only be used +// when connecting to a database that is in "Full" mode. +const ( + IndexModePersonal IndexMode = iota + IndexModeFull + IndexModeNone +) + +const defaultSyncBatchSize = 1 + +var ( + // ErrInsufficientFunds is returned when there are not enough funds to + // fund a transaction. + ErrInsufficientFunds = errors.New("insufficient funds") + // ErrAlreadyReserved is returned when trying to reserve an output that is + // already reserved. + ErrAlreadyReserved = errors.New("output already reserved") +) + +type ( + // An IndexMode determines the chain state that the wallet manager stores. + IndexMode uint8 + + // A ChainManager manages the consensus state + ChainManager interface { + PoolTransactions() []types.Transaction + V2PoolTransactions() []types.V2Transaction + + Tip() types.ChainIndex + BestIndex(height uint64) (types.ChainIndex, bool) + + OnReorg(fn func(types.ChainIndex)) (cancel func()) + UpdatesSince(index types.ChainIndex, max int) (rus []chain.RevertUpdate, aus []chain.ApplyUpdate, err error) + } + + // A Store is a persistent store of wallet data. + Store interface { + UpdateChainState(reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error + ResetChainState() error + + WalletUnconfirmedEvents(id ID, index types.ChainIndex, timestamp time.Time, v1 []types.Transaction, v2 []types.V2Transaction) (annotated []Event, err error) + WalletEvents(walletID ID, offset, limit int) ([]Event, error) + AddWallet(Wallet) (Wallet, error) + UpdateWallet(Wallet) (Wallet, error) + DeleteWallet(walletID ID) error + WalletBalance(walletID ID) (Balance, error) + WalletAddress(ID, types.Address) (Address, error) + WalletSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + WalletSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + WalletAddresses(walletID ID) ([]Address, error) + Wallets() ([]Wallet, error) + + AddWalletAddress(walletID ID, address Address) error + RemoveWalletAddress(walletID ID, address types.Address) error + + AddressBalance(address types.Address) (balance Balance, err error) + AddressEvents(address types.Address, offset, limit int) (events []Event, err error) + AddressSiacoinOutputs(address types.Address, index types.ChainIndex, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) + AddressSiafundOutputs(address types.Address, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) + + Events(eventIDs []types.Hash256) ([]Event, error) + AnnotateV1Events(index types.ChainIndex, timestamp time.Time, v1 []types.Transaction) (annotated []Event, err error) + + SiacoinElement(types.SiacoinOutputID) (types.SiacoinElement, error) + SiafundElement(types.SiafundOutputID) (types.SiafundElement, error) + // SiacoinElementSpentEvent returns the event of a spent siacoin element. + // If the element is not spent, the return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiacoinElementSpentEvent(types.SiacoinOutputID) (Event, bool, error) + // SiafundElementSpentEvent returns the event of a spent siafund element. + // If the element is not spent, the second return value will be (Event{}, false, nil). + // If the element is not found, the error will be ErrNotFound. An element + // is only tracked for 144 blocks after it is spent. + SiafundElementSpentEvent(types.SiafundOutputID) (Event, bool, error) + + SetIndexMode(IndexMode) error + LastCommittedIndex() (types.ChainIndex, error) + } + + // A Manager manages wallets. + Manager struct { + indexMode IndexMode + syncBatchSize int + lockDuration time.Duration + + chain ChainManager + store Store + log *zap.Logger + tg *threadgroup.ThreadGroup + + mu sync.Mutex // protects the fields below + used map[types.Hash256]time.Time + } +) + +// String returns the string representation of the index mode. +func (i IndexMode) String() string { + switch i { + case IndexModePersonal: + return "personal" + case IndexModeFull: + return "full" + case IndexModeNone: + return "none" + default: + return "unknown" + } +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +func (i *IndexMode) UnmarshalText(buf []byte) error { + switch string(buf) { + case "personal": + *i = IndexModePersonal + case "full": + *i = IndexModeFull + case "none": + *i = IndexModeNone + default: + return fmt.Errorf("unknown index mode %q", buf) + } + return nil +} + +// MarshalText implements the encoding.TextMarshaler interface. +func (i IndexMode) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// lockUTXOs locks the given UTXOs for the duration of the lock duration. +// The lock duration is used to prevent double spending when building transactions. +// It is expected that the caller holds the manager's lock. +func (m *Manager) lockUTXOs(ids ...types.Hash256) { + ts := time.Now().Add(m.lockDuration) + for _, id := range ids { + m.used[id] = ts + } +} + +// utxosLocked returns an error if any of the given UTXOs are locked. +// It is expected that the caller holds the manager's lock. +func (m *Manager) utxosLocked(ids ...types.Hash256) error { + for _, id := range ids { + if m.used[id].After(time.Now()) { + return fmt.Errorf("failed to lock output %q: %w", id, ErrAlreadyReserved) + } + } + return nil +} + +// Tip returns the last scanned chain index of the manager. +func (m *Manager) Tip() (types.ChainIndex, error) { + return m.store.LastCommittedIndex() +} + +// AddWallet adds the given wallet. +func (m *Manager) AddWallet(w Wallet) (Wallet, error) { + return m.store.AddWallet(w) +} + +// UpdateWallet updates the given wallet. +func (m *Manager) UpdateWallet(w Wallet) (Wallet, error) { + return m.store.UpdateWallet(w) +} + +// DeleteWallet deletes the given wallet. +func (m *Manager) DeleteWallet(walletID ID) error { + return m.store.DeleteWallet(walletID) +} + +// Wallets returns the wallets of the wallet manager. +func (m *Manager) Wallets() ([]Wallet, error) { + return m.store.Wallets() +} + +// AddAddress adds the given address to the given wallet. +func (m *Manager) AddAddress(walletID ID, addr Address) error { + return m.store.AddWalletAddress(walletID, addr) +} + +// RemoveAddress removes the given address from the given wallet. +func (m *Manager) RemoveAddress(walletID ID, addr types.Address) error { + return m.store.RemoveWalletAddress(walletID, addr) +} + +// Addresses returns the addresses of the given wallet. +func (m *Manager) Addresses(walletID ID) ([]Address, error) { + return m.store.WalletAddresses(walletID) +} + +// WalletEvents returns the events of the given wallet. +func (m *Manager) WalletEvents(walletID ID, offset, limit int) ([]Event, error) { + return m.store.WalletEvents(walletID, offset, limit) +} + +// UnspentSiacoinOutputs returns a paginated list of matured siacoin outputs +// relevant to the wallet +func (m *Manager) UnspentSiacoinOutputs(walletID ID, offset, limit int) ([]types.SiacoinElement, types.ChainIndex, error) { + return m.store.WalletSiacoinOutputs(walletID, offset, limit) +} + +// UnspentSiafundOutputs returns a paginated list of siafund outputs relevant to +// the wallet +func (m *Manager) UnspentSiafundOutputs(walletID ID, offset, limit int) ([]types.SiafundElement, types.ChainIndex, error) { + return m.store.WalletSiafundOutputs(walletID, offset, limit) +} + +// WalletUnconfirmedEvents returns the unconfirmed events of the given wallet. +func (m *Manager) WalletUnconfirmedEvents(walletID ID) ([]Event, error) { + index := m.chain.Tip() + index.Height++ + index.ID = types.BlockID{} + return m.store.WalletUnconfirmedEvents(walletID, index, time.Now(), m.chain.PoolTransactions(), m.chain.V2PoolTransactions()) +} + +// WalletBalance returns the balance of the given wallet. +func (m *Manager) WalletBalance(walletID ID) (Balance, error) { + return m.store.WalletBalance(walletID) +} + +// Events returns the events with the given IDs. +func (m *Manager) Events(eventIDs []types.Hash256) ([]Event, error) { + return m.store.Events(eventIDs) +} + +// UnconfirmedEvents returns all unconfirmed events in the transaction pool. +func (m *Manager) UnconfirmedEvents() ([]Event, error) { + v1, v2 := m.chain.PoolTransactions(), m.chain.V2PoolTransactions() + + unconfirmedIndex := m.chain.Tip() + unconfirmedIndex.Height++ + unconfirmedIndex.ID = types.BlockID{} + timestamp := time.Now() + + events, err := m.store.AnnotateV1Events(unconfirmedIndex, timestamp, v1) + if err != nil { + return nil, err + } + + for _, txn := range v2 { + events = append(events, Event{ + ID: types.Hash256(txn.ID()), + Index: unconfirmedIndex, + Timestamp: timestamp, + Type: EventTypeV2Transaction, + Data: EventV2Transaction(txn), + }) + } + return events, nil +} + +// Reserve reserves the given ids for the given duration. +func (m *Manager) Reserve(ids []types.Hash256) error { + m.mu.Lock() + defer m.mu.Unlock() + + // check if any of the ids are already reserved + if err := m.utxosLocked(ids...); err != nil { + return err + } + m.lockUTXOs(ids...) + return nil +} + +// Release releases the given ids. +func (m *Manager) Release(ids []types.Hash256) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, id := range ids { + delete(m.used, id) + } +} + +// WalletAddress returns an address from the wallet. +func (m *Manager) WalletAddress(id ID, addr types.Address) (Address, error) { + return m.store.WalletAddress(id, addr) +} + +// SelectSiacoinElements selects siacoin elements from the wallet that sum to +// at least the given amount. Returns the elements, the element basis, and the +// change amount. +func (m *Manager) SelectSiacoinElements(walletID ID, amount types.Currency, useUnconfirmed bool) ([]types.SiacoinElement, types.ChainIndex, types.Currency, error) { + // sanity check that the wallet exists + if _, err := m.WalletBalance(walletID); err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + knownAddresses := make(map[types.Address]bool) + relevantAddr := func(addr types.Address) (bool, error) { + if exists, ok := knownAddresses[addr]; ok { + return exists, nil + } + _, err := m.store.WalletAddress(walletID, addr) + if errors.Is(err, ErrNotFound) { + knownAddresses[addr] = false + return false, nil + } else if err != nil { + return false, err + } + knownAddresses[addr] = true + return true, nil + } + + ephemeral := make(map[types.SiacoinOutputID]types.SiacoinElement) + inPool := make(map[types.SiacoinOutputID]bool) + for _, txn := range m.chain.PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + inPool[sci.ParentID] = true + delete(ephemeral, sci.ParentID) + } + for i, sco := range txn.SiacoinOutputs { + exists, err := relevantAddr(sco.Address) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) + } else if exists { + scoid := txn.SiacoinOutputID(i) + ephemeral[scoid] = types.SiacoinElement{ + ID: scoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiacoinOutput: sco, + } + } + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, sci := range txn.SiacoinInputs { + inPool[sci.Parent.ID] = true + delete(ephemeral, sci.Parent.ID) + } + for i, sco := range txn.SiacoinOutputs { + exists, err := relevantAddr(sco.Address) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to check if address %q is relevant: %w", sco.Address, err) + } else if exists { + sce := txn.EphemeralSiacoinOutput(i) + ephemeral[sce.ID] = sce + } + } + } + + var inputSum types.Currency + var selected []types.SiacoinElement + var utxoIDs []types.Hash256 + var basis types.ChainIndex + const utxoBatchSize = 100 +top: + for i := 0; ; i += utxoBatchSize { + var utxos []types.SiacoinElement + var err error + // extra large wallets may need to paginate through utxos + // to find enough to cover the amount + utxos, basis, err = m.store.WalletSiacoinOutputs(walletID, i, utxoBatchSize) + if err != nil { + return nil, types.ChainIndex{}, types.ZeroCurrency, fmt.Errorf("failed to get siacoin elements: %w", err) + } else if len(utxos) == 0 { + break top + } + + for _, sce := range utxos { + if inPool[sce.ID] || m.utxosLocked(types.Hash256(sce.ID)) != nil { + continue + } + + selected = append(selected, sce) + utxoIDs = append(utxoIDs, types.Hash256(sce.ID)) + inputSum = inputSum.Add(sce.SiacoinOutput.Value) + if inputSum.Cmp(amount) >= 0 { + break top + } + } + } + + if inputSum.Cmp(amount) < 0 { + if !useUnconfirmed { + return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + } + + for _, sce := range ephemeral { + if inPool[sce.ID] || m.utxosLocked(types.Hash256(sce.ID)) != nil { + continue + } + + selected = append(selected, sce) + inputSum = inputSum.Add(sce.SiacoinOutput.Value) + if inputSum.Cmp(amount) >= 0 { + break + } + } + } + + if inputSum.Cmp(amount) < 0 { + return nil, types.ChainIndex{}, types.ZeroCurrency, ErrInsufficientFunds + } + m.lockUTXOs(utxoIDs...) + return selected, basis, inputSum.Sub(amount), nil +} + +// SelectSiafundElements selects siafund elements from the wallet that sum to +// at least the given amount. Returns the elements, the element basis, and the +// change amount. +func (m *Manager) SelectSiafundElements(walletID ID, amount uint64) ([]types.SiafundElement, types.ChainIndex, uint64, error) { + // sanity check that the wallet exists + if _, err := m.WalletBalance(walletID); err != nil { + return nil, types.ChainIndex{}, 0, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + if amount == 0 { + return nil, m.chain.Tip(), 0, nil + } + + knownAddresses := make(map[types.Address]bool) + relevantAddr := func(addr types.Address) (bool, error) { + if exists, ok := knownAddresses[addr]; ok { + return exists, nil + } + _, err := m.store.WalletAddress(walletID, addr) + if errors.Is(err, ErrNotFound) { + knownAddresses[addr] = false + return false, nil + } else if err != nil { + return false, err + } + knownAddresses[addr] = true + return true, nil + } + + ephemeral := make(map[types.SiafundOutputID]types.SiafundElement) + inPool := make(map[types.SiafundOutputID]bool) + for _, txn := range m.chain.PoolTransactions() { + for _, sfi := range txn.SiafundInputs { + inPool[sfi.ParentID] = true + delete(ephemeral, sfi.ParentID) + } + for i, sfo := range txn.SiafundOutputs { + exists, err := relevantAddr(sfo.Address) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) + } else if exists { + sfoid := txn.SiafundOutputID(i) + ephemeral[sfoid] = types.SiafundElement{ + ID: sfoid, + StateElement: types.StateElement{LeafIndex: types.UnassignedLeafIndex}, + SiafundOutput: sfo, + } + } + } + } + for _, txn := range m.chain.V2PoolTransactions() { + for _, sfi := range txn.SiafundInputs { + inPool[sfi.Parent.ID] = true + delete(ephemeral, sfi.Parent.ID) + } + for i, sfo := range txn.SiafundOutputs { + exists, err := relevantAddr(sfo.Address) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to check if address %q is relevant: %w", sfo.Address, err) + } else if exists { + sfe := txn.EphemeralSiafundOutput(i) + ephemeral[sfe.ID] = sfe + } + } + } + + var inputSum uint64 + var selected []types.SiafundElement + var utxoIDs []types.Hash256 + var basis types.ChainIndex + const utxoBatchSize = 100 +top: + for i := 0; ; i += utxoBatchSize { + var utxos []types.SiafundElement + var err error + + utxos, basis, err = m.store.WalletSiafundOutputs(walletID, i, utxoBatchSize) + if err != nil { + return nil, types.ChainIndex{}, 0, fmt.Errorf("failed to get siafund elements: %w", err) + } else if len(utxos) == 0 { + break top + } + + for _, sfe := range utxos { + if inPool[sfe.ID] || m.utxosLocked(types.Hash256(sfe.ID)) != nil { + continue + } + + selected = append(selected, sfe) + utxoIDs = append(utxoIDs, types.Hash256(sfe.ID)) + inputSum += sfe.SiafundOutput.Value + if inputSum >= amount { + break top + } + } + } + + if inputSum < amount { + return nil, types.ChainIndex{}, 0, ErrInsufficientFunds + } + + m.lockUTXOs(utxoIDs...) + return selected, basis, inputSum - amount, nil +} + +// Scan rescans the chain starting from the given index. The scan will complete +// when the chain manager reaches the current tip or the context is canceled. +func (m *Manager) Scan(ctx context.Context, index types.ChainIndex) error { + if m.indexMode != IndexModePersonal { + return fmt.Errorf("scans are disabled in index mode %s", m.indexMode) + } + + ctx, cancel, err := m.tg.AddWithContext(ctx) + if err != nil { + return err + } + defer cancel() + + m.mu.Lock() + defer m.mu.Unlock() + return syncStore(ctx, m.store, m.chain, index, m.syncBatchSize) +} + +// IndexMode returns the index mode of the wallet manager. +func (m *Manager) IndexMode() IndexMode { + return m.indexMode +} + +// SiacoinElement returns the unspent siacoin element with the given id. +func (m *Manager) SiacoinElement(id types.SiacoinOutputID) (types.SiacoinElement, error) { + return m.store.SiacoinElement(id) +} + +// SiafundElement returns the unspent siafund element with the given id. +func (m *Manager) SiafundElement(id types.SiafundOutputID) (types.SiafundElement, error) { + return m.store.SiafundElement(id) +} + +// SiacoinElementSpentEvent returns the event of a spent siacoin element. +// If the element is not spent, the return value will be (Event{}, false, nil). +// If the element is not found, the error will be ErrNotFound. An element +// is only tracked for 144 blocks after it is spent. +func (m *Manager) SiacoinElementSpentEvent(id types.SiacoinOutputID) (Event, bool, error) { + return m.store.SiacoinElementSpentEvent(id) +} + +// SiafundElementSpentEvent returns the event of a spent siafund element. +// If the element is not spent, the second return value will be (Event{}, false, nil). +// If the element is not found, the error will be ErrNotFound. An element +// is only tracked for 144 blocks after it is spent. +func (m *Manager) SiafundElementSpentEvent(id types.SiafundOutputID) (Event, bool, error) { + return m.store.SiafundElementSpentEvent(id) +} + +// Close closes the wallet manager. +func (m *Manager) Close() error { + m.tg.Stop() + return nil +} + +func syncStore(ctx context.Context, store Store, cm ChainManager, index types.ChainIndex, batchSize int) error { + for index != cm.Tip() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + crus, caus, err := cm.UpdatesSince(index, batchSize) + if err != nil { + return fmt.Errorf("failed to subscribe to chain manager: %w", err) + } else if err := store.UpdateChainState(crus, caus); err != nil { + return fmt.Errorf("failed to update chain state: %w", err) + } + + switch { + case len(caus) > 0: + index = caus[len(caus)-1].State.Index + case len(crus) > 0: + index = crus[len(crus)-1].State.Index + } + } + return nil +} + +// NewManager creates a new wallet manager. +func NewManager(cm ChainManager, store Store, opts ...Option) (*Manager, error) { + m := &Manager{ + indexMode: IndexModePersonal, + syncBatchSize: defaultSyncBatchSize, + lockDuration: time.Hour, + + chain: cm, + store: store, + log: zap.NewNop(), + tg: threadgroup.New(), + + used: make(map[types.Hash256]time.Time), + } + + for _, opt := range opts { + opt(m) + } + + // if the index mode is none, skip setting the index mode in the store + // and return the manager + if m.indexMode == IndexModeNone { + return m, nil + } else if err := store.SetIndexMode(m.indexMode); err != nil { + return nil, err + } + + // start a goroutine to sync the store with the chain manager + reorgChan := make(chan struct{}, 1) + reorgChan <- struct{}{} + unsubscribe := cm.OnReorg(func(index types.ChainIndex) { + select { + case reorgChan <- struct{}{}: + default: + } + }) + + go func() { + ctx, cancel, err := m.tg.AddWithContext(context.Background()) + if errors.Is(err, threadgroup.ErrClosed) { + return + } else if err != nil { + log.Panic("failed to add to threadgroup", zap.Error(err)) + } + defer cancel() + + t := time.NewTicker(m.lockDuration / 2) + defer t.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + m.mu.Lock() + for id, ts := range m.used { + if ts.Before(time.Now()) { + delete(m.used, id) + } + } + m.mu.Unlock() + } + } + }() + + go func() { + defer unsubscribe() + + log := m.log.Named("sync") + ctx, cancel, err := m.tg.AddWithContext(context.Background()) + if err != nil { + log.Panic("failed to add to threadgroup", zap.Error(err)) + } + defer cancel() + + for { + select { + case <-ctx.Done(): + return + case <-reorgChan: + } + + m.mu.Lock() + // update the store + lastTip, err := store.LastCommittedIndex() + if err != nil { + log.Panic("failed to get last committed index", zap.Error(err)) + } + err = syncStore(ctx, store, cm, lastTip, m.syncBatchSize) + if err != nil { + switch { + case errors.Is(err, context.Canceled): + m.mu.Unlock() + return + case strings.Contains(err.Error(), "missing block at index"): // unfortunate, but not exposed by coreutils + log.Warn("missing block at index, resetting chain state", zap.Stringer("id", lastTip.ID), zap.Uint64("height", lastTip.Height)) + if err := store.ResetChainState(); err != nil { + log.Panic("failed to reset wallet state", zap.Error(err)) + } + // trigger resync + select { + case reorgChan <- struct{}{}: + default: + } + default: + panic("failed to sync store: " + err.Error()) + } + } + m.mu.Unlock() + } + }() + return m, nil +} diff --git a/wallet/options.go b/wallet/options.go new file mode 100644 index 0000000..34e19af --- /dev/null +++ b/wallet/options.go @@ -0,0 +1,41 @@ +package wallet + +import ( + "time" + + "go.uber.org/zap" +) + +// An Option configures a wallet Manager. +type Option func(*Manager) + +// WithLogger sets the logger used by the manager. +func WithLogger(log *zap.Logger) Option { + return func(m *Manager) { + m.log = log + } +} + +// WithIndexMode sets the index mode used by the manager. +func WithIndexMode(mode IndexMode) Option { + return func(m *Manager) { + m.indexMode = mode + } +} + +// WithSyncBatchSize sets the number of blocks to batch when scanning +// the blockchain. The default is 64. Increasing this value can +// improve performance at the cost of memory usage. +func WithSyncBatchSize(size int) Option { + return func(m *Manager) { + m.syncBatchSize = size + } +} + +// WithLockDuration sets the duration that a UTXO is locked after +// being selected as an input to a transaction. The default is 1 hour. +func WithLockDuration(d time.Duration) Option { + return func(m *Manager) { + m.lockDuration = d + } +} diff --git a/wallet/seed.go b/wallet/seed.go index 4a9046a..ecc4a65 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -8,7 +8,7 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" - "go.sia.tech/core/wallet" + "go.sia.tech/coreutils/wallet" "lukechampine.com/frand" ) @@ -70,13 +70,19 @@ func (sav *SeedAddressVault) OwnsAddress(addr types.Address) bool { // NewAddress returns a new address derived from the seed, along with // descriptive metadata. -func (sav *SeedAddressVault) NewAddress(desc string) (types.Address, json.RawMessage) { +func (sav *SeedAddressVault) NewAddress(desc string) Address { sav.mu.Lock() defer sav.mu.Unlock() index := uint64(len(sav.addrs)) - sav.lookahead + 1 sav.gen(index + sav.lookahead) - addr := types.StandardAddress(sav.seed.PublicKey(index)) - return addr, json.RawMessage(fmt.Sprintf(`{"desc":"%s","keyIndex":%d}`, desc, index)) + policy := types.PolicyPublicKey(sav.seed.PublicKey(index)) + addr := policy.Address() + return Address{ + Address: addr, + Description: desc, + SpendPolicy: &policy, + Metadata: json.RawMessage(fmt.Sprintf(`{"keyIndex":%d}`, index)), + } } // SignTransaction signs the specified transaction using keys derived from the diff --git a/wallet/update.go b/wallet/update.go new file mode 100644 index 0000000..da9544d --- /dev/null +++ b/wallet/update.go @@ -0,0 +1,281 @@ +package wallet + +import ( + "fmt" + + "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" + "go.uber.org/zap" +) + +type ( + // A stateTreeUpdater is an interface for applying and reverting + // Merkle tree updates. + stateTreeUpdater interface { + UpdateElementProof(*types.StateElement) + ForEachTreeNode(fn func(row uint64, col uint64, h types.Hash256)) + } + + // A ProofUpdater is an interface for updating Merkle proofs. + ProofUpdater interface { + UpdateElementProof(*types.StateElement) + } + + // AddressBalance pairs an address with its balance. + AddressBalance struct { + Address types.Address `json:"address"` + Balance + } + + // SpentSiacoinElement pairs a spent siacoin element with the ID of the + // transaction that spent it. + SpentSiacoinElement struct { + types.SiacoinElement + EventID types.TransactionID + } + + // SpentSiafundElement pairs a spent siafund element with the ID of the + // transaction that spent it. + SpentSiafundElement struct { + types.SiafundElement + EventID types.TransactionID + } + + // AppliedState contains all state changes made to a store after applying a chain + // update. + AppliedState struct { + NumLeaves uint64 + Events []Event + CreatedSiacoinElements []types.SiacoinElement + SpentSiacoinElements []SpentSiacoinElement + CreatedSiafundElements []types.SiafundElement + SpentSiafundElements []SpentSiafundElement + } + + // RevertedState contains all state changes made to a store after reverting + // a chain update. + RevertedState struct { + NumLeaves uint64 + UnspentSiacoinElements []types.SiacoinElement + DeletedSiacoinElements []types.SiacoinElement + UnspentSiafundElements []types.SiafundElement + DeletedSiafundElements []types.SiafundElement + } + + // A TreeNodeUpdate contains the hash of a Merkle tree node and its row and + // column indices. + TreeNodeUpdate struct { + Hash types.Hash256 + Row int + Column int + } + + // An UpdateTx atomically updates the state of a store. + UpdateTx interface { + UpdateStateElementProofs(ProofUpdater) error + UpdateStateTree([]TreeNodeUpdate) error + + AddressRelevant(types.Address) (bool, error) + + ApplyIndex(types.ChainIndex, AppliedState) error + RevertIndex(types.ChainIndex, RevertedState) error + } +) + +// updateStateElements updates the state elements in a store according to the +// changes made by a chain update. +func updateStateElements(tx UpdateTx, update stateTreeUpdater, indexMode IndexMode) error { + if indexMode == IndexModeNone { + panic("updateStateElements called with IndexModeNone") // developer error + } + + if indexMode == IndexModeFull { + var updates []TreeNodeUpdate + update.ForEachTreeNode(func(row, col uint64, h types.Hash256) { + updates = append(updates, TreeNodeUpdate{h, int(row), int(col)}) + }) + return tx.UpdateStateTree(updates) + } else { + return tx.UpdateStateElementProofs(update) + } +} + +// applyChainUpdate atomically applies a chain update to a store +func applyChainUpdate(tx UpdateTx, cau chain.ApplyUpdate, indexMode IndexMode) error { + applied := AppliedState{ + NumLeaves: cau.State.Elements.NumLeaves, + } + + spentEventIDs := make(map[types.Hash256]types.TransactionID) + for _, txn := range cau.Block.Transactions { + txnID := txn.ID() + for _, input := range txn.SiacoinInputs { + spentEventIDs[types.Hash256(input.ParentID)] = txnID + } + for _, input := range txn.SiafundInputs { + spentEventIDs[types.Hash256(input.ParentID)] = txnID + } + } + for _, txn := range cau.Block.V2Transactions() { + txnID := txn.ID() + for _, input := range txn.SiacoinInputs { + spentEventIDs[types.Hash256(input.Parent.ID)] = txnID + } + for _, input := range txn.SiafundInputs { + spentEventIDs[types.Hash256(input.Parent.ID)] = txnID + } + } + + // add new siacoin elements to the store + for _, sced := range cau.SiacoinElementDiffs() { + sce := sced.SiacoinElement + if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { + continue + } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { + panic(err) + } else if !relevant { + continue + } + if sced.Spent { + spentTxnID, ok := spentEventIDs[types.Hash256(sce.ID)] + if !ok { + panic(fmt.Errorf("missing transaction ID for spent siacoin element %v", sce.ID)) + } + applied.SpentSiacoinElements = append(applied.SpentSiacoinElements, SpentSiacoinElement{ + SiacoinElement: sce, + EventID: spentTxnID, + }) + } else { + applied.CreatedSiacoinElements = append(applied.CreatedSiacoinElements, sce) + } + } + for _, sfed := range cau.SiafundElementDiffs() { + sfe := sfed.SiafundElement + if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { + continue + } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { + panic(err) + } else if !relevant { + continue + } + if sfed.Spent { + spentTxnID, ok := spentEventIDs[types.Hash256(sfe.ID)] + if !ok { + panic(fmt.Errorf("missing transaction ID for spent siafund element %v", sfe.ID)) + } + applied.SpentSiafundElements = append(applied.SpentSiafundElements, SpentSiafundElement{ + SiafundElement: sfe, + EventID: spentTxnID, + }) + } else { + applied.CreatedSiafundElements = append(applied.CreatedSiafundElements, sfe) + } + } + + // add events + relevant := func(addr types.Address) bool { + relevant, err := tx.AddressRelevant(addr) + if err != nil { + panic(fmt.Errorf("failed to check if address is relevant: %w", err)) + } + return relevant + } + applied.Events = AppliedEvents(cau.State, cau.Block, cau, relevant) + + if err := updateStateElements(tx, cau, indexMode); err != nil { + return fmt.Errorf("failed to update state elements: %w", err) + } else if err := tx.ApplyIndex(cau.State.Index, applied); err != nil { + return fmt.Errorf("failed to apply index: %w", err) + } + return nil +} + +// revertChainUpdate atomically reverts a chain update from a store +func revertChainUpdate(tx UpdateTx, cru chain.RevertUpdate, revertedIndex types.ChainIndex, indexMode IndexMode) error { + reverted := RevertedState{ + NumLeaves: cru.State.Elements.NumLeaves, + } + + // determine which siacoin and siafund elements are ephemeral + // + // note: I thought we could use LeafIndex == EphemeralLeafIndex, but + // it seems to be set before the subscriber is called. + created := make(map[types.Hash256]bool) + ephemeral := make(map[types.Hash256]bool) + for _, txn := range cru.Block.Transactions { + for i := range txn.SiacoinOutputs { + created[types.Hash256(txn.SiacoinOutputID(i))] = true + } + for _, input := range txn.SiacoinInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + for i := range txn.SiafundOutputs { + created[types.Hash256(txn.SiafundOutputID(i))] = true + } + for _, input := range txn.SiafundInputs { + ephemeral[types.Hash256(input.ParentID)] = created[types.Hash256(input.ParentID)] + } + } + + for _, sced := range cru.SiacoinElementDiffs() { + sce := sced.SiacoinElement + if (sced.Created && sced.Spent) || sce.SiacoinOutput.Value.IsZero() { + continue + } else if relevant, err := tx.AddressRelevant(sce.SiacoinOutput.Address); err != nil { + panic(err) + } else if !relevant { + continue + } + if sced.Spent { + // re-add any spent siacoin elements + reverted.UnspentSiacoinElements = append(reverted.UnspentSiacoinElements, sce) + } else { + // delete any created siacoin elements + reverted.DeletedSiacoinElements = append(reverted.DeletedSiacoinElements, sce) + } + } + for _, sfed := range cru.SiafundElementDiffs() { + sfe := sfed.SiafundElement + if (sfed.Created && sfed.Spent) || sfe.SiafundOutput.Value == 0 { + continue + } else if relevant, err := tx.AddressRelevant(sfe.SiafundOutput.Address); err != nil { + panic(err) + } else if !relevant { + continue + } + if sfed.Spent { + reverted.UnspentSiafundElements = append(reverted.UnspentSiafundElements, sfe) + } else { + reverted.DeletedSiafundElements = append(reverted.DeletedSiafundElements, sfe) + } + } + + if err := tx.RevertIndex(revertedIndex, reverted); err != nil { + return fmt.Errorf("failed to revert index: %w", err) + } + return updateStateElements(tx, cru, indexMode) +} + +// UpdateChainState atomically updates the state of a store with a set of +// updates from the chain manager. +func UpdateChainState(tx UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate, indexMode IndexMode, log *zap.Logger) error { + for _, cru := range reverted { + revertedIndex := types.ChainIndex{ + ID: cru.Block.ID(), + Height: cru.State.Index.Height + 1, + } + if err := revertChainUpdate(tx, cru, revertedIndex, indexMode); err != nil { + return fmt.Errorf("failed to revert chain update %q: %w", revertedIndex, err) + } + log.Debug("reverted chain update", zap.Stringer("blockID", revertedIndex.ID), zap.Uint64("height", revertedIndex.Height)) + } + + for _, cau := range applied { + // apply the chain update + if err := applyChainUpdate(tx, cau, indexMode); err != nil { + return fmt.Errorf("failed to apply chain update %q: %w", cau.State.Index, err) + } + log.Debug("applied chain update", zap.Stringer("blockID", cau.State.Index.ID), zap.Uint64("height", cau.State.Index.Height)) + } + return nil +} diff --git a/wallet/wallet.go b/wallet/wallet.go index 1d788b5..0174db4 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -2,13 +2,122 @@ package wallet import ( "encoding/json" - "fmt" + "errors" + "strconv" "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/wallet" ) +// event types indicate the source of an event. Events can +// either be created by sending Siacoins between addresses or they can be +// created by consensus (e.g. a miner payout, a siafund claim, or a contract). +const ( + EventTypeMinerPayout = wallet.EventTypeMinerPayout + EventTypeFoundationSubsidy = wallet.EventTypeFoundationSubsidy + EventTypeSiafundClaim = wallet.EventTypeSiafundClaim + + EventTypeV1Transaction = wallet.EventTypeV1Transaction + EventTypeV1ContractResolution = wallet.EventTypeV1ContractResolution + + EventTypeV2Transaction = wallet.EventTypeV2Transaction + EventTypeV2ContractResolution = wallet.EventTypeV2ContractResolution +) + +type ( + // An EventPayout represents a miner payout, siafund claim, or foundation + // subsidy. + EventPayout = wallet.EventPayout + // An EventV1Transaction pairs a v1 transaction with its spent siacoin and + // siafund elements. + EventV1Transaction = wallet.EventV1Transaction + // An EventV1ContractResolution represents a file contract payout from a v1 + // contract. + EventV1ContractResolution = wallet.EventV1ContractResolution + // EventV2Transaction is a transaction event that includes the transaction + EventV2Transaction = wallet.EventV2Transaction + // An EventV2ContractResolution represents a file contract payout from a v2 + // contract. + EventV2ContractResolution = wallet.EventV2ContractResolution + + // EventData is the data associated with an event. + EventData = wallet.EventData + // An Event is a record of a consensus event that affects the wallet. + Event = wallet.Event +) + +type ( + // Balance is a summary of a siacoin and siafund balance + Balance struct { + Siacoins types.Currency `json:"siacoins"` + ImmatureSiacoins types.Currency `json:"immatureSiacoins"` + Siafunds uint64 `json:"siafunds"` + } + + // An ID is a unique identifier for a wallet. + ID int64 + + // A Wallet is a collection of addresses and metadata. + Wallet struct { + ID ID `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DateCreated time.Time `json:"dateCreated"` + LastUpdated time.Time `json:"lastUpdated"` + Metadata json.RawMessage `json:"metadata"` + } + + // A Address is an address associated with a wallet. + Address struct { + Address types.Address `json:"address"` + Description string `json:"description"` + SpendPolicy *types.SpendPolicy `json:"spendPolicy,omitempty"` + Metadata json.RawMessage `json:"metadata"` + } + + // An UnspentSiacoinElement is an unspent siacoin output paired + // with the number of confirmations. + UnspentSiacoinElement struct { + types.SiacoinElement + Confirmations uint64 `json:"confirmations"` + } + + // An UnspentSiafundElement is an unspent siafund output paired + // with the number of confirmations. + UnspentSiafundElement struct { + types.SiafundElement + Confirmations uint64 `json:"confirmations"` + } + + // A ChainUpdate is a set of changes to the consensus state. + ChainUpdate interface { + SiacoinElementDiffs() []consensus.SiacoinElementDiff + SiafundElementDiffs() []consensus.SiafundElementDiff + FileContractElementDiffs() []consensus.FileContractElementDiff + V2FileContractElementDiffs() []consensus.V2FileContractElementDiff + } +) + +// ErrNotFound is returned when a requested wallet or address is not found. +var ErrNotFound = errors.New("not found") + +// UnmarshalText implements encoding.TextUnmarshaler. +func (w *ID) UnmarshalText(buf []byte) error { + id, err := strconv.ParseInt(string(buf), 10, 64) + if err != nil { + return err + } + *w = ID(id) + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (w ID) MarshalText() ([]byte, error) { + return []byte(strconv.FormatInt(int64(w), 10)), nil +} + // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. @@ -34,307 +143,42 @@ func SignTransaction(cs consensus.State, txn *types.Transaction, sigIndex int, k tsig.Signature = sig[:] } -// A PoolTransaction summarizes the wallet-relevant data in a txpool -// transaction. -type PoolTransaction struct { - ID types.TransactionID `json:"id"` - Raw types.Transaction `json:"raw"` - Type string `json:"type"` - Sent types.Currency `json:"sent"` - Received types.Currency `json:"received"` - Locked types.Currency `json:"locked"` -} - -// Annotate annotates a txpool transaction. -func Annotate(txn types.Transaction, ownsAddress func(types.Address) bool) PoolTransaction { - ptxn := PoolTransaction{ID: txn.ID(), Raw: txn, Type: "unknown"} - - var totalValue types.Currency - for _, sco := range txn.SiacoinOutputs { - totalValue = totalValue.Add(sco.Value) - } - for _, fc := range txn.FileContracts { - totalValue = totalValue.Add(fc.Payout) - } - for _, fee := range txn.MinerFees { - totalValue = totalValue.Add(fee) - } - - var ownedIn, ownedOut int - for _, sci := range txn.SiacoinInputs { - if ownsAddress(sci.UnlockConditions.UnlockHash()) { - ownedIn++ - } - } - for _, sco := range txn.SiacoinOutputs { - if ownsAddress(sco.Address) { - ownedOut++ - } - } - var ins, outs string - switch { - case ownedIn == 0: - ins = "none" - case ownedIn < len(txn.SiacoinInputs): - ins = "some" - case ownedIn == len(txn.SiacoinInputs): - ins = "all" - } - switch { - case ownedOut == 0: - outs = "none" - case ownedOut < len(txn.SiacoinOutputs): - outs = "some" - case ownedOut == len(txn.SiacoinOutputs): - outs = "all" - } - - switch { - case ins == "none" && outs == "none": - ptxn.Type = "unrelated" - case ins == "all": - ptxn.Sent = totalValue - switch { - case outs == "all": - ptxn.Type = "redistribution" - case len(txn.FileContractRevisions) > 0: - ptxn.Type = "contract revision" - case len(txn.StorageProofs) > 0: - ptxn.Type = "storage proof" - case len(txn.ArbitraryData) > 0: - ptxn.Type = "announcement" - default: - ptxn.Type = "send" - } - case ins == "none" && outs != "none": - ptxn.Type = "receive" - for _, sco := range txn.SiacoinOutputs { - if ownsAddress(sco.Address) { - ptxn.Received = ptxn.Received.Add(sco.Value) - } - } - case ins == "some" && len(txn.FileContracts) > 0: - ptxn.Type = "contract" - for _, fc := range txn.FileContracts { - var validLocked, missedLocked types.Currency - for _, sco := range fc.ValidProofOutputs { - if ownsAddress(sco.Address) { - validLocked = validLocked.Add(fc.Payout) - } - } - for _, sco := range fc.MissedProofOutputs { - if ownsAddress(sco.Address) { - missedLocked = missedLocked.Add(fc.Payout) - } - } - if validLocked.Cmp(missedLocked) > 0 { - ptxn.Locked = ptxn.Locked.Add(validLocked) - } else { - ptxn.Locked = ptxn.Locked.Add(missedLocked) - } - } - } - - return ptxn -} - -// An Event is something interesting that happened on the Sia blockchain. -type Event struct { - Index types.ChainIndex - Timestamp time.Time - Relevant []types.Address - Val interface{ eventType() string } -} - -func (*EventTransaction) eventType() string { return "transaction" } -func (*EventMinerPayout) eventType() string { return "miner payout" } -func (*EventMissedFileContract) eventType() string { return "missed file contract" } - -// MarshalJSON implements json.Marshaler. -func (e Event) MarshalJSON() ([]byte, error) { - val, _ := json.Marshal(e.Val) - return json.Marshal(struct { - Timestamp time.Time `json:"timestamp"` - Index types.ChainIndex `json:"index"` - Relevant []types.Address `json:"relevant"` - Type string `json:"type"` - Val json.RawMessage `json:"val"` - }{ - Timestamp: e.Timestamp, - Index: e.Index, - Relevant: e.Relevant, - Type: e.Val.eventType(), - Val: val, - }) -} - -// UnmarshalJSON implements json.Unarshaler. -func (e *Event) UnmarshalJSON(data []byte) error { - var s struct { - Timestamp time.Time - Index types.ChainIndex - Relevant []types.Address - Type string - Val json.RawMessage - } - if err := json.Unmarshal(data, &s); err != nil { - return err - } - e.Timestamp = s.Timestamp - e.Index = s.Index - e.Relevant = s.Relevant - switch s.Type { - case (*EventTransaction)(nil).eventType(): - e.Val = new(EventTransaction) - case (*EventMinerPayout)(nil).eventType(): - e.Val = new(EventMinerPayout) - case (*EventMissedFileContract)(nil).eventType(): - e.Val = new(EventMissedFileContract) - } - if e.Val == nil { - return fmt.Errorf("unknown event type %q", s.Type) - } - return json.Unmarshal(s.Val, e.Val) -} - -// A HostAnnouncement represents a host announcement within an EventTransaction. -type HostAnnouncement struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` -} - -// A SiafundInput represents a siafund input within an EventTransaction. -type SiafundInput struct { - SiafundElement types.SiafundElement `json:"siafundElement"` - ClaimElement types.SiacoinElement `json:"claimElement"` -} - -// A FileContract represents a file contract within an EventTransaction. -type FileContract struct { - FileContract types.FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - ValidOutputs []types.SiacoinElement `json:"validOutputs,omitempty"` -} - -// A V2FileContract represents a v2 file contract within an EventTransaction. -type V2FileContract struct { - FileContract types.V2FileContractElement `json:"fileContract"` - // only non-nil if transaction revised contract - Revision *types.V2FileContract `json:"revision,omitempty"` - // only non-nil if transaction resolved contract - Resolution types.V2FileContractResolutionType `json:"resolution,omitempty"` - Outputs []types.SiacoinElement `json:"outputs,omitempty"` -} - -type EventTransaction struct { - ID types.TransactionID `json:"id"` - SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` - SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` - SiafundInputs []SiafundInput `json:"siafundInputs"` - SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` - FileContracts []FileContract `json:"fileContracts"` - V2FileContracts []V2FileContract `json:"v2FileContracts"` - HostAnnouncements []HostAnnouncement `json:"hostAnnouncements"` - Fee types.Currency `json:"fee"` -} - -type EventMinerPayout struct { - SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` -} - -type EventMissedFileContract struct { - FileContract types.FileContractElement `json:"fileContract"` - MissedOutputs []types.SiacoinElement `json:"missedOutputs"` -} - -type ChainUpdate interface { - ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) - ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) - ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) - ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) -} - // AppliedEvents extracts a list of relevant events from a chain update. -func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { - var events []Event - addEvent := func(v interface{ eventType() string }, relevant []types.Address) { +func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) (events []Event) { + addEvent := func(id types.Hash256, maturityHeight uint64, eventType string, v wallet.EventData, relevant []types.Address) { // dedup relevant addresses seen := make(map[types.Address]bool) unique := relevant[:0] for _, addr := range relevant { if !seen[addr] { - relevant = append(relevant, addr) + unique = append(unique, addr) seen[addr] = true } } events = append(events, Event{ - Timestamp: b.Timestamp, - Index: cs.Index, - Relevant: unique, - Val: v, + ID: id, + Timestamp: b.Timestamp, + Index: cs.Index, + MaturityHeight: maturityHeight, + Relevant: unique, + Type: eventType, + Data: v, }) } - // do a first pass to see if there's anything relevant in the block - relevantContract := func(fc types.FileContract) (addrs []types.Address) { - for _, sco := range fc.ValidProofOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + anythingRelevant := func() bool { + for _, sced := range cu.SiacoinElementDiffs() { + if relevant(sced.SiacoinElement.SiacoinOutput.Address) { + return true } } - for _, sco := range fc.MissedProofOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + for _, sfed := range cu.SiafundElementDiffs() { + if relevant(sfed.SiafundElement.SiafundOutput.Address) { + return true } } - return - } - relevantV2Contract := func(fc types.V2FileContract) (addrs []types.Address) { - if relevant(fc.RenterOutput.Address) { - addrs = append(addrs, fc.RenterOutput.Address) - } - if relevant(fc.HostOutput.Address) { - addrs = append(addrs, fc.HostOutput.Address) - } - return - } - relevantV2ContractResolution := func(res types.V2FileContractResolutionType) (addrs []types.Address) { - switch r := res.(type) { - case *types.V2FileContractFinalization: - return relevantV2Contract(types.V2FileContract(*r)) - case *types.V2FileContractRenewal: - return append(relevantV2Contract(r.InitialRevision), relevantV2Contract(r.FinalRevision)...) - } - return - } - anythingRelevant := func() (ok bool) { - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - if ok || relevant(sce.SiacoinOutput.Address) { - ok = true - } - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - if ok || relevant(sfe.SiafundOutput.Address) { - ok = true - } - }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if ok || len(relevantContract(fce.FileContract)) > 0 || (rev != nil && len(relevantContract(rev.FileContract)) > 0) { - ok = true - } - }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - if ok || - len(relevantV2Contract(fce.V2FileContract)) > 0 || - (rev != nil && len(relevantV2Contract(rev.V2FileContract)) > 0) || - (res != nil && len(relevantV2ContractResolution(res)) > 0) { - ok = true - } - }) - return + return false }() if !anythingRelevant { return nil @@ -343,272 +187,225 @@ func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant f // collect all elements sces := make(map[types.SiacoinOutputID]types.SiacoinElement) sfes := make(map[types.SiafundOutputID]types.SiafundElement) - fces := make(map[types.FileContractID]types.FileContractElement) - v2fces := make(map[types.FileContractID]types.V2FileContractElement) - cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { - sce.MerkleProof = nil - sces[types.SiacoinOutputID(sce.ID)] = sce - }) - cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { - sfe.MerkleProof = nil - sfes[types.SiafundOutputID(sfe.ID)] = sfe - }) - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - fce.MerkleProof = nil - fces[types.FileContractID(fce.ID)] = fce - }) - cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - fce.MerkleProof = nil - v2fces[types.FileContractID(fce.ID)] = fce - }) - - relevantTxn := func(txn types.Transaction) (addrs []types.Address) { + for _, sced := range cu.SiacoinElementDiffs() { + sce := sced.SiacoinElement + sce.StateElement.MerkleProof = nil + sces[sce.ID] = sce + } + for _, sfed := range cu.SiafundElementDiffs() { + sfe := sfed.SiafundElement + sfe.StateElement.MerkleProof = nil + sfes[sfe.ID] = sfe + } + + // handle v1 transactions + for _, txn := range b.Transactions { + addresses := make(map[types.Address]bool) + e := &wallet.EventV1Transaction{ + Transaction: txn, + SpentSiacoinElements: make([]types.SiacoinElement, 0, len(txn.SiacoinInputs)), + SpentSiafundElements: make([]types.SiafundElement, 0, len(txn.SiafundInputs)), + } + for _, sci := range txn.SiacoinInputs { - if sce := sces[sci.ParentID]; relevant(sce.SiacoinOutput.Address) { - addrs = append(addrs, sce.SiacoinOutput.Address) + sce, ok := sces[sci.ParentID] + if !ok { + continue + } + + e.SpentSiacoinElements = append(e.SpentSiacoinElements, sce) + if relevant(sce.SiacoinOutput.Address) { + addresses[sce.SiacoinOutput.Address] = true } } for _, sco := range txn.SiacoinOutputs { if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + addresses[sco.Address] = true } } + for _, sfi := range txn.SiafundInputs { - if sfe := sfes[sfi.ParentID]; relevant(sfe.SiafundOutput.Address) { - addrs = append(addrs, sfe.SiafundOutput.Address) + sfe, ok := sfes[sfi.ParentID] + if !ok { + continue + } + + e.SpentSiafundElements = append(e.SpentSiafundElements, sfe) + if relevant(sfe.SiafundOutput.Address) { + addresses[sfe.SiafundOutput.Address] = true + } + + sce, ok := sces[sfi.ParentID.ClaimOutputID()] + if ok && relevant(sce.SiacoinOutput.Address) && !sce.SiacoinOutput.Value.IsZero() { + addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ + SiacoinElement: sce, + }, []types.Address{sfi.ClaimAddress}) } } for _, sfo := range txn.SiafundOutputs { if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) + addresses[sfo.Address] = true } } - for _, fc := range txn.FileContracts { - addrs = append(addrs, relevantContract(fc)...) - } - for _, fcr := range txn.FileContractRevisions { - addrs = append(addrs, relevantContract(fcr.FileContract)...) + + // skip transactions with no relevant addresses + if len(addresses) == 0 { + continue } - for _, sp := range txn.StorageProofs { - addrs = append(addrs, relevantContract(fces[sp.ParentID].FileContract)...) + + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) } - return + + addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeV1Transaction, e, relevant) // transaction maturity height is the current block height } - relevantV2Txn := func(txn types.V2Transaction) (addrs []types.Address) { + // handle v2 transactions + for _, txn := range b.V2Transactions() { + addresses := make(map[types.Address]bool) for _, sci := range txn.SiacoinInputs { - if relevant(sci.Parent.SiacoinOutput.Address) { - addrs = append(addrs, sci.Parent.SiacoinOutput.Address) + if !relevant(sci.Parent.SiacoinOutput.Address) { + continue } + addresses[sci.Parent.SiacoinOutput.Address] = true } for _, sco := range txn.SiacoinOutputs { - if relevant(sco.Address) { - addrs = append(addrs, sco.Address) + if !relevant(sco.Address) { + continue } + addresses[sco.Address] = true } for _, sfi := range txn.SiafundInputs { - if relevant(sfi.Parent.SiafundOutput.Address) { - addrs = append(addrs, sfi.Parent.SiafundOutput.Address) + if !relevant(sfi.Parent.SiafundOutput.Address) { + continue } - } - for _, sfo := range txn.SiafundOutputs { - if relevant(sfo.Address) { - addrs = append(addrs, sfo.Address) + addresses[sfi.Parent.SiafundOutput.Address] = true + + sce, ok := sces[types.SiafundOutputID(sfi.Parent.ID).V2ClaimOutputID()] + if ok && relevant(sfi.ClaimAddress) && !sce.SiacoinOutput.Value.IsZero() { + addEvent(types.Hash256(sce.ID), sce.MaturityHeight, EventTypeSiafundClaim, wallet.EventPayout{ + SiacoinElement: sce, + }, []types.Address{sfi.ClaimAddress}) } } - for _, fc := range txn.FileContracts { - addrs = append(addrs, relevantV2Contract(fc)...) - } - for _, fcr := range txn.FileContractRevisions { - addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) - addrs = append(addrs, relevantV2Contract(fcr.Revision)...) - } - for _, fcr := range txn.FileContractResolutions { - addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) - switch r := fcr.Resolution.(type) { - case *types.V2FileContractFinalization: - addrs = append(addrs, relevantV2Contract(types.V2FileContract(*r))...) - case *types.V2FileContractRenewal: - addrs = append(addrs, relevantV2Contract(r.InitialRevision)...) - addrs = append(addrs, relevantV2Contract(r.FinalRevision)...) + for _, sco := range txn.SiafundOutputs { + if !relevant(sco.Address) { + continue } + addresses[sco.Address] = true } - return - } - // handle v1 transactions - for _, txn := range b.Transactions { - relevant := relevantTxn(txn) - if len(relevant) == 0 { + // skip transactions with no relevant addresses + if len(addresses) == 0 { continue } - e := &EventTransaction{ - ID: txn.ID(), - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), + ev := wallet.EventV2Transaction(txn) + relevant := make([]types.Address, 0, len(addresses)) + for addr := range addresses { + relevant = append(relevant, addr) } + addEvent(types.Hash256(txn.ID()), cs.Index.Height, EventTypeV2Transaction, ev, relevant) // transaction maturity height is the current block height + } - for i := range txn.SiacoinInputs { - e.SiacoinInputs[i] = sces[txn.SiacoinInputs[i].ParentID] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(i)] - } - for i := range txn.SiafundInputs { - e.SiafundInputs[i] = SiafundInput{ - SiafundElement: sfes[txn.SiafundInputs[i].ParentID], - ClaimElement: sces[txn.SiafundClaimOutputID(i)], - } - } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(i)] + // handle contracts + for _, fced := range cu.FileContractElementDiffs() { + if !fced.Resolved { + continue } - addContract := func(id types.FileContractID) *FileContract { - for i := range e.FileContracts { - if types.FileContractID(e.FileContracts[i].FileContract.ID) == id { - return &e.FileContracts[i] + + fce := fced.FileContractElement + fce.StateElement.MerkleProof = nil + + if fced.Valid { + for i := range fce.FileContract.ValidProofOutputs { + address := fce.FileContract.ValidProofOutputs[i].Address + if !relevant(address) { + continue } + + element := sces[types.FileContractID(fce.ID).ValidOutputID(i)] + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ + Parent: fce, + SiacoinElement: element, + Missed: false, + }, []types.Address{address}) } - e.FileContracts = append(e.FileContracts, FileContract{FileContract: fces[id]}) - return &e.FileContracts[len(e.FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.FileContractID(i)) - } - for i := range txn.FileContractRevisions { - fc := addContract(txn.FileContractRevisions[i].ParentID) - rev := txn.FileContractRevisions[i].FileContract - fc.Revision = &rev - } - for i := range txn.StorageProofs { - fc := addContract(txn.StorageProofs[i].ParentID) - fc.ValidOutputs = make([]types.SiacoinElement, len(fc.FileContract.FileContract.ValidProofOutputs)) - for i := range fc.ValidOutputs { - fc.ValidOutputs[i] = sces[types.FileContractID(fc.FileContract.ID).ValidOutputID(i)] - } - } - for _, arb := range txn.ArbitraryData { - var prefix types.Specifier - var uk types.UnlockKey - d := types.NewBufDecoder(arb) - prefix.DecodeFrom(d) - netAddress := d.ReadString() - uk.DecodeFrom(d) - if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") && - uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: *(*types.PublicKey)(uk.Key), - NetAddress: netAddress, - }) + } else { + for i := range fce.FileContract.MissedProofOutputs { + address := fce.FileContract.MissedProofOutputs[i].Address + if !relevant(address) { + continue + } + + element := sces[types.FileContractID(fce.ID).MissedOutputID(i)] + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV1ContractResolution, wallet.EventV1ContractResolution{ + Parent: fce, + SiacoinElement: element, + Missed: true, + }, []types.Address{address}) } } - for i := range txn.MinerFees { - e.Fee = e.Fee.Add(txn.MinerFees[i]) - } - - addEvent(e, relevant) } - // handle v2 transactions - for _, txn := range b.V2Transactions() { - relevant := relevantV2Txn(txn) - if len(relevant) == 0 { + for _, fced := range cu.V2FileContractElementDiffs() { + fce := fced.V2FileContractElement + res := fced.Resolution + if res == nil { continue } + fce.StateElement.MerkleProof = nil - txid := txn.ID() - e := &EventTransaction{ - ID: txid, - SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), - SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), - SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), - SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), - } - for i := range txn.SiacoinInputs { - // NOTE: here (and elsewhere), we fetch the element from our maps, - // rather than using the parent directly, because our copy has its - // Merkle proof nil'd out - e.SiacoinInputs[i] = sces[types.SiacoinOutputID(txn.SiacoinInputs[i].Parent.ID)] - } - for i := range txn.SiacoinOutputs { - e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(txid, i)] - } - for i := range txn.SiafundInputs { - sfoid := types.SiafundOutputID(txn.SiafundInputs[i].Parent.ID) - e.SiafundInputs[i] = SiafundInput{ - SiafundElement: sfes[sfoid], - ClaimElement: sces[sfoid.ClaimOutputID()], - } - } - for i := range txn.SiafundOutputs { - e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(txid, i)] - } - addContract := func(id types.FileContractID) *V2FileContract { - for i := range e.V2FileContracts { - if types.FileContractID(e.V2FileContracts[i].FileContract.ID) == id { - return &e.V2FileContracts[i] - } - } - e.V2FileContracts = append(e.V2FileContracts, V2FileContract{FileContract: v2fces[id]}) - return &e.V2FileContracts[len(e.V2FileContracts)-1] - } - for i := range txn.FileContracts { - addContract(txn.V2FileContractID(txid, i)) - } - for _, fcr := range txn.FileContractRevisions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Revision = &fcr.Revision - } - for _, fcr := range txn.FileContractResolutions { - fc := addContract(types.FileContractID(fcr.Parent.ID)) - fc.Resolution = fcr.Resolution - fc.Outputs = []types.SiacoinElement{ - sces[types.FileContractID(fcr.Parent.ID).V2RenterOutputID()], - sces[types.FileContractID(fcr.Parent.ID).V2HostOutputID()], - } - } - for _, a := range txn.Attestations { - if a.Key == "HostAnnouncement" { - e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ - PublicKey: a.PublicKey, - NetAddress: string(a.Value), - }) - } + var missed bool + if _, ok := res.(*types.V2FileContractExpiration); ok { + missed = true } - e.Fee = txn.MinerFee - addEvent(e, relevant) - } + if relevant(fce.V2FileContract.HostOutput.Address) { + element := sces[types.FileContractID(fce.ID).V2HostOutputID()] + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ + Resolution: types.V2FileContractResolution{ + Parent: fce, + Resolution: res, + }, + SiacoinElement: element, + Missed: missed, + }, []types.Address{fce.V2FileContract.HostOutput.Address}) + } - // handle missed contracts - cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { - if resolved && !valid { - relevant := relevantContract(fce.FileContract) - if len(relevant) == 0 { - return - } - missedOutputs := make([]types.SiacoinElement, len(fce.FileContract.MissedProofOutputs)) - for i := range missedOutputs { - missedOutputs[i] = sces[types.FileContractID(fce.ID).MissedOutputID(i)] - } - addEvent(&EventMissedFileContract{ - FileContract: fce, - MissedOutputs: missedOutputs, - }, relevant) + if relevant(fce.V2FileContract.RenterOutput.Address) { + element := sces[types.FileContractID(fce.ID).V2RenterOutputID()] + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeV2ContractResolution, wallet.EventV2ContractResolution{ + Resolution: types.V2FileContractResolution{ + Parent: fce, + Resolution: res, + }, + SiacoinElement: element, + Missed: missed, + }, []types.Address{fce.V2FileContract.RenterOutput.Address}) } - }) + } // handle block rewards for i := range b.MinerPayouts { if relevant(b.MinerPayouts[i].Address) { - addEvent(&EventMinerPayout{ - SiacoinOutput: sces[cs.Index.ID.MinerOutputID(i)], + element := sces[cs.Index.ID.MinerOutputID(i)] + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeMinerPayout, wallet.EventPayout{ + SiacoinElement: element, }, []types.Address{b.MinerPayouts[i].Address}) } } + // handle foundation subsidy + if relevant(cs.FoundationManagementAddress) { + element, ok := sces[cs.Index.ID.FoundationOutputID()] + if ok { + addEvent(types.Hash256(element.ID), element.MaturityHeight, EventTypeFoundationSubsidy, wallet.EventPayout{ + SiacoinElement: element, + }, []types.Address{element.SiacoinOutput.Address}) + } + } + return events } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go new file mode 100644 index 0000000..d9d3c18 --- /dev/null +++ b/wallet/wallet_test.go @@ -0,0 +1,4463 @@ +package wallet_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math/bits" + "path/filepath" + "reflect" + "sort" + "testing" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/testutil" + "go.sia.tech/walletd/persist/sqlite" + "go.sia.tech/walletd/wallet" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +func waitForBlock(tb testing.TB, cm *chain.Manager, ws wallet.Store) { + tb.Helper() + for i := 0; i < 1000; i++ { + time.Sleep(10 * time.Millisecond) + tip, _ := ws.LastCommittedIndex() + if tip == cm.Tip() { + return + } + } + tb.Fatal("timed out waiting for block") +} + +func mineAndSync(tb testing.TB, cm *chain.Manager, ws wallet.Store, addr types.Address, n int) { + tb.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(tb, cm, addr, 1) + waitForBlock(tb, cm, ws) + } +} + +func testV1Network(siafundAddr types.Address) (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 1000 + n.HardforkV2.RequireHeight = 1000 + return n, genesisBlock +} + +func testV2Network(siafundAddr types.Address) (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = siafundAddr + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 100 + n.HardforkV2.RequireHeight = 110 + return n, genesisBlock +} + +func mineBlock(state consensus.State, txns []types.Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + Transactions: txns, + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + } + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func mineV2Block(state consensus.State, txns []types.V2Transaction, minerAddr types.Address) types.Block { + b := types.Block{ + ParentID: state.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: minerAddr, Value: state.BlockReward()}}, + + V2: &types.V2BlockData{ + Transactions: txns, + Height: state.Index.Height + 1, + }, + } + b.V2.Commitment = state.Commitment(state.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + for b.ID().CmpWork(state.ChildTarget) < 0 { + b.Nonce += state.NonceFactor() + } + return b +} + +func TestReserve(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testutil.V2Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithLockDuration(2*time.Second)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sk := types.GeneratePrivateKey() + sp := types.SpendPolicy{Type: types.PolicyTypePublicKey(sk.PublicKey())} + addr := sp.Address() + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &sp, + }) + if err != nil { + t.Fatal(err) + } + + scoID := types.Hash256(frand.Entropy256()) + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } + + // output should be locked + if err := wm.Reserve([]types.Hash256{scoID}); !errors.Is(err, wallet.ErrAlreadyReserved) { + t.Fatalf("expected output locked error, got %v", err) + } + + time.Sleep(3 * time.Second) + + // output should be unlocked + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } + + // output should be locked + if err := wm.Reserve([]types.Hash256{scoID}); !errors.Is(err, wallet.ErrAlreadyReserved) { + t.Fatalf("expected output locked error, got %v", err) + } + + wm.Release([]types.Hash256{scoID}) + // output should be unlocked + if err := wm.Reserve([]types.Hash256{scoID}); err != nil { + t.Fatal(err) + } +} + +func TestSelectSiacoins(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testutil.Network() + network.InitialCoinbase = types.Siacoins(100) + network.MinimumCoinbase = types.Siacoins(100) + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sk := types.GeneratePrivateKey() + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{sk.PublicKey().UnlockKey()}, + SignaturesRequired: 1, + } + addr := uc.UnlockHash() + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + mineAndSync := func(t *testing.T, addr types.Address, n int) { + t.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(t, cm, addr, 1) + waitForBlock(t, cm, db) + } + } + // mine enough utxos to ensure the pagination works + mineAndSync(t, addr, 200) + // mine until all the wallet's outputs are mature + mineAndSync(t, types.VoidAddress, int(cm.TipState().Network.MaturityDelay)) + + // check that the wallet has 200 matured outputs + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 200 { + t.Fatalf("expected 200 outputs, got %v", len(utxos)) + } + + balance, err := wm.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } + + // fund a transaction with more than the wallet balance + _, _, _, err = wm.SelectSiacoinElements(w.ID, balance.Siacoins.Add(types.Siacoins(1)), false) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + + // fund multiple overlapping transactions to ensure no double spends + var selected []types.Hash256 + seen := make(map[types.SiacoinOutputID]bool) + for i := 0; i < len(utxos); i++ { + utxos, _, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { // one UTXO should always be enough to cover + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if seen[utxos[0].ID] { + t.Fatalf("double spend %v", utxos[0].ID) + } else if !change.Equals(types.Siacoins(99)) { + t.Fatalf("expected 99 SC change, got %v", change) + } + seen[utxos[0].ID] = true + selected = append(selected, types.Hash256(utxos[0].ID)) + } + + // all available outputs should be locked + _, _, _, err = wm.SelectSiacoinElements(w.ID, types.Siacoins(1), false) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + // release the selected outputs + wm.Release(selected) + + // fund and broadcast a transaction + utxos, basis, change, err := wm.SelectSiacoinElements(w.ID, types.Siacoins(101), false) // uses two outputs + if err != nil { + t.Fatal(err) + } else if len(utxos) != 2 { + t.Fatalf("expected 2 outputs, got %v", len(utxos)) + } else if !change.Equals(types.Siacoins(99)) { + t.Fatalf("expected 99 SC change, got %v", change) + } else if basis != cm.Tip() { + t.Fatalf("expected tip, got %v", basis) + } + txn := types.Transaction{ + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(101)}, + {Address: addr, Value: change}, + }, + } + for _, utxo := range utxos { + txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ + ParentID: types.SiacoinOutputID(utxo.ID), + UnlockConditions: uc, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxo.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + for i := range txn.Signatures { + sigHash := cm.TipState().WholeSigHash(txn, txn.Signatures[i].ParentID, 0, 0, nil) + sig := sk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + + known, err := cm.AddPoolTransactions([]types.Transaction{txn}) + if err != nil { + t.Fatal(err) + } else if known { + t.Fatal("transaction was already known") + } + + mineAndSync(t, types.VoidAddress, 1) + + events, err := wm.WalletEvents(w.ID, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if !events[0].SiacoinOutflow().Sub(events[0].SiacoinInflow()).Equals(types.Siacoins(101)) { + t.Fatalf("expected transaction value 101 SC, got %v", events[0].SiacoinOutflow().Sub(events[0].SiacoinInflow())) + } +} + +func TestSelectSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + sk := types.GeneratePrivateKey() + uc := types.UnlockConditions{ + PublicKeys: []types.UnlockKey{sk.PublicKey().UnlockKey()}, + SignaturesRequired: 1, + } + addr := uc.UnlockHash() + + network, genesisBlock := testutil.Network() + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + network.InitialCoinbase = types.Siacoins(100) + network.MinimumCoinbase = types.Siacoins(100) + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + mineAndSync := func(t *testing.T, addr types.Address, n int) { + t.Helper() + + for i := 0; i < n; i++ { + testutil.MineBlocks(t, cm, addr, 1) + waitForBlock(t, cm, db) + } + } + mineAndSync(t, types.VoidAddress, 1) + + // check that the wallet has a siafund utxo + utxos, _, err := wm.UnspentSiafundOutputs(w.ID, 0, 1000) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 outputs, got %v", len(utxos)) + } + + balance, err := wm.WalletBalance(w.ID) + if err != nil { + t.Fatal(err) + } + + // fund a transaction with more than the wallet balance + _, _, _, err = wm.SelectSiafundElements(w.ID, balance.Siafunds+1) + if !errors.Is(err, wallet.ErrInsufficientFunds) { + t.Fatal("expected insufficient funds error") + } + + // fund and broadcast a transaction + utxos, basis, change, err := wm.SelectSiafundElements(w.ID, balance.Siafunds/2) // uses two outputs + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 utxo, got %v", len(utxos)) + } else if change != balance.Siafunds/2 { + t.Fatalf("expected %v SF change, got %v", balance.Siafunds/2, change) + } else if basis != cm.Tip() { + t.Fatalf("expected tip, got %v", basis) + } + txn := types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: types.VoidAddress, Value: balance.Siafunds / 2}, + {Address: addr, Value: change}, + }, + } + for _, utxo := range utxos { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: types.SiafundOutputID(utxo.ID), + UnlockConditions: uc, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(utxo.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + for i := range txn.Signatures { + sigHash := cm.TipState().WholeSigHash(txn, txn.Signatures[i].ParentID, 0, 0, nil) + sig := sk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + + known, err := cm.AddPoolTransactions([]types.Transaction{txn}) + if err != nil { + t.Fatal(err) + } else if known { + t.Fatal("transaction was already known") + } + + mineAndSync(t, types.VoidAddress, 1) + + events, err := wm.WalletEvents(w.ID, 0, 1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if events[0].SiafundOutflow()-events[0].SiafundInflow() != balance.Siafunds/2 { + t.Fatalf("expected transaction value %v SF, got %v", balance.Siafunds/2, events[0].SiafundOutflow()-events[0].SiafundInflow()) + } +} + +func TestReorg(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + setupNode := func(t *testing.T, mode wallet.IndexMode) (consensus.State, *sqlite.Store, *chain.Manager, *wallet.Manager) { + t.Helper() + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { bdb.Close() }) + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(mode)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wm.Close() }) + return genesisState, db, cm, wm + } + + testReorg := func(t *testing.T, genesisState consensus.State, db *sqlite.Store, cm *chain.Manager, wm *wallet.Manager) { + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } + + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo has not matured + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) + } + + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 10; i++ { + block := mineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + state.Index.ID = block.ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the payout event was reverted + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // check that the utxo was removed + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } + + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo has not matured + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) + } + + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + waitForBlock(t, cm, db) + + // check that the balance was updated + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that only the single utxo still exists + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + } + + t.Run("IndexModePersonal", func(t *testing.T) { + state, db, cm, w := setupNode(t, wallet.IndexModePersonal) + testReorg(t, state, db, cm, w) + }) + + t.Run("IndexModeFull", func(t *testing.T) { + state, db, cm, w := setupNode(t, wallet.IndexModeFull) + testReorg(t, state, db, cm, w) + }) +} + +func TestEphemeralBalance(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + 1 + block := mineBlock(cm.TipState(), nil, addr) + minerPayoutID := block.ID().MinerOutputID(0) + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{block}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + balance, err := wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(minerPayoutID) { + t.Fatalf("expected %v, got %v", minerPayoutID, events[0].ID) + } + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // create a transaction that spends the matured payout + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + unlockConditions := types.StandardUnlockConditions(pk.PublicKey()) + parentTxn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: types.Siacoins(100)}, + {Address: types.VoidAddress, Value: utxos[0].SiacoinOutput.Value.Sub(types.Siacoins(100))}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + parentSigHash := cm.TipState().WholeSigHash(parentTxn, types.Hash256(utxos[0].ID), 0, 0, nil) + parentSig := pk.SignHash(parentSigHash) + parentTxn.Signatures[0].Signature = parentSig[:] + + outputID := parentTxn.SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: outputID, + UnlockConditions: unlockConditions, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(100)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(outputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(outputID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + txnset := []types.Transaction{parentTxn, txn} + + // broadcast the transactions + revertState := cm.TipState() + if err := cm.AddBlocks([]types.Block{mineBlock(revertState, txnset, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was spent + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.IsZero() { + t.Fatalf("expected 0, got %v", balance.Siacoins) + } + + // check that both transactions were added + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 3 { // 1 payout, 2 transactions + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[2].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[2].Type) + } else if events[1].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[1].Type) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[1].ID != types.Hash256(parentTxn.ID()) { // parent txn first + t.Fatalf("expected %v, got %v", parentTxn.ID(), events[1].ID) + } else if events[0].ID != types.Hash256(txn.ID()) { // child txn second + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } + + // trigger a reorg + var blocks []types.Block + state := revertState + for i := 0; i < 2; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the transaction was reverted + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.Siacoins) + } + + // check that only the payout event remains + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } +} + +func TestWalletAddresses(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // Add a wallet + w := wallet.Wallet{ + Name: "test", + Description: "hello, world!", + Metadata: json.RawMessage(`{"foo": "bar"}`), + } + w, err = wm.AddWallet(w) + if err != nil { + t.Fatal(err) + } + + wallets, err := wm.Wallets() + if err != nil { + t.Fatal(err) + } else if len(wallets) != 1 { + t.Fatal("expected 1 wallet, got", len(wallets)) + } else if wallets[0].ID != w.ID { + t.Fatal("unexpected wallet ID", wallets[0].ID) + } else if wallets[0].Name != "test" { + t.Fatal("unexpected wallet name", wallets[0].Name) + } else if wallets[0].Description != "hello, world!" { + t.Fatal("unexpected description", wallets[0].Description) + } else if !bytes.Equal(wallets[0].Metadata, []byte(`{"foo": "bar"}`)) { + t.Fatal("unexpected metadata", wallets[0].Metadata) + } else if wallets[0].DateCreated.IsZero() || !wallets[0].DateCreated.Equal(w.DateCreated) { + t.Fatalf("expected creation date %s, got %s", w.DateCreated, wallets[0].DateCreated) + } else if wallets[0].LastUpdated.IsZero() || !wallets[0].LastUpdated.Equal(w.LastUpdated) { + t.Fatalf("expected last updated date %s, got %s", w.LastUpdated, wallets[0].LastUpdated) + } + + // Add an address + pk := types.GeneratePrivateKey() + spendPolicy := types.PolicyPublicKey(pk.PublicKey()) + address := spendPolicy.Address() + + addr := wallet.Address{ + Address: address, + SpendPolicy: &spendPolicy, + Description: "hello, world", + } + err = wm.AddAddress(w.ID, addr) + if err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err := wm.Addresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "hello, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } + + // update the addresses metadata and description + addr.Description = "goodbye, world" + addr.Metadata = json.RawMessage(`{"foo": "bar"}`) + + if err := wm.AddAddress(w.ID, addr); err != nil { + t.Fatal(err) + } + + // Check that the address was added + addresses, err = wm.Addresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 1 { + t.Fatal("expected 1 address, got", len(addresses)) + } else if addresses[0].Address != address { + t.Fatal("unexpected address", addresses[0].Address) + } else if addresses[0].Description != "goodbye, world" { + t.Fatal("unexpected description", addresses[0].Description) + } else if *addresses[0].SpendPolicy != spendPolicy { + t.Fatal("unexpected spend policy", addresses[0].SpendPolicy) + } else if string(addresses[0].Metadata) != `{"foo": "bar"}` { + t.Fatal("unexpected metadata", addresses[0].Metadata) + } + + // Remove the address + err = wm.RemoveAddress(w.ID, address) + if err != nil { + t.Fatal(err) + } + + // Check that the address was removed + addresses, err = wm.Addresses(w.ID) + if err != nil { + t.Fatal(err) + } else if len(addresses) != 0 { + t.Fatal("expected 0 addresses, got", len(addresses)) + } +} + +func TestScan(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + // rescan to get the genesis Siafund state + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + checkBalance := func(siacoin, immature types.Currency) error { + waitForBlock(t, cm, db) + + // note: the siafund balance is currently hardcoded to the number of + // siafunds in genesis. If we ever modify this test to also spend + // siafunds, this will need to be updated. + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if b.Siafunds != network.GenesisState().SiafundCount() { + return fmt.Errorf("expected siafund balance %v, got %v", network.GenesisState().SiafundCount(), b.Siafunds) + } + return nil + } + + // check that the wallet has no balance + if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + expectedBalance1 := cm.TipState().BlockReward() + + // mine a block to fund the first address + b, ok := coreutils.MineBlock(cm, addr, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + expectedBalance2 := cm.TipState().BlockReward() + + // mine a block to fund the second address + b, ok = coreutils.MineBlock(cm, addr2, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the wallet has one immature payout + if err := checkBalance(types.ZeroCurrency, expectedBalance1); err != nil { + t.Fatal(err) + } + + // mine until the first payout matures + for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance did not change + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // add the second address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + if err := checkBalance(expectedBalance1, expectedBalance2); err != nil { + t.Fatal(err) + } + + // mine a block to mature the second payout + b, ok = coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // sanity check + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } +} + +func TestSiafunds(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr1 + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + checkBalance := func(walletID wallet.ID, siafunds uint64) error { + waitForBlock(t, cm, db) + + b, err := wm.WalletBalance(walletID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if b.Siafunds != siafunds { + return fmt.Errorf("expected siafund balance %v, got %v", siafunds, b.Siafunds) + } + return nil + } + + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, network.GenesisState().SiafundCount()); err != nil { + t.Fatal(err) + } + + // split the siafunds between the two addresses + sendAmount := network.GenesisState().SiafundCount() / 2 + parentID := genesisBlock.Transactions[0].SiafundOutputID(0) + txn := types.Transaction{ + SiafundInputs: []types.SiafundInput{ + { + ParentID: parentID, + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr2, Value: sendAmount}, + {Address: addr1, Value: sendAmount}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(parentID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + state := cm.TipState() + sigHash := state.WholeSigHash(txn, txn.Signatures[0].ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + // check that the transaction made it into the pool + if events, err := wm.WalletUnconfirmedEvents(w1.ID); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].ID != types.Hash256(txn.ID()) { + t.Fatalf("expected %v, got %v", txn.ID(), events[0].ID) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected %v, got %v", addr1, events[0].Relevant[0]) + } + + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // rescan for sanity check + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // add a second wallet + w2, err := wm.AddWallet(wallet.Wallet{Name: "test2"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // wallet should have no balance since it hasn't been scanned + if err := checkBalance(w2.ID, 0); err != nil { + t.Fatal(err) + } + + // rescan for the second wallet + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } else if err := checkBalance(w2.ID, sendAmount); err != nil { + t.Fatal(err) + } else if err := checkBalance(w1.ID, sendAmount); err != nil { + t.Fatal(err) + } + + // add the first address to the second wallet + if err := wm.AddAddress(w2.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + // rescan shouldn't be necessary since the address was already scanned + if err := checkBalance(w2.ID, network.GenesisState().SiafundCount()); err != nil { + t.Fatal(err) + } +} + +func TestOrphans(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + network.HardforkV2.AllowHeight = 200 + network.HardforkV2.RequireHeight = 201 + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // mine until the maturity height + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } + return nil + } + + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo was created + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + resetState := cm.TipState() + + // send a transaction that will be orphaned + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: expectedPayout.Div64(2)}, // send the other half to the void + {Address: addr, Value: expectedPayout.Div64(2)}, // send half the payout back to the wallet + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } else if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), []types.Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + if err := assertBalance(expectedPayout.Div64(2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was recorded + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // simulate an interrupted rescan by closing the wallet manager, resetting the + // last scan index, and initializing a new wallet manager. + if err := wm.Close(); err != nil { + t.Fatal(err) + } else if err := db.ResetLastIndex(); err != nil { + t.Fatal(err) + } + + // mine to trigger a reorg. The underlying store must properly revert the + // orphaned blocks that will not be cleanly reverted since the rescan was + // interrupted. + var blocks []types.Block + state := resetState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + // check that the transaction was reverted + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was reverted + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } + + // check that the utxo was reverted + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if !utxos[0].SiacoinOutput.Value.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } +} + +func TestFullIndex(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(addr2) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + assertBalance := func(t *testing.T, address types.Address, siacoin, immature types.Currency, siafund uint64) { + t.Helper() + + b, err := wm.AddressBalance(address) + if err != nil { + t.Fatal(err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if b.Siafunds != siafund { + t.Fatalf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } + } + + // check the events are empty for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // assert that the airdropped siafunds are on the second address + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + // check the events for the air dropped siafunds + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + // mine a block and send the payout to the first address + expectedBalance1 := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check the payout was received + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } + + assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // check that the events did not change + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } + + assertBalance(t, addr, expectedBalance1, types.ZeroCurrency, 0) + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + + // send half siacoins to the second address + utxos, _, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + for _, se := range utxos { + if sce, err := wm.SiacoinElement(types.SiacoinOutputID(se.ID)); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(sce, se) { + t.Fatalf("expected %v, got %v", se, sce) + } + } + + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + // check the events for the second address + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + + for _, se := range sf { + if sfe, err := wm.SiafundElement(types.SiafundOutputID(se.ID)); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(sfe, se) { + t.Fatalf("expected %v, got %v", se, sfe) + } + } + + // send the siafunds to the first address + policy = types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk2.PublicKey())) + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sf[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + ClaimAddress: addr2, // claim address shouldn't create an event since the value is 0 + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sf[0].SiafundOutput.Value}, + }, + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk2.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + + // check the events for the transaction + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } + + // check the events for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } +} + +func TestEvents(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(addr2) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + assertBalance := func(t *testing.T, address types.Address, siacoin, immature types.Currency, siafund uint64) { + t.Helper() + + b, err := wm.AddressBalance(address) + if err != nil { + t.Fatal(err) + } else if !b.ImmatureSiacoins.Equals(immature) { + t.Fatalf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + t.Fatalf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if b.Siafunds != siafund { + t.Fatalf("expected siafund balance %v, got %v", siafund, b.Siafunds) + } + } + + // check the events are empty for the first address + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // assert that the airdropped siafunds are on the second address + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + // check the events for the air dropped siafunds + if events, err := wm.AddressEvents(addr2, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + // mine a block and send the payout to the first address + expectedBalance1 := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check the payout was received + events, err := wm.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } + + expected := events[0] + expected.Relevant = nil // clear the relevant field for deep equal + events2, err := wm.Events([]types.Hash256{events[0].ID}) + if err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2[0]) + } + + assertBalance(t, addr, types.ZeroCurrency, expectedBalance1, 0) + + // mine until the payout matures + for i := cm.TipState().Index.Height; i < maturityHeight; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + // check that the events did not change + if events, err := wm.AddressEvents(addr, 0, 100); err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected miner payout event, got %v", events[0].Type) + } + + assertBalance(t, addr, expectedBalance1, types.ZeroCurrency, 0) + assertBalance(t, addr2, types.ZeroCurrency, types.ZeroCurrency, cm.TipState().SiafundCount()) + + // send half siacoins to the second address + utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") + } + + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: utxos[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + mineAndSync(t, cm, db, types.VoidAddress, 1) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + + // check the events for the transaction + events, err = wm.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) + } + + // check the events for the second address + events, err = wm.AddressEvents(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + events2, err = wm.Events([]types.Hash256{events[0].ID}) + if err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2[0]) + } + + sf, _, err := wm.AddressSiafundOutputs(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } + + // send the siafunds to the first address + policy = types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk2.PublicKey())) + txn = types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sf[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{ + Type: policy, + }, + }, + ClaimAddress: addr2, // claim address shouldn't create an event since the value is 0 + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sf[0].SiafundOutput.Value}, + }, + } + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk2.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance(t, addr, expectedBalance1.Div64(2), types.ZeroCurrency, cm.TipState().SiafundCount()) + assertBalance(t, addr2, expectedBalance1.Div64(2), types.ZeroCurrency, 0) + + // check the events for the transaction + events, err = wm.AddressEvents(addr2, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 4 events, got %v", len(events)) + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) + } + + // check the events for the first address + events, err = wm.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 3 { + t.Fatalf("expected 3 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } + + expected = events[0] + expected.Relevant = nil // clear the relevant field for deep equal + if events2, err := wm.Events([]types.Hash256{expected.ID}); err != nil { + t.Fatalf("expected to get event: %v", err) + } else if !reflect.DeepEqual(events2[0], expected) { + t.Fatalf("expected event %v to match %v", expected, events2) + } +} + +func TestWalletUnconfirmedEvents(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + // mine a block sending the payout to the wallet + mineAndSync(t, cm, db, addr1, 1) + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + + utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + // generate a second address to send the payout to + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a transaction that splits the payout + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr1, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event was recorded + events, err := wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant address, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } + + txnData := events[0].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != utxos[0].ID { + t.Fatalf("expected siacoin output %v, got %v", utxos[0].ID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != utxos[0].SiacoinOutput.Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // add the second address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event's relevant addresses were updated + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if len(events[0].Relevant) != 2 { + t.Fatalf("expected 2 relevant addresses, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } else if events[0].Relevant[1] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[0].Relevant[1]) + } + + // spend the ephemeral output + ephemeralOutputID := txn.SiacoinOutputID(0) + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: ephemeralOutputID, + UnlockConditions: types.StandardUnlockConditions(pk2.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: txn.SiacoinOutputs[0].Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(ephemeralOutputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash = cm.TipState().WholeSigHash(txn2, txn2.Signatures[0].ParentID, 0, 0, nil) + sig2 := pk2.SignHash(sigHash) + txn2.Signatures[0].Signature = sig2[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn, txn2}); err != nil { + t.Fatal(err) + } + + // check that the new unconfirmed event was recorded + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[1].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[1].Relevant) != 1 { // second event is only relevant to the second address + t.Fatalf("expected 1 relevant addresses, got %v", len(events[1].Relevant)) + } + + txnData = events[1].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != ephemeralOutputID { + t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // mine the transactions + mineAndSync(t, cm, db, types.VoidAddress, 1) + + // check that the unconfirmed events were removed + events, err = wm.WalletUnconfirmedEvents(w1.ID) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } +} + +func TestAddressUnconfirmedEvents(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr1 := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // create a wallet with no addresses + w1, err := wm.AddWallet(wallet.Wallet{Name: "test1"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr1}); err != nil { + t.Fatal(err) + } + + // mine a block sending the payout to the wallet + mineAndSync(t, cm, db, addr1, 1) + // mine until the payout matures + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + + utxos, _, err := wm.UnspentSiacoinOutputs(w1.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } + + // generate a second address to send the payout to + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a transaction that splits the payout + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: types.SiacoinOutputID(utxos[0].ID), + UnlockConditions: types.StandardUnlockConditions(pk.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr2, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + {Address: addr1, Value: utxos[0].SiacoinOutput.Value.Div64(2)}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(utxos[0].ID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(utxos[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + // check that the unconfirmed event was recorded + events, err := wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant address, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr1 { + t.Fatalf("expected address %v, got %v", addr1, events[0].Relevant[0]) + } + + txnData := events[0].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != utxos[0].ID { + t.Fatalf("expected siacoin output %v, got %v", utxos[0].ID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != utxos[0].SiacoinOutput.Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // add the second address to the wallet + if err := wm.AddAddress(w1.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } + + // check that the address now shows an unconfirmed event + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if len(events[0].Relevant) != 1 { + t.Fatalf("expected 1 relevant addresses, got %v", len(events[0].Relevant)) + } else if events[0].Relevant[0] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[0].Relevant[1]) + } + + // spend the ephemeral output + ephemeralOutputID := txn.SiacoinOutputID(0) + txn2 := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + { + ParentID: ephemeralOutputID, + UnlockConditions: types.StandardUnlockConditions(pk2.PublicKey()), + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: txn.SiacoinOutputs[0].Value}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(ephemeralOutputID), + PublicKeyIndex: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + + sigHash = cm.TipState().WholeSigHash(txn2, txn2.Signatures[0].ParentID, 0, 0, nil) + sig2 := pk2.SignHash(sigHash) + txn2.Signatures[0].Signature = sig2[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn, txn2}); err != nil { + t.Fatal(err) + } + + // check that the first address still shows only one unconfirmed event + events, err = wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } + + // check that the second address now shows two unconfirmed events + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 event, got %v", len(events)) + } else if events[1].Type != wallet.EventTypeV1Transaction { + t.Fatalf("expected unconfirmed event, got %v", events[0].Type) + } else if len(events[1].Relevant) != 1 { // second event is only relevant to the second address + t.Fatalf("expected 1 relevant addresses, got %v", len(events[1].Relevant)) + } else if events[1].Relevant[0] != addr2 { + t.Fatalf("expected address %v, got %v", addr2, events[1].Relevant[0]) + } + + txnData = events[1].Data.(wallet.EventV1Transaction) + if txnData.SpentSiacoinElements[0].ID != ephemeralOutputID { + t.Fatalf("expected siacoin output %v, got %v", ephemeralOutputID, txnData.SpentSiacoinElements[0].ID) + } else if txnData.SpentSiacoinElements[0].SiacoinOutput.Value != txn.SiacoinOutputs[0].Value { + t.Fatalf("expected siacoin value %v, got %v", utxos[0].SiacoinOutput.Value, txnData.SpentSiacoinElements[0].SiacoinOutput.Value) + } + + // mine the transactions + mineAndSync(t, cm, db, types.VoidAddress, 1) + + // check that the unconfirmed events were removed + events, err = wm.AddressUnconfirmedEvents(addr1) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } + + events, err = wm.AddressUnconfirmedEvents(addr2) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 event, got %v", len(events)) + } +} + +func TestV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + mineAndSync(t, cm, db, addr, 1) + + // check that the payout was received + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.ImmatureSiacoins.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // mine until the payout matures + mineAndSync(t, cm, db, types.VoidAddress, int(network.MaturityDelay)) + + // create a v2 transaction that spends the matured payout + utxos, basis, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value.Sub(types.Siacoins(100))}, + {Address: addr, Value: types.Siacoins(100)}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + mineAndSync(t, cm, db, types.VoidAddress, 1) + + // check that the change was received + balance, err = wm.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } else if !balance.Siacoins.Equals(types.Siacoins(100)) { + t.Fatalf("expected %v, got %v", expectedPayout, balance.ImmatureSiacoins) + } + + // check that a transaction event was recorded + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeV2Transaction { + t.Fatalf("expected transaction event, got %v", events[0].Type) + } else if events[0].Relevant[0] != addr { + t.Fatalf("expected address %v, got %v", addr, events[0].Relevant[0]) + } +} + +func TestScanV2(t *testing.T) { + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // mine a single payout to the wallet + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testV2Network(addr) + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + pk2 := types.GeneratePrivateKey() + addr2 := types.StandardUnlockHash(pk2.PublicKey()) + + // create a wallet with no addresses + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + // add the address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + // rescan to get the genesis Siafund state + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + checkBalance := func(siacoin, immature types.Currency) error { + waitForBlock(t, cm, db) + + // note: the siafund balance is currently hardcoded to the number of + // siafunds in genesis. If we ever modify this test to also spend + // siafunds, this will need to be updated. + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if b.Siafunds != network.GenesisState().SiafundCount() { + return fmt.Errorf("expected siafund balance %v, got %v", network.GenesisState().SiafundCount(), b.Siafunds) + } + return nil + } + + // check that the wallet has no balance + if err := checkBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + expectedBalance1 := cm.TipState().BlockReward() + // mine a block to fund the first address + if b, ok := coreutils.MineBlock(cm, addr, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // mine a block to fund the second address + expectedBalance2 := cm.TipState().BlockReward() + if b, ok := coreutils.MineBlock(cm, addr2, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the wallet has one immature payout + if err := checkBalance(types.ZeroCurrency, expectedBalance1); err != nil { + t.Fatal(err) + } + + // mine until the first payout matures + for i := cm.Tip().Height; i < genesisState.MaturityHeight(); i++ { + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance did not change + if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // add the second address to the wallet + if err := wm.AddAddress(w.ID, wallet.Address{Address: addr2}); err != nil { + t.Fatal(err) + } else if err := checkBalance(expectedBalance1, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // scan for changes + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + if err := checkBalance(expectedBalance1, expectedBalance2); err != nil { + t.Fatal(err) + } + + // mine a block to mature the second payout + if b, ok := coreutils.MineBlock(cm, types.VoidAddress, 5*time.Second); !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // sanity check + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // check that the wallet balance has matured + if err := checkBalance(expectedBalance1.Add(expectedBalance2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + utxos, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") + } + + // spend the payout + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + mineAndSync(t, cm, db, types.VoidAddress, 1) + + // check that the first address has a balance of zero + if err := checkBalance(expectedBalance2, types.ZeroCurrency); err != nil { + t.Fatal(err) + } +} + +func TestReorgV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } + return nil + } + + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo has not matured + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) + } + + // mine to trigger a reorg + var blocks []types.Block + state := genesisState + for i := 0; i < 10; i++ { + block := mineBlock(state, nil, types.VoidAddress) + blocks = append(blocks, block) + state.Index.ID = block.ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance was reverted + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the payout event was reverted + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 0 { + t.Fatalf("expected 0 events, got %v", len(events)) + } + + // check that the utxo was removed + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 outputs, got %v", len(utxos)) + } + + // mine a new payout + expectedPayout = cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the payout was received + if err := assertBalance(types.ZeroCurrency, expectedPayout); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo has not matured + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected no outputs, got %v", len(utxos)) + } + + // mine until the payout matures + var prevState consensus.State + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + if i == maturityHeight-5 { + prevState = cm.TipState() + } + } + waitForBlock(t, cm, db) + + // check that the balance was updated + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // reorg the last few blocks to re-mature the payout + blocks = nil + state = prevState + for i := 0; i < 10; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that only the single utxo still exists + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + // spend the payout + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that all UTXOs have been spent + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 output, got %v", len(utxos)) + } +} + +func TestOrphansV2(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV2Network(types.VoidAddress) // don't care about siafunds + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + expectedPayout := cm.TipState().BlockReward() + maturityHeight := cm.TipState().MaturityHeight() + // mine a block sending the payout to the wallet + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, addr)}); err != nil { + t.Fatal(err) + } + + // mine until the maturity height + for i := cm.TipState().Index.Height; i < maturityHeight+1; i++ { + if err := cm.AddBlocks([]types.Block{mineBlock(cm.TipState(), nil, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + + assertBalance := func(siacoin, immature types.Currency) error { + b, err := wm.WalletBalance(w.ID) + if err != nil { + return fmt.Errorf("failed to check balance: %w", err) + } else if !b.ImmatureSiacoins.Equals(immature) { + return fmt.Errorf("expected immature siacoin balance %v, got %v", immature, b.ImmatureSiacoins) + } else if !b.Siacoins.Equals(siacoin) { + return fmt.Errorf("expected siacoin balance %v, got %v", siacoin, b.Siacoins) + } + return nil + } + + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that a payout event was recorded + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } else if events[0].Type != wallet.EventTypeMinerPayout { + t.Fatalf("expected payout event, got %v", events[0].Type) + } + + // check that the utxo was created + utxos, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if utxos[0].SiacoinOutput.Value.Cmp(expectedPayout) != 0 { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } else if utxos[0].MaturityHeight != maturityHeight { + t.Fatalf("expected %v, got %v", maturityHeight, utxos[0].MaturityHeight) + } + + resetState := cm.TipState() + + // send a transaction that will be orphaned + sce := utxos[0] + policy := types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())) + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: expectedPayout.Div64(2)}, // send the other half to the void + {Address: addr, Value: expectedPayout.Div64(2)}, // send half the payout back to the wallet + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + // broadcast the transaction + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + if err := assertBalance(expectedPayout.Div64(2), types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was recorded + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // simulate an interrupted rescan by closing the wallet manager, resetting the + // last scan index, and initializing a new wallet manager. + if err := wm.Close(); err != nil { + t.Fatal(err) + } else if err := db.ResetLastIndex(); err != nil { + t.Fatal(err) + } + + // mine to trigger a reorg. The underlying store must properly revert the + // orphaned blocks that will not be cleanly reverted since the rescan was + // interrupted. + var blocks []types.Block + state := resetState + for i := 0; i < 5; i++ { + blocks = append(blocks, mineBlock(state, nil, types.VoidAddress)) + state.Index.ID = blocks[len(blocks)-1].ID() + state.Index.Height++ + } + if err := cm.AddBlocks(blocks); err != nil { + t.Fatal(err) + } + + wm, err = wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm, db) + + // check that the transaction was reverted + if err := assertBalance(expectedPayout, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that the transaction event was reverted + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 1 { + t.Fatalf("expected 1 event, got %v", len(events)) + } + + // check that the utxo was reverted + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 1 { + t.Fatalf("expected 1 output, got %v", len(utxos)) + } else if !utxos[0].SiacoinOutput.Value.Equals(expectedPayout) { + t.Fatalf("expected %v, got %v", expectedPayout, utxos[0].SiacoinOutput.Value) + } + + // spend the payout + txn = types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: policy}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: sce.SiacoinOutput.Value}, + }, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(cm.TipState().InputSigHash(txn))} + + if err := cm.AddBlocks([]types.Block{mineV2Block(cm.TipState(), []types.V2Transaction{txn}, types.VoidAddress)}); err != nil { + t.Fatal(err) + } + waitForBlock(t, cm, db) + + // check that the balance is correct + if err := assertBalance(types.ZeroCurrency, types.ZeroCurrency); err != nil { + t.Fatal(err) + } + + // check that all UTXOs have been spent + utxos, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(utxos) != 0 { + t.Fatalf("expected 0 output, got %v", len(utxos)) + } +} + +func TestDeleteWallet(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesisBlock := testV1Network(types.VoidAddress) // don't care about siafunds + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } else if err := wm.AddAddress(w.ID, wallet.Address{Address: addr}); err != nil { + t.Fatal(err) + } + + if err := wm.DeleteWallet(w.ID); err != nil { + t.Fatal(err) + } +} + +// NOTE: due to a bug in the transaction validation code, calculating payouts +// is way harder than it needs to be. Tax is calculated on the post-tax +// contract payout (instead of the sum of the renter and host payouts). So the +// equation for the payout is: +// +// payout = renterPayout + hostPayout + payout*tax +// ∴ payout = (renterPayout + hostPayout) / (1 - tax) +// +// This would work if 'tax' were a simple fraction, but because the tax must +// be evenly distributed among siafund holders, 'tax' is actually a function +// that multiplies by a fraction and then rounds down to the nearest multiple +// of the siafund count. Thus, when inverting the function, we have to make an +// initial guess and then fix the rounding error. +func taxAdjustedPayout(target types.Currency) types.Currency { + // compute initial guess as target * (1 / 1-tax); since this does not take + // the siafund rounding into account, the guess will be up to + // types.SiafundCount greater than the actual payout value. + guess := target.Mul64(1000).Div64(961) + + // now, adjust the guess to remove the rounding error. We know that: + // + // (target % types.SiafundCount) == (payout % types.SiafundCount) + // + // therefore, we can simply adjust the guess to have this remainder as + // well. The only wrinkle is that, since we know guess >= payout, if the + // guess remainder is smaller than the target remainder, we must subtract + // an extra types.SiafundCount. + // + // for example, if target = 87654321 and types.SiafundCount = 10000, then: + // + // initial_guess = 87654321 * (1 / (1 - tax)) + // = 91211572 + // target % 10000 = 4321 + // adjusted_guess = 91204321 + + mod64 := func(c types.Currency, v uint64) types.Currency { + var r uint64 + if c.Hi < v { + _, r = bits.Div64(c.Hi, c.Lo, v) + } else { + _, r = bits.Div64(0, c.Hi, v) + _, r = bits.Div64(r, c.Lo, v) + } + return types.NewCurrency64(r) + } + sfc := (consensus.State{}).SiafundCount() + tm := mod64(target, sfc) + gm := mod64(guess, sfc) + if gm.Cmp(tm) < 0 { + guess = guess.Sub(types.NewCurrency64(sfc)) + } + return guess.Add(tm).Sub(gm) +} + +func TestEventTypes(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zap.NewNop() + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + // create a new test network with the Siafund airdrop going to the wallet address + network, genesisBlock := testV2Network(addr) + // raise the require height to test v1 events + network.HardforkV2.RequireHeight = 250 + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + // helper to mine blocks + mineBlock := func(n int, addr types.Address) { + t.Helper() + for i := 0; i < n; i++ { + b, ok := coreutils.MineBlock(cm, addr, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } + } + waitForBlock(t, cm, db) + } + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + spendableSiacoinUTXOs := func(t *testing.T) ([]types.SiacoinElement, types.ChainIndex) { + t.Helper() + + sces, basis, err := wm.AddressSiacoinOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") + } + filtered := sces[:0] + height := cm.Tip().Height + for _, sce := range sces { + if sce.MaturityHeight > height { + continue + } + filtered = append(filtered, sce) + } + sort.Slice(filtered, func(i, j int) bool { + return filtered[i].SiacoinOutput.Value.Cmp(filtered[j].SiacoinOutput.Value) < 0 + }) + return filtered, basis + } + + assertEvent := func(t *testing.T, id types.Hash256, eventType string, expectedInflow, expectedOutflow types.Currency, maturityHeight uint64) { + t.Helper() + + events, err := wm.AddressEvents(addr, 0, 100) + if err != nil { + t.Fatal(err) + } + + for _, event := range events { + if event.ID == id { + if event.Type != eventType { + t.Fatalf("expected %v event, got %v", eventType, event.Type) + } else if event.MaturityHeight != maturityHeight { + t.Fatalf("expected maturity height %v, got %v", maturityHeight, event.MaturityHeight) + } + + if !event.SiacoinInflow().Equals(expectedInflow) { + t.Fatalf("expected inflow %v, got %v", expectedInflow, event.SiacoinInflow()) + } else if !event.SiacoinOutflow().Equals(expectedOutflow) { + t.Fatalf("expected outflow %v, got %v", expectedOutflow, event.SiacoinOutflow()) + } + return + } + } + t.Fatalf("event not found") + } + + // miner payout event + mineBlock(1, addr) + assertEvent(t, types.Hash256(cm.Tip().ID.MinerOutputID(0)), wallet.EventTypeMinerPayout, genesisState.BlockReward(), types.ZeroCurrency, genesisState.MaturityHeight()) + + // mine until the payout matures + mineBlock(int(cm.TipState().MaturityHeight()), types.VoidAddress) + + // v1 transaction + t.Run("v1 transaction", func(t *testing.T) { + sce, _ := spendableSiacoinUTXOs(t) + + // v1 only supports unlock conditions + uc := types.StandardUnlockConditions(pk.PublicKey()) + + // create a transaction + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + {ParentID: types.SiacoinOutputID(sce[0].ID), UnlockConditions: uc}, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1000)}, + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000))}, + }, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(sce[0].ID), + PublicKeyIndex: 0, + Timelock: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + + // sign the transaction + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(sce[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(txn.ID()), wallet.EventTypeV1Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + }) + + t.Run("v1 contract resolution - missed", func(t *testing.T) { + // v1 contract resolution - only one type of resolution is supported. + // The only difference is `missed == true` or `missed == false` + + sce, _ := spendableSiacoinUTXOs(t) + uc := types.StandardUnlockConditions(pk.PublicKey()) + + // create a storage contract + contractPayout := types.Siacoins(10000) + fc := types.FileContract{ + WindowStart: cm.TipState().Index.Height + 10, + WindowEnd: cm.TipState().Index.Height + 20, + Payout: taxAdjustedPayout(contractPayout), + ValidProofOutputs: []types.SiacoinOutput{ + {Address: addr, Value: contractPayout}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: addr, Value: contractPayout}, + }, + } + + // create a transaction with the contract + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{ + {ParentID: types.SiacoinOutputID(sce[0].ID), UnlockConditions: uc}, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(fc.Payout)}, // return the remainder to the wallet + }, + FileContracts: []types.FileContract{fc}, + Signatures: []types.TransactionSignature{ + { + ParentID: types.Hash256(sce[0].ID), + PublicKeyIndex: 0, + Timelock: 0, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }, + }, + } + sigHash := cm.TipState().WholeSigHash(txn, types.Hash256(sce[0].ID), 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[0].Signature = sig[:] + + // broadcast the transaction + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + + txn.FileContractID(0).MissedOutputID(0) + + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + // mine until the contract expires to trigger the resolution event + blocksRemaining := int(fc.WindowEnd - cm.Tip().Height) + mineBlock(blocksRemaining, types.VoidAddress) + assertEvent(t, types.Hash256(txn.FileContractID(0).MissedOutputID(0)), wallet.EventTypeV1ContractResolution, contractPayout, types.ZeroCurrency, fc.WindowEnd+144) + }) + + t.Run("v2 transaction", func(t *testing.T) { + sce, basis := spendableSiacoinUTXOs(t) + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: types.Siacoins(1000)}, + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000))}, + }, + } + sigHash := cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(txn.ID()), wallet.EventTypeV2Transaction, sce[0].SiacoinOutput.Value.Sub(types.Siacoins(1000)), sce[0].SiacoinOutput.Value, cm.Tip().Height) + }) + + t.Run("v2 contract resolution - expired", func(t *testing.T) { + sce, basis := spendableSiacoinUTXOs(t) + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract expires + mineBlock(int(fc.ExpirationHeight-cm.Tip().Height), types.VoidAddress) + + // this is kind of annoying because we have to keep the file contract + // proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { + cau.UpdateElementProof(&fce.StateElement) + } + + resolutionTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &types.V2FileContractExpiration{}, + }, + }, + } + // broadcast the expire resolution + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the resolution + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + }) + + t.Run("v2 contract resolution - storage proof", func(t *testing.T) { + sce, basis := spendableSiacoinUTXOs(t) + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + HostOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sce[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sce[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract proof window + mineBlock(int(fc.ProofHeight-cm.Tip().Height), types.VoidAddress) + + // this is even more annoying because we have to keep the file contract + // proof and the chain index proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { + cau.UpdateElementProof(&fce.StateElement) + } + // get the proof index element + indexElement := applied[len(applied)-1].ChainIndexElement() + + resolutionTxn := types.V2Transaction{ + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &types.V2StorageProof{ + ProofIndex: indexElement, + // proof is nil since there's no data + }, + }, + }, + } + + // broadcast the expire resolution + if _, err := cm.AddV2PoolTransactions(cm.Tip(), []types.V2Transaction{resolutionTxn}); err != nil { + t.Fatal(err) + } + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2HostOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + }) + + t.Run("v2 contract resolution - renewal", func(t *testing.T) { + sces, basis := spendableSiacoinUTXOs(t) + + // using the UnlockConditions policy for brevity + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a storage contract + renterPayout := types.Siacoins(10000) + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: addr, + Value: renterPayout, + }, + HostOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: types.ZeroCurrency, + }, + ProofHeight: cm.TipState().Index.Height + 10, + ExpirationHeight: cm.TipState().Index.Height + 20, + + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + contractValue := renterPayout.Add(cm.TipState().V2FileContractTax(fc)) + sigHash := cm.TipState().ContractSigHash(fc) + sig := pk.SignHash(sigHash) + fc.RenterSignature = sig + fc.HostSignature = sig + + // create a transaction with the contract + txn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(contractValue)}, + }, + } + sigHash = cm.TipState().InputSigHash(txn) + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // current tip + tip := cm.Tip() + // mine until the contract proof window + mineBlock(1, types.VoidAddress) + + // this is even more annoying because we have to keep the file contract + // proof and the chain index proof up to date. + _, applied, err := cm.UpdatesSince(tip, 1000) + if err != nil { + t.Fatal(err) + } + + // get the confirmed file contract element + fce := applied[0].V2FileContractElementDiffs()[0].V2FileContractElement + for _, cau := range applied[1:] { + cau.UpdateElementProof(&fce.StateElement) + } + + // create a renewal + renewal := types.V2FileContractRenewal{ + FinalHostOutput: fc.HostOutput, + FinalRenterOutput: fc.RenterOutput, + NewContract: types.V2FileContract{ + RenterOutput: fc.RenterOutput, + ProofHeight: fc.ProofHeight + 10, + ExpirationHeight: fc.ExpirationHeight + 10, + + RenterPublicKey: fc.RenterPublicKey, + HostPublicKey: fc.HostPublicKey, + }, + } + + renewalSigHash := cm.TipState().RenewalSigHash(renewal) + renewalSig := pk.SignHash(renewalSigHash) + renewal.RenterSignature = renewalSig + renewal.HostSignature = renewalSig + contractSigHash := cm.TipState().ContractSigHash(renewal.NewContract) + renewal.NewContract.RenterSignature = pk.SignHash(contractSigHash) + renewal.NewContract.HostSignature = pk.SignHash(contractSigHash) + + sces, basis = spendableSiacoinUTXOs(t) + newContractValue := renterPayout.Add(cm.TipState().V2FileContractTax(renewal.NewContract)) + + // create the renewal transaction + resolutionTxn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{ + { + Parent: sces[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + }, + }, + SiacoinOutputs: []types.SiacoinOutput{ + {Address: addr, Value: sces[0].SiacoinOutput.Value.Sub(newContractValue)}, + }, + FileContractResolutions: []types.V2FileContractResolution{ + { + Parent: fce, + Resolution: &renewal, + }, + }, + } + resolutionTxnSigHash := cm.TipState().InputSigHash(resolutionTxn) + resolutionTxn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(resolutionTxnSigHash)} + + // broadcast the renewal + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{resolutionTxn}); err != nil { + t.Fatal(err) + } + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(types.FileContractID(fce.ID).V2RenterOutputID()), wallet.EventTypeV2ContractResolution, renterPayout, types.ZeroCurrency, cm.Tip().Height+144) + }) + + t.Run("siafund claim", func(t *testing.T) { + sfe, basis, err := wm.AddressSiafundOutputs(addr, 0, 100) + if err != nil { + t.Fatal(err) + } else if basis != cm.Tip() { + t.Fatalf("expected basis to be the current tip") + } + + policy := types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(pk.PublicKey())), + } + + // create a transaction + txn := types.V2Transaction{ + SiafundInputs: []types.V2SiafundInput{ + { + Parent: sfe[0], + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: policy, + }, + ClaimAddress: addr, + }, + }, + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: sfe[0].SiafundOutput.Value}, + }, + } + sigHash := cm.TipState().InputSigHash(txn) + txn.SiafundInputs[0].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + claimValue := cm.TipState().SiafundTaxRevenue + + // broadcast the transaction + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + // mine a block to confirm the transaction + mineBlock(1, types.VoidAddress) + assertEvent(t, types.Hash256(types.SiafundOutputID(sfe[0].ID).V2ClaimOutputID()), wallet.EventTypeSiafundClaim, claimValue, types.ZeroCurrency, cm.Tip().Height+144) + }) +} + +func TestSiafundClaims(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesis := testutil.Network() + // send the siafunds to the owned address + genesis.Transactions[0].SiafundOutputs[0].Address = addr + siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value + + store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + uc := types.StandardUnlockConditions(pk.PublicKey()) + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &types.SpendPolicy{ + Type: types.PolicyTypeUnlockConditions(uc), + }, + }) + if err != nil { + t.Fatal(err) + } + + // rescan to index the genesis block + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // claim the siafunds. Since tax revenue is 0, no claim event or utxo should be indexed. + siafunds, _, change, err := wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn := types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: addr, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + cs := cm.TipState() + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(txn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(siacoins) != 0 { + t.Fatalf("expected no siacoin outputs, got %v", siacoins) + } + + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { // airdrop + siafund transaction + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // fund the wallet with some siacoins + testutil.MineBlocks(t, cm, addr, 5) + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + payout := types.Siacoins(100000) + fundAmount := taxAdjustedPayout(payout) + expectedTaxRevenue := fundAmount.Sub(payout) + fc := types.FileContract{ + UnlockHash: addr, + Payout: fundAmount, + ValidProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + MissedProofOutputs: []types.SiacoinOutput{ + {Address: types.VoidAddress, Value: payout}, + }, + WindowStart: cm.Tip().Height + 10, + WindowEnd: cm.Tip().Height + 20, + } + + fcTxn := types.Transaction{ + FileContracts: []types.FileContract{fc}, + } + + siacoins, _, scChange, err := wm.SelectSiacoinElements(w.ID, fundAmount, false) + if err != nil { + t.Fatal(err) + } + + if !scChange.IsZero() { + fcTxn.SiacoinOutputs = append(fcTxn.SiacoinOutputs, types.SiacoinOutput{ + Address: addr, + Value: scChange, + }) + } + + for _, sce := range siacoins { + fcTxn.SiacoinInputs = append(fcTxn.SiacoinInputs, types.SiacoinInput{ + ParentID: sce.ID, + UnlockConditions: uc, + }) + fcTxn.Signatures = append(fcTxn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sce.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + + cs = cm.TipState() + for i, sig := range fcTxn.Signatures { + sigHash := cs.WholeSigHash(fcTxn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + fcTxn.Signatures[i].Signature = sig[:] + } + + if _, err := cm.AddPoolTransactions([]types.Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + cs = cm.TipState() + if !cs.SiafundTaxRevenue.Equals(expectedTaxRevenue) { + t.Fatalf("expected %v tax revenue, got %v", expectedTaxRevenue, cs.SiafundTaxRevenue) + } + + // claim the siafunds again. A claim event should be created to account for the + // tax revenue. + siafunds, _, change, err = wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn = types.Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.SiafundInput{ + ParentID: sfe.ID, + UnlockConditions: uc, + ClaimAddress: addr, + }) + txn.Signatures = append(txn.Signatures, types.TransactionSignature{ + ParentID: types.Hash256(sfe.ID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }) + } + cs = cm.TipState() + for i, sig := range txn.Signatures { + sigHash := cs.WholeSigHash(txn, sig.ParentID, 0, 0, nil) + sig := pk.SignHash(sigHash) + txn.Signatures[i].Signature = sig[:] + } + if _, err := cm.AddPoolTransactions([]types.Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 10 { // airdrop + 2x siafund transaction + 5x miner payouts + 1x file contract + 1x siafund claim + t.Fatalf("expected 10 events, got %v", len(events)) + } + + // check the siafund claim event + expectedID := txn.SiafundInputs[0].ParentID.ClaimOutputID() + claimEvent := events[0] + switch { + case claimEvent.ID != types.Hash256(expectedID): + t.Fatalf("expected siafund claim output %q, got %q", expectedID, claimEvent.ID) + case claimEvent.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected siafund claim event, got %v", claimEvent.Type) + case !claimEvent.SiacoinInflow().Equals(expectedTaxRevenue): + t.Fatalf("expected %v tax revenue, got %v", expectedTaxRevenue, claimEvent.SiacoinInflow()) + case !claimEvent.SiacoinOutflow().IsZero(): + t.Fatalf("expected no outflow, got %v", claimEvent.SiacoinOutflow()) + } + + // mine until the siafund claim output is mature + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + // check that the output is now spendable + siacoins, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + for _, sce := range siacoins { + if sce.ID == expectedID && sce.SiacoinOutput.Value.Equals(expectedTaxRevenue) { + return + } + } + t.Fatalf("expected siafund claim output %q with value %v not found", expectedID, expectedTaxRevenue) +} + +func TestV2SiafundClaims(t *testing.T) { + pk := types.GeneratePrivateKey() + addr := types.StandardAddress(pk.PublicKey()) + + log := zaptest.NewLogger(t) + dir := t.TempDir() + db, err := sqlite.OpenDatabase(filepath.Join(dir, "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(dir, "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + network, genesis := testutil.V2Network() + // send the siafunds to the owned address + genesis.Transactions[0].SiafundOutputs[0].Address = addr + siafundValue := genesis.Transactions[0].SiafundOutputs[0].Value + + store, genesisState, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, genesisState) + + wm, err := wallet.NewManager(cm, db, wallet.WithLogger(log.Named("wallet"))) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + // activate the v2 hardfork + testutil.MineBlocks(t, cm, types.VoidAddress, 2) + + w, err := wm.AddWallet(wallet.Wallet{Name: "test"}) + if err != nil { + t.Fatal(err) + } + + sp := types.SpendPolicy{ + Type: types.PolicyTypePublicKey(pk.PublicKey()), + } + err = wm.AddAddress(w.ID, wallet.Address{ + Address: addr, + SpendPolicy: &sp, + }) + if err != nil { + t.Fatal(err) + } + + // rescan to index the genesis block + if err := wm.Scan(context.Background(), types.ChainIndex{}); err != nil { + t.Fatal(err) + } + + // claim the siafunds. Since tax revenue is 0, no claim event or utxo should be indexed. + siafunds, basis, change, err := wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn := types.V2Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ + Parent: sfe, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + ClaimAddress: addr, + }) + } + cs := cm.TipState() + sigHash := cs.InputSigHash(txn) + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + siacoins, _, err := wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(siacoins) != 0 { + t.Fatalf("expected no siacoin outputs, got %v", siacoins) + } + + events, err := wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 2 { // airdrop + siafund transaction + t.Fatalf("expected 2 events, got %v", len(events)) + } + + // fund the wallet with some siacoins + testutil.MineBlocks(t, cm, addr, 5) + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + payout := types.Siacoins(100000) + cs = cm.TipState() + fc := types.V2FileContract{ + RenterOutput: types.SiacoinOutput{ + Address: types.VoidAddress, + Value: payout, + }, + ProofHeight: cs.Index.Height + 10, + ExpirationHeight: cs.Index.Height + 20, + RenterPublicKey: pk.PublicKey(), + HostPublicKey: pk.PublicKey(), + } + sigHash = cs.ContractSigHash(fc) + fc.RenterSignature = pk.SignHash(sigHash) + fc.HostSignature = pk.SignHash(sigHash) + + expectedTax := cs.V2FileContractTax(fc) + fundAmount := payout.Add(expectedTax) + + fcTxn := types.V2Transaction{ + FileContracts: []types.V2FileContract{fc}, + } + + siacoins, basis, scChange, err := wm.SelectSiacoinElements(w.ID, fundAmount, false) + if err != nil { + t.Fatal(err) + } + + if !scChange.IsZero() { + fcTxn.SiacoinOutputs = append(fcTxn.SiacoinOutputs, types.SiacoinOutput{ + Address: addr, + Value: scChange, + }) + } + + for _, sce := range siacoins { + fcTxn.SiacoinInputs = append(fcTxn.SiacoinInputs, types.V2SiacoinInput{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: sp, + }, + }) + } + + sigHash = cs.InputSigHash(fcTxn) + for i := range fcTxn.SiacoinInputs { + fcTxn.SiacoinInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{fcTxn}); err != nil { + t.Fatal(err) + } + + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + cs = cm.TipState() + if !cs.SiafundTaxRevenue.Equals(expectedTax) { + t.Fatalf("expected %v tax revenue, got %v", expectedTax, cs.SiafundTaxRevenue) + } + + // claim the siafunds again. A claim event should be created to account for the + // tax revenue. + siafunds, basis, change, err = wm.SelectSiafundElements(w.ID, siafundValue) + if err != nil { + t.Fatal(err) + } else if change != 0 { + t.Fatalf("expected no change, got %v", change) + } + txn = types.V2Transaction{ + SiafundOutputs: []types.SiafundOutput{ + {Address: addr, Value: siafundValue}, + }, + } + for _, sfe := range siafunds { + txn.SiafundInputs = append(txn.SiafundInputs, types.V2SiafundInput{ + Parent: sfe, + SatisfiedPolicy: types.SatisfiedPolicy{Policy: sp}, + ClaimAddress: addr, + }) + } + + cs = cm.TipState() + sigHash = cs.InputSigHash(txn) + for i := range txn.SiafundInputs { + txn.SiafundInputs[i].SatisfiedPolicy.Signatures = []types.Signature{pk.SignHash(sigHash)} + } + if _, err := cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}); err != nil { + t.Fatal(err) + } + testutil.MineBlocks(t, cm, types.VoidAddress, 1) + waitForBlock(t, cm, db) + + events, err = wm.WalletEvents(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } else if len(events) != 10 { // airdrop + 2x siafund transaction + 5x miner payouts + 1x file contract + 1x siafund claim + t.Fatalf("expected 10 events, got %v", len(events)) + } + + // check the siafund claim event + expectedID := txn.SiafundInputs[0].Parent.ID.V2ClaimOutputID() + claimEvent := events[0] + switch { + case claimEvent.ID != types.Hash256(expectedID): + t.Fatalf("expected siafund claim output %q, got %q", expectedID, claimEvent.ID) + case claimEvent.Type != wallet.EventTypeSiafundClaim: + t.Fatalf("expected siafund claim event, got %v", claimEvent.Type) + case !claimEvent.SiacoinInflow().Equals(expectedTax): + t.Fatalf("expected %v tax revenue, got %v", expectedTax, claimEvent.SiacoinInflow()) + case !claimEvent.SiacoinOutflow().IsZero(): + t.Fatalf("expected no outflow, got %v", claimEvent.SiacoinOutflow()) + } + + // mine until the siafund claim output is mature + testutil.MineBlocks(t, cm, types.VoidAddress, int(network.MaturityDelay)) + waitForBlock(t, cm, db) + + // check that the output is now spendable + siacoins, _, err = wm.UnspentSiacoinOutputs(w.ID, 0, 100) + if err != nil { + t.Fatal(err) + } + for _, sce := range siacoins { + if sce.ID == expectedID && sce.SiacoinOutput.Value.Equals(expectedTax) { + return + } + } + t.Fatalf("expected siafund claim output %q with value %v not found", expectedID, expectedTax) +} + +func TestReset(t *testing.T) { + log := zaptest.NewLogger(t) + + pk := types.GeneratePrivateKey() + addr := types.StandardUnlockHash(pk.PublicKey()) + + network, genesisBlock := testutil.Network() + // send the siafunds to the owned address + genesisBlock.Transactions[0].SiafundOutputs[0].Address = addr + + bdb, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus.db")) + if err != nil { + t.Fatal(err) + } + defer bdb.Close() + + store, genesisState, err := chain.NewDBStore(bdb, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm1 := chain.NewManager(store, genesisState) + + bdb2, err := coreutils.OpenBoltChainDB(filepath.Join(t.TempDir(), "consensus2.db")) + if err != nil { + t.Fatal(err) + } + defer bdb2.Close() + store2, genesisState2, err := chain.NewDBStore(bdb2, network, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm2 := chain.NewManager(store2, genesisState2) + + // mine blocks before starting the wallet manager + for i := 0; i < 25; i++ { + // blocks on the first chain manager go to the void + b1, ok := coreutils.MineBlock(cm1, types.VoidAddress, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm1.AddBlocks([]types.Block{b1}); err != nil { + t.Fatal(err) + } + + // blocks on the second one go to the primary address + b2, ok := coreutils.MineBlock(cm2, addr, 15*time.Second) + if !ok { + t.Fatal("failed to mine block") + } else if err := cm2.AddBlocks([]types.Block{b2}); err != nil { + t.Fatal(err) + } + } + + db, err := sqlite.OpenDatabase(filepath.Join(t.TempDir(), "walletd.sqlite3"), log.Named("sqlite3")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // wait for the manager to sync to the first chain + wm, err := wallet.NewManager(cm1, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm1, db) + + assertBalance := func(t *testing.T, addr types.Address, siacoin, immature types.Currency, siafund uint64) { + t.Helper() + + balance, err := db.AddressBalance(addr) + if err != nil { + t.Fatal(err) + } + switch { + case !balance.Siacoins.Equals(siacoin): + t.Fatalf("expected %v SC, got %v", siacoin, balance.Siacoins) + case !balance.ImmatureSiacoins.Equals(immature): + t.Fatalf("expected immature %v SC, got %v", siacoin, balance.Siacoins) + case balance.Siafunds != siafund: + t.Fatalf("expected %v siafunds, got %v", siafund, balance.Siafunds) + } + } + + assertBalance(t, addr, types.ZeroCurrency, types.ZeroCurrency, 10000) + + // close the manager + if err := wm.Close(); err != nil { + t.Fatal() + } + + // calculate the expected balances + _, applied, err := cm2.UpdatesSince(types.ChainIndex{}, 1000) + if err != nil { + t.Fatal(err) + } + + var siacoinElements []types.SiacoinElement + for _, cau := range applied { + for _, sced := range cau.SiacoinElementDiffs() { + if sced.Created && sced.SiacoinElement.SiacoinOutput.Address == addr { + siacoinElements = append(siacoinElements, sced.SiacoinElement) + } + } + } + + var expectedSiacoins, expectedImmature types.Currency + for _, sce := range siacoinElements { + if sce.MaturityHeight > cm2.Tip().Height { + expectedImmature = expectedImmature.Add(sce.SiacoinOutput.Value) + } else { + expectedSiacoins = expectedSiacoins.Add(sce.SiacoinOutput.Value) + } + } + + wm, err = wallet.NewManager(cm2, db, wallet.WithLogger(log.Named("wallet")), wallet.WithIndexMode(wallet.IndexModeFull)) + if err != nil { + t.Fatal(err) + } + defer wm.Close() + + waitForBlock(t, cm2, db) + + assertBalance(t, addr, expectedSiacoins, expectedImmature, genesisState.SiafundCount()) +}