Better job retry, code quality

.changeset/lemon-gifts-check.md

@@ -0,0 +1,13 @@
+---
+'magic-supplier': patch
+---
+
+Better backoff and retry logic for some worker jobs:
+
+- The balances job runs less often (now every 5 minutes)
+- The `finalize` jobs retry every 10 minutes, and retry for more instances, to account for expected block times and confirmation delays
+
+Various small fixes regarding code quality:
+
+- Instead of checking for `undefined` with `typeof val === 'undefined'`, added `isNullish` and `isNotNullish` helpers that check against `null` and `undefined`
+- Removed unused imports and variables

.eslintrc.js

@@ -1,23 +1,29 @@
 module.exports = {
-  extends: ["@stacks/eslint-config"],
+  extends: ['@stacks/eslint-config'],
   settings: {
     react: {
-      version: "999.999.999",
+      version: '999.999.999',
     },
   },
+  plugins: ['unused-imports'],
   parserOptions: {
-    project: "tsconfig.json",
+    project: 'tsconfig.json',
   },
   rules: {
-    "@typescript-eslint/no-unused-vars": [0],
-    "@typescript-eslint/explicit-module-boundary-types": [0],
-    "@typescript-eslint/no-non-null-assertion": [0],
-    "@typescript-eslint/strict-boolean-expressions": [
+    '@typescript-eslint/no-unused-vars': [0],
+    '@typescript-eslint/explicit-module-boundary-types': [0],
+    '@typescript-eslint/no-non-null-assertion': [0],
+    '@typescript-eslint/strict-boolean-expressions': [
       2,
       {
         allowNullableString: true,
         allowNullableBoolean: true,
       },
     ],
+    'unused-imports/no-unused-imports': 'error',
+    'unused-imports/no-unused-vars': [
+      'error',
+      { vars: 'all', varsIgnorePattern: '^_', args: 'after-used', argsIgnorePattern: '^_' },
+    ],
   },
 };

.gitignore

@@ -39,4 +39,7 @@ yarn-error.log*
 .yalc
 
 settings/Testnet.toml
-scripts/migrate-redis.ts
+scripts/migrate-redis.ts
+scripts/accounting.ts
+
+tmpdata

package.json

@@ -71,6 +71,7 @@
     "app.json": "^1.3.0",
     "eslint": "7",
     "eslint-plugin-prettier": "^3.1.4",
+    "eslint-plugin-unused-imports": "2.0.0",
     "jest": "^27.0.3",
     "jest-fetch-mock": "3.0.3",
     "nodemon": "^2.0.15",

scripts/analytics.ts

@@ -0,0 +1,131 @@
+import 'cross-fetch/polyfill';
+import { getContractEventsUntil } from '../src/stacks-api';
+import { writeFile, readFile } from 'fs/promises';
+import {
+  isFinalizeInboundEvent,
+  isEscrowEvent,
+  isInitiateOutboundEvent,
+  isFinalizeOutboundEvent,
+  isRevokeInboundEvent,
+  isRevokeOutboundEvent,
+} from '../src/events';
+import { resolve } from 'path';
+import { satsToBtc } from '../src/utils';
+import { bytesToHex } from 'micro-stacks/common';
+import { parseEventsJSON, stringifyEvents } from './helpers';
+
+const [command] = process.argv.slice(2);
+
+async function run() {
+  const dataFile = resolve(__dirname, '../tmpdata/events.json');
+  // const totalInbound =
+  if (command === 'write') {
+    const events = await getContractEventsUntil(null);
+    console.log(`Found ${events.length} events`);
+    const json = stringifyEvents(events);
+    await writeFile(dataFile, json, {
+      encoding: 'utf-8',
+    });
+    return;
+  } else if (command === 'pending') {
+    const eventsJSON = await readFile(dataFile, { encoding: 'utf-8' });
+    const events = parseEventsJSON(eventsJSON);
+    events.reverse();
+
+    // const ins: string[] = [];
+    const ins = new Set<string>();
+    const outs = new Set<bigint>();
+    events.forEach(event => {
+      if (isEscrowEvent(event)) {
+        console.log('start', event.print.swapper, bytesToHex(event.print.txid));
+        // console.log(event.print);
+        ins.add(bytesToHex(event.print.txid));
+      } else if (isInitiateOutboundEvent(event)) {
+        console.log('start', event.print.swapId);
+        outs.add(event.print.swapId);
+      } else if (isFinalizeInboundEvent(event)) {
+        // console.log('end', bytesToHex(event.print.txid));
+        ins.delete(bytesToHex(event.print.txid));
+      } else if (isFinalizeOutboundEvent(event)) {
+        // console.log('end', event.print.swapId);
+        outs.delete(event.print.swapId);
+        console.log('finalized', event.print.swapId);
+      } else if (isRevokeInboundEvent(event)) {
+        ins.delete(bytesToHex(event.print.txid));
+      } else if (isRevokeOutboundEvent(event)) {
+        outs.delete(event.print.swapId);
+      }
+    });
+
+    ins.forEach(txid => {
+      console.log(txid);
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+      // const event = events.find(e => (e as any).print.txid === hexToBytes(txid));
+      const event = events.find(e => {
+        if (isEscrowEvent(e)) {
+          console.log(bytesToHex(e.print.txid));
+          return bytesToHex(e.print.txid) === txid;
+        }
+        return false;
+      });
+      console.log(event);
+    });
+
+    console.log(ins);
+    console.log(outs);
+
+    return;
+  }
+  const eventsJSON = await readFile(dataFile, { encoding: 'utf-8' });
+  const events = parseEventsJSON(eventsJSON);
+  console.log('Found events', events.length);
+  let totalInbound = 0n;
+  let totalOutbound = 0n;
+  let totalFees = 0n;
+  let swapsCount = 0;
+
+  events.forEach(event => {
+    if (isEscrowEvent(event)) {
+      totalInbound += event.print.sats;
+      totalFees += event.print.sats - event.print.xbtc;
+      swapsCount += 1;
+    } else if (isInitiateOutboundEvent(event)) {
+      totalOutbound += event.print.xbtc;
+      totalFees += event.print.xbtc - event.print.sats;
+      swapsCount += 1;
+    }
+  });
+
+  const total = totalInbound + totalOutbound;
+
+  console.log('Total volume:', satsToBtc(total));
+  console.log('Total inbound:', satsToBtc(totalInbound));
+  console.log('Total outbound:', satsToBtc(totalOutbound));
+  console.log('Total fees:', satsToBtc(totalFees));
+  const avgFee = Number(totalFees) / swapsCount;
+  console.log('Average fee:', satsToBtc(avgFee));
+  console.log('Average fee %:', ((Number(totalFees) / Number(total)) * 100).toFixed(3));
+  console.log('Total swaps:', swapsCount);
+  // console.log('Avg fee:', ((swapsCount / Number(totalFees)) * 100).toFixed(3));
+
+  //   // const api = getQueryApi();
+
+  //   // if (api) {
+  //   //   const lines = await api.collectLines('SELECT "ping" FROM "testPing" limit 1');
+  //   //   console.log(lines.length, 'lines');
+  //   // }
+
+  //   // events.forEach(event => {
+  //   //   if (isFinalizeInboundEvent(event)) {
+  //   //     event.
+  //   //   }
+  //   //   // event.
+  //   // });
+  // }
+}
+
+run()
+  .catch(console.error)
+  .finally(() => {
+    process.exit();
+  });

scripts/consolidate.ts

@@ -1,7 +1,7 @@
 import { getFeeRate, listUnspent, txWeight, withElectrumClient } from '../src/wallet';
 import { Psbt } from 'bitcoinjs-lib';
 import { getBtcPayment, getBtcNetwork, getBtcSigner } from '../src/config';
-import { btcToSats, getBtcTxUrl, satsToBtc } from '../src/utils';
+import { getBtcTxUrl, satsToBtc } from '../src/utils';
 import { prompt } from 'inquirer';
 
 async function run() {

scripts/finalize-inbound.ts

@@ -1,11 +1,8 @@
 import 'cross-fetch/polyfill';
 import { hexToBytes } from 'micro-stacks/common';
-import { FinalizeInboundPrint } from '../src/events';
-import { processFinalizedInbound, redeem } from '../src/processors/redeem-htlc';
+import { redeem } from '../src/processors/redeem-htlc';
 import { bridgeContract, stacksProvider } from '../src/stacks';
-import { createRedisClient } from '../src/store';
 import { getBtcTxUrl } from '../src/utils';
-import { deserializeJob } from '../src/worker/jobs';
 
 const [txidHex] = process.argv.slice(2);
 

scripts/helpers.ts

@@ -7,7 +7,8 @@ import { stacksProvider } from '../src/stacks';
 import { getTxUrl, stxToUstx, ustxToStx } from '../src/utils';
 import { getStxBalance } from '../src/wallet';
 import { getStxNetwork, getStxPrivateKey } from '../src/config';
-import { IntegerType } from 'micro-stacks/common';
+import { bytesToHex, hexToBytes, IntegerType } from 'micro-stacks/common';
+import { Prints, Event } from '../src/events';
 
 type UnknownTx = ContractCallTyped<TypedAbiArg<unknown, string>[], unknown>;
 
@@ -81,3 +82,33 @@ export async function getFeeEstimate(tx: UnknownTx, options: Partial<ContractCal
   const fee = transaction.auth.spendingCondition.fee;
   return fee;
 }
+
+function replacer(key: string, value: any): any {
+  if (typeof value === 'bigint') {
+    return { __bigintval__: value.toString() };
+  } else if (value instanceof Uint8Array) {
+    return { __buff__: bytesToHex(value) };
+  }
+  return value;
+}
+
+function reviver(key: string, value: any): any {
+  if (value != null && typeof value === 'object' && '__bigintval__' in value) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return BigInt(value['__bigintval__']);
+  }
+  if (value != null && typeof value === 'object' && '__buff__' in value) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    return hexToBytes(value['__buff__']);
+  }
+  return value;
+}
+
+export function stringifyEvents(events: Event<Prints>[]) {
+  return JSON.stringify(events, replacer);
+}
+
+export function parseEventsJSON(json: string) {
+  const events = JSON.parse(json, reviver) as Event<Prints>[];
+  return events;
+}

scripts/register-supplier.ts

@@ -1,7 +1,7 @@
 import { prompt } from 'inquirer';
 import 'cross-fetch/polyfill';
-import { stacksProvider, bridgeContract } from '../src/stacks';
-import { bpsToPercent, btcToSats, satsToBtc, shiftInt, stxToUstx } from '../src/utils';
+import { bridgeContract } from '../src/stacks';
+import { bpsToPercent, btcToSats, satsToBtc } from '../src/utils';
 import {
   getBtcAddress,
   getNetworkKey,
@@ -13,8 +13,6 @@ import {
 import { PostConditionMode } from 'micro-stacks/transactions';
 import BigNumber from 'bignumber.js';
 import { getBalances } from '../src/wallet';
-import { AnchorMode } from 'micro-stacks/transactions';
-import { bytesToHex } from 'micro-stacks/common';
 import { askStxFee, broadcastAndLog } from './helpers';
 
 interface Answers {
@@ -28,7 +26,6 @@ interface Answers {
 }
 
 async function run() {
-  const provider = stacksProvider();
   const bridge = bridgeContract();
 
   try {

scripts/remove-funds.ts

@@ -2,7 +2,7 @@ import { prompt } from 'inquirer';
 import { PostConditionMode } from 'micro-stacks/transactions';
 import { getSupplierId } from '../src/config';
 import { stacksProvider, bridgeContract } from '../src/stacks';
-import { btcToSats, satsToBtc, shiftInt } from '../src/utils';
+import { btcToSats, satsToBtc } from '../src/utils';
 import { getStxBalance } from '../src/wallet';
 import { askStxFee, broadcastAndLog } from './helpers';
 

scripts/test-events.ts

@@ -10,7 +10,7 @@ async function run() {
       if (e.tx_id !== f.tx_id || e.event_index !== f.event_index) {
         throw new Error('Mismatch');
       }
-      const event = getPrintFromRawEvent(f);
+      getPrintFromRawEvent(f);
     })
   );
 }

src/config.ts

@@ -1,7 +1,6 @@
 import { ECPair, networks, payments } from 'bitcoinjs-lib';
 import { StacksNetworkVersion } from 'micro-stacks/crypto';
 import { StacksMainnet, StacksMocknet, StacksNetwork, StacksTestnet } from 'micro-stacks/network';
-import { getPublicKey as _getPublicKey } from 'noble-secp256k1';
 import { logger } from './logger';
 import { bridgeContract, stacksProvider } from './stacks';
 import { makeStxAddress } from './utils';

src/events.ts

@@ -1,6 +1,7 @@
 import { contracts } from './clarigen/next';
 import { hexToCvValue, TypedAbiArg, TypedAbiFunction } from '@clarigen/core';
 import { ApiEvent, getTransactionEvent } from './stacks-api';
+import { isNullish } from './utils';
 
 type ResponseType<T> = T extends TypedAbiFunction<TypedAbiArg<unknown, string>[], infer R>
   ? R
@@ -91,7 +92,7 @@ export const isRevokeOutboundEvent = (val: Event): val is Event<RevokeOutboundPr
 
 export function getEventWithPrint<T extends Prints>(prints: Prints[], topic: T['topic']): T {
   const [found] = prints.filter(p => p.topic === topic);
-  if (typeof found === 'undefined') {
+  if (isNullish(found)) {
     throw new Error(`No print with topic '${topic}'`);
   }
   return found as T;
@@ -148,7 +149,7 @@ export function getPrintFromRawEvent<T = Prints>(event: ApiEvent): Event<T> | nu
 
 export async function deserializeEvent<T>(eventData: SerializedEvent<T>): Promise<Event<T>> {
   const { index, txid } = eventData;
-  const apiEvent = await getTransactionEvent(eventData.txid, eventData.index);
+  const apiEvent = await getTransactionEvent(txid, index);
   const event = getPrintFromRawEvent<T>(apiEvent);
   if (event === null) throw new Error('Invalid event');
   return event;

src/processors/outbound.ts

@@ -1,5 +1,5 @@
 import { networks, payments } from 'bitcoinjs-lib';
-import { bytesToBigInt, bytesToHex } from 'micro-stacks/common';
+import { bytesToBigInt } from 'micro-stacks/common';
 import { getBtcNetwork, getSupplierId } from '../config';
 import {
   getSentOutbound,

src/processors/redeem-htlc.ts

@@ -1,5 +1,5 @@
 import { getBtcAddress, getBtcNetwork, getBtcSigner, getSupplierId } from '../config';
-import { networks, Psbt, script as bScript, payments, opcodes } from 'bitcoinjs-lib';
+import { Psbt, script as bScript, payments, opcodes } from 'bitcoinjs-lib';
 import { getRedeemedHTLC, setRedeemedHTLC, RedisClient } from '../store';
 import { logger as _logger } from '../logger';
 import { getFeeRate, tryBroadcast, withElectrumClient } from '../wallet';
@@ -13,7 +13,7 @@ const logger = _logger.child({ topic: 'redeemHTLC' });
 export async function processFinalizedInbound(event: Event, client: RedisClient) {
   const { print } = event;
   if (!isFinalizeInboundPrint(print)) return;
-  const { preimage, supplier } = print;
+  const { preimage } = print;
   if (print.supplier !== BigInt(getSupplierId())) return;
   const txidHex = bytesToHex(print.txid);
   const l = logger.child({