Cosmos delegators

.gitignore

@@ -20,3 +20,4 @@
 .idea/
 .vscode/
 **.DS_Store
+cmd/arkeodrop/__debug_bin

cli/cosmos.go

@@ -0,0 +1,41 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/ArkeoNetwork/airdrop/indexer"
+	"github.com/ArkeoNetwork/common/utils"
+	"github.com/spf13/cobra"
+)
+
+var (
+	indexDelegatorsCmd = &cobra.Command{
+		Use:   "delegators",
+		Short: "gather cosmos-sdk chain data store in our db",
+		Run:   runDelegatorsIndexer,
+		Args:  cobra.ExactValidArgs(1),
+	}
+)
+
+func runDelegatorsIndexer(cmd *cobra.Command, args []string) {
+	flags := cmd.InheritedFlags()
+	envPath, _ := flags.GetString("env")
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		fmt.Print("error: no config loaded")
+		return
+	}
+
+	chain := args[0]
+	params := indexer.CosmosIndexerParams{Chain: chain, DB: *c}
+	indxr, err := indexer.NewCosmosIndexer(params)
+	if err != nil {
+		cmd.PrintErrf("error creating cosmos indexer: %+v", err)
+		return
+	}
+
+	if err := indxr.IndexDelegators(); err != nil {
+		fmt.Printf("error indexing delegators: %+v", err)
+		return
+	}
+}

cli/export.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/ArkeoNetwork/airdrop/pkg/db"
+	"github.com/ArkeoNetwork/common/utils"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -31,32 +32,24 @@ func runExport(cmd *cobra.Command, args []string) {
 	log.Infof("starting export process for %s, %s", args[0], args[1])
 	flags := cmd.InheritedFlags()
 	envPath, _ := flags.GetString("env")
-	c := readConfig(envPath)
-
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErrln("db config undefined")
+		return
+	}
 	flags = cmd.Flags()
 	fileName, _ := flags.GetString("output")
 	if fileName == "" {
 		fileName = fmt.Sprintf("/tmp/airdrop_%s_%s.csv", args[0], args[1])
 	}
 
-	params := db.DBConfig{
-		Host:         c.DBHost,
-		Port:         c.DBPort,
-		User:         c.DBUser,
-		Pass:         c.DBPass,
-		DBName:       c.DBName,
-		PoolMaxConns: c.DBPoolMaxConns,
-		PoolMinConns: c.DBPoolMinConns,
-		SSLMode:      c.DBSSLMode,
-	}
-
-	err := exportWeightedTokenAvgs(params, args[0], args[1], fileName)
+	err := exportWeightedTokenAvgs(*c, args[0], args[1], fileName)
 	if err != nil {
 		log.Errorf("error exporting: %+v", err)
 	}
 }
 
-func exportWeightedTokenAvgs(dbConfig db.DBConfig, chain, token, fileName string) error {
+func exportWeightedTokenAvgs(dbConfig utils.DBConfig, chain, token, fileName string) error {
 	d, err := db.New(dbConfig)
 	if err != nil {
 		return errors.Wrapf(err, "error connecting to the db")

cli/indexer.go

@@ -2,14 +2,12 @@ package cli
 
 import (
 	"github.com/ArkeoNetwork/airdrop/indexer"
-	"github.com/ArkeoNetwork/airdrop/pkg/db"
-	"github.com/ArkeoNetwork/directory/pkg/logging"
+	"github.com/ArkeoNetwork/common/logging"
+	"github.com/ArkeoNetwork/common/utils"
 	"github.com/spf13/cobra"
 )
 
 type Config struct {
-	SnapshotStart  uint64 `mapstructure:"SNAPSHOT_START"`
-	SnapshotEnd    uint64 `mapstructure:"SNAPSHOT_END"`
 	DBHost         string `mapstructure:"DB_HOST"`
 	DBPort         uint   `mapstructure:"DB_PORT"`
 	DBUser         string `mapstructure:"DB_USER"`
@@ -40,22 +38,16 @@ func runIndexer(cmd *cobra.Command, args []string) {
 	log.Info("starting data generation process")
 	flags := cmd.InheritedFlags()
 	envPath, _ := flags.GetString("env")
-	c := readConfig(envPath)
+	c := utils.ReadDBConfig(envPath)
+	if c == nil {
+		cmd.PrintErr("db config undefined")
+		return
+	}
 
 	indexerApp := indexer.NewIndexer(indexer.IndexerAppParams{
-		SnapshotStart: c.SnapshotStart,
-		SnapshotEnd:   c.SnapshotEnd,
-		DBConfig: db.DBConfig{
-			Host:         c.DBHost,
-			Port:         c.DBPort,
-			User:         c.DBUser,
-			Pass:         c.DBPass,
-			DBName:       c.DBName,
-			PoolMaxConns: c.DBPoolMaxConns,
-			PoolMinConns: c.DBPoolMinConns,
-			SSLMode:      c.DBSSLMode,
-		},
+		DBConfig: *c,
 	})
+
 	indexerApp.Start()
 	log.Debug("finished data generation process")
 }

cli/root.go

@@ -26,12 +26,19 @@ var (
 		Short: "gather chain data store in our db",
 		Run:   runIndexer,
 	}
+
+	indexCosmosCmd = &cobra.Command{
+		Use:   "index-cosmos",
+		Short: "cosmos-sdk indexing",
+	}
 )
 
 func init() {
 	rootCmd.PersistentFlags().StringP("env", "e", "docker/dev/docker.env", "env file to source")
 	rootCmd.AddCommand(versionCmd)
 	rootCmd.AddCommand(indexCmd)
+	indexCosmosCmd.AddCommand(indexDelegatorsCmd)
+	rootCmd.AddCommand(indexCosmosCmd)
 	rootCmd.AddCommand(exportCmd)
 }
 

cmd/indexer/main.go

@@ -4,9 +4,8 @@ import (
 	"flag"
 
 	"github.com/ArkeoNetwork/airdrop/indexer"
-	"github.com/ArkeoNetwork/airdrop/pkg/db"
-	"github.com/ArkeoNetwork/directory/pkg/config"
-	"github.com/ArkeoNetwork/directory/pkg/logging"
+	"github.com/ArkeoNetwork/common/logging"
+	"github.com/ArkeoNetwork/common/utils"
 )
 
 type Config struct {
@@ -44,28 +43,24 @@ func main() {
 	flag.Parse()
 	c := &Config{}
 	if *envPath == "" {
-		if err := config.LoadFromEnv(c, configNames...); err != nil {
+		if err := utils.LoadFromEnv(c, configNames...); err != nil {
 			log.Panicf("failed to load config from env: %+v", err)
 		}
 	} else {
-		if err := config.Load(*envPath, c); err != nil {
+		if err := utils.Load(*envPath, c); err != nil {
 			log.Panicf("failed to load config: %+v", err)
 		}
 	}
 
+	dbc := utils.ReadDBConfig(*envPath)
+	if dbc == nil {
+		log.Error("db config undefined")
+		return
+	}
 	indexerApp := indexer.NewIndexer(indexer.IndexerAppParams{
 		SnapshotStart: c.SnapshotStart,
 		SnapshotEnd:   c.SnapshotEnd,
-		DBConfig: db.DBConfig{
-			Host:         c.DBHost,
-			Port:         c.DBPort,
-			User:         c.DBUser,
-			Pass:         c.DBPass,
-			DBName:       c.DBName,
-			PoolMaxConns: c.DBPoolMaxConns,
-			PoolMinConns: c.DBPoolMinConns,
-			SSLMode:      c.DBSSLMode,
-		},
+		DBConfig:      *dbc,
 	})
 	indexerApp.Start()
 	log.Debug("finished data generation process")

db/007_chains_decimals.sql

@@ -0,0 +1,5 @@
+
+alter table chains add column decimals integer;
+---- create above / drop below ----
+-- undo --
+alter table chains drop column decimals;

db/008_cosmos_staking.sql

@@ -0,0 +1,22 @@
+create table cosmos_staking_events
+(
+    id           bigserial not null
+                    constraint cosmos_staking_events_pk
+                        primary key,
+    created      timestamptz default now() not null,
+    updated      timestamptz default now() not null,
+    chain        text not null,
+    event_type   text not null,
+    delegator    text not null,
+    validator    text not null,
+    amount       numeric not null,
+    block_number numeric not null check ( block_number >= 0 ),
+    txhash       text not null,
+    event_index  integer not null,
+    constraint cosmos_staking_events_unique
+        unique (chain,txhash,event_index,validator,delegator)
+);
+
+---- create above / drop below ----
+-- undo --
+drop table cosmos_staking_events;

db/009_cosmos1.sql

@@ -0,0 +1,5 @@
+insert into chains(name,rpc_url,snapshot_start_block,snapshot_end_block,decimals)
+values ('GAIA','https://daemon.cosmos.shapeshift.com:443/rpc',13050670,13714445,6);
+
+---- create above / drop below ----
+delete from chains where name = 'GAIA';

docker/dev/docker.env

@@ -3,7 +3,7 @@ SNAPSHOT_END=1672268704
 
 # db
 DB_HOST="localhost"
-DB_PORT="5432"
+DB_PORT="5439"
 DB_USER="arkeo"
 DB_PASS="arkeo123"
 DB_NAME="arkeo_airdrop"

go.mod

@@ -3,14 +3,27 @@ module github.com/ArkeoNetwork/airdrop
 go 1.19
 
 require (
-	github.com/ArkeoNetwork/directory v0.0.0-20221226175704-8f01136fa856
+	github.com/ArkeoNetwork/common v0.0.1
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/jackc/pgx/v4 v4.17.2
 	github.com/spf13/cobra v1.5.0
+	github.com/stretchr/testify v1.8.0
+	github.com/tendermint/tendermint v0.34.22
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd v0.22.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cosmos/gorocksdb v1.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/gogo/protobuf v1.3.3 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -21,6 +34,21 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jackc/pgtype v1.12.0 // indirect
 	github.com/jackc/puddle v1.3.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.12.2 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.34.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/sasha-s/go-deadlock v0.3.1 // indirect
+	golang.org/x/net v0.3.0 // indirect
+	google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959 // indirect
+	google.golang.org/grpc v1.50.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )
 
 require (
@@ -30,7 +58,7 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/georgysavva/scany v1.2.1
-	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
@@ -50,8 +78,8 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.13.0 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
-	github.com/tklauser/go-sysconf v0.3.5 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect
@@ -61,3 +89,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools v2.2.0+incompatible
 )
+
+replace (
+	github.com/ArkeoNetwork/common => /Users/adamsamere/chaintech/oss/arkeo/common
+	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
+)