From 6b71657e5be20933bf0978c669af79add85ad8be Mon Sep 17 00:00:00 2001 From: Mohamed Sohail Date: Wed, 11 May 2022 12:32:11 +0300 Subject: [PATCH] sohail/update deps structure (#5) * refactor: syncer structure - move syncer jobs to internal dir * refactor: queries struct and pkg updates - update cic-go to latest - separate sql queries by logic * ci: add dependabot --- .github/dependabot.yml | 6 +++ cmd/cache_syncer.go | 28 -------------- cmd/init.go | 55 +++++++++------------------ cmd/main.go | 13 +++---- cmd/syncer.go | 42 +++++++++++++++++++++ cmd/token_syncer.go | 71 ----------------------------------- cmd/ussd_syncer.go | 24 ------------ dev/docker-compose.yaml | 2 +- go.mod | 24 ++++++------ go.sum | 49 ++++++++++-------------- internal/syncer/cache.go | 30 +++++++++++++++ internal/syncer/syncer.go | 24 ++++++++++++ internal/syncer/token.go | 78 +++++++++++++++++++++++++++++++++++++++ internal/syncer/ussd.go | 26 +++++++++++++ queries.sql | 41 -------------------- queries/core.sql | 41 ++++++++++++++++++++ queries/dashboard.sql | 0 17 files changed, 303 insertions(+), 251 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 cmd/cache_syncer.go create mode 100644 cmd/syncer.go delete mode 100644 cmd/token_syncer.go delete mode 100644 cmd/ussd_syncer.go create mode 100644 internal/syncer/cache.go create mode 100644 internal/syncer/syncer.go create mode 100644 internal/syncer/token.go create mode 100644 internal/syncer/ussd.go create mode 100644 queries/core.sql create mode 100644 queries/dashboard.sql diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..20e1ef1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/cmd/cache_syncer.go b/cmd/cache_syncer.go deleted file mode 100644 index 954b1d6..0000000 --- a/cmd/cache_syncer.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "context" - "github.com/georgysavva/scany/pgxscan" - "github.com/hibiken/asynq" - "github.com/rs/zerolog/log" -) - -type tableCount struct { - Count int `db:"count"` -} - -func cacheSyncer(ctx context.Context, t *asynq.Task) error { - _, err := db.Exec(ctx, queries["cache-syncer"]) - if err != nil { - return asynq.SkipRetry - } - - var count tableCount - if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from transactions"); err != nil { - return asynq.SkipRetry - } - - log.Info().Msgf("=> %d transactions synced", count.Count) - - return nil -} diff --git a/cmd/init.go b/cmd/init.go index 99f1e1a..d504d5c 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -2,8 +2,9 @@ package main import ( "context" + "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/grassrootseconomics/cic_go/cic_net" + cic_net "github.com/grassrootseconomics/cic-go/net" "github.com/hibiken/asynq" "github.com/jackc/pgx/v4/pgxpool" "github.com/knadh/koanf" @@ -11,7 +12,6 @@ import ( "github.com/knadh/koanf/providers/env" "github.com/knadh/koanf/providers/file" "github.com/nleof/goyesql" - "github.com/rs/zerolog/log" "strings" ) @@ -27,6 +27,11 @@ type config struct { Syncers map[string]string `koanf:"syncers"` } +type queries struct { + core goyesql.Queries + dashboard goyesql.Queries +} + func loadConfig(configFilePath string, k *koanf.Koanf) error { confFile := file.Provider(configFilePath) if err := k.Load(confFile, toml.Parser()); err != nil { @@ -77,45 +82,21 @@ func connectCicNet(rpcProvider string, tokenIndex common.Address) error { return nil } -func loadQueries(sqlFile string) error { - var err error - queries, err = goyesql.ParseFile(sqlFile) +func loadQueries(sqlFilesPath string) error { + coreQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/core.sql", sqlFilesPath)) if err != nil { return err } - return nil -} - -func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) { - scheduler := asynq.NewScheduler(redis, nil) - - for k, v := range conf.Syncers { - task := asynq.NewTask(k, nil) - - _, err := scheduler.Register(v, task) - if err != nil { - return nil, err - } - - log.Info().Msgf("successfully registered %s syncer", k) + dashboardQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/dashboard.sql", sqlFilesPath)) + if err != nil { + return err } - return scheduler, nil -} - -func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) { - processorServer := asynq.NewServer( - redis, - asynq.Config{ - Concurrency: 5, - }, - ) - - mux := asynq.NewServeMux() - mux.HandleFunc("token", tokenSyncer) - mux.HandleFunc("cache", cacheSyncer) - mux.HandleFunc("ussd", ussdSyncer) - - return processorServer, mux + preparedQueries = &queries{ + core: coreQueries, + dashboard: dashboardQueries, + } + + return nil } diff --git a/cmd/main.go b/cmd/main.go index 3150d83..88fbf1f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,11 +1,10 @@ package main import ( - "github.com/grassrootseconomics/cic_go/cic_net" + cic_net "github.com/grassrootseconomics/cic-go/net" "github.com/jackc/pgx/v4/pgxpool" "github.com/knadh/koanf" "github.com/lmittmann/w3" - "github.com/nleof/goyesql" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "golang.org/x/sys/unix" @@ -16,10 +15,10 @@ import ( var ( k = koanf.New(".") - queries goyesql.Queries - conf config - db *pgxpool.Pool - cicnetClient *cic_net.CicNet + preparedQueries *queries + conf config + db *pgxpool.Pool + cicnetClient *cic_net.CicNet ) func init() { @@ -29,7 +28,7 @@ func init() { log.Fatal().Err(err).Msg("failed to load config") } - if err := loadQueries("queries.sql"); err != nil { + if err := loadQueries("queries"); err != nil { log.Fatal().Err(err).Msg("failed to load sql file") } diff --git a/cmd/syncer.go b/cmd/syncer.go new file mode 100644 index 0000000..89502a3 --- /dev/null +++ b/cmd/syncer.go @@ -0,0 +1,42 @@ +package main + +import ( + "cic-dw/internal/syncer" + "github.com/hibiken/asynq" + "github.com/rs/zerolog/log" +) + +func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) { + scheduler := asynq.NewScheduler(redis, nil) + + for k, v := range conf.Syncers { + task := asynq.NewTask(k, nil) + + _, err := scheduler.Register(v, task) + if err != nil { + return nil, err + } + + log.Info().Msgf("successfully registered %s syncer", k) + } + + return scheduler, nil +} + +func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) { + processorServer := asynq.NewServer( + redis, + asynq.Config{ + Concurrency: 5, + }, + ) + + syncer := syncer.New(db, redis, cicnetClient, preparedQueries.core) + + mux := asynq.NewServeMux() + mux.HandleFunc("token", syncer.TokenSyncer) + mux.HandleFunc("cache", syncer.CacheSyncer) + mux.HandleFunc("ussd", syncer.UssdSyncer) + + return processorServer, mux +} diff --git a/cmd/token_syncer.go b/cmd/token_syncer.go deleted file mode 100644 index dfb3e20..0000000 --- a/cmd/token_syncer.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "context" - "github.com/georgysavva/scany/pgxscan" - "github.com/hibiken/asynq" - "github.com/jackc/pgx/v4" - "github.com/lmittmann/w3" - "github.com/rs/zerolog/log" - "math/big" - "strconv" -) - -type tokenCursor struct { - CursorPos string `db:"cursor_pos"` -} - -func tokenSyncer(ctx context.Context, t *asynq.Task) error { - var lastCursor tokenCursor - - if err := pgxscan.Get(ctx, db, &lastCursor, queries["cursor-pos"], 3); err != nil { - return err - } - latestChainIdx, err := cicnetClient.EntryCount(ctx) - if err != nil { - return err - } - - lastCursorPos, err := strconv.ParseInt(lastCursor.CursorPos, 10, 64) - if err != nil { - return err - } - - latestChainPos := latestChainIdx.Int64() - 1 - log.Info().Msgf("=> %d tokens synced", lastCursorPos) - if latestChainPos >= lastCursorPos { - batch := &pgx.Batch{} - - for i := lastCursorPos; i <= latestChainPos; i++ { - nextTokenAddress, err := cicnetClient.AddressAtIndex(ctx, big.NewInt(i)) - if err != nil { - return err - } - tokenInfo, err := cicnetClient.TokenInfo(ctx, w3.A(nextTokenAddress)) - if err != nil { - return err - } - - batch.Queue(queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64()) - } - - res := db.SendBatch(ctx, batch) - for i := 0; i < batch.Len(); i++ { - _, err := res.Exec() - if err != nil { - return err - } - } - err := res.Close() - if err != nil { - return err - } - - _, err = db.Exec(ctx, queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3) - if err != nil { - return err - } - } - - return nil -} diff --git a/cmd/ussd_syncer.go b/cmd/ussd_syncer.go deleted file mode 100644 index b7f9b24..0000000 --- a/cmd/ussd_syncer.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "context" - "github.com/georgysavva/scany/pgxscan" - "github.com/hibiken/asynq" - "github.com/rs/zerolog/log" -) - -func ussdSyncer(ctx context.Context, t *asynq.Task) error { - _, err := db.Exec(ctx, queries["ussd-syncer"]) - if err != nil { - return asynq.SkipRetry - } - - var count tableCount - if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from users"); err != nil { - return asynq.SkipRetry - } - - log.Info().Msgf("=> %d users synced", count.Count) - - return nil -} diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 74caddd..5e88cd3 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -21,4 +21,4 @@ services: volumes: cic-dw-db: - driver: local \ No newline at end of file + driver: local diff --git a/go.mod b/go.mod index 5213039..a2c7d92 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.18 require ( github.com/ethereum/go-ethereum v1.10.17 github.com/georgysavva/scany v0.3.0 - github.com/grassrootseconomics/cic_go v0.1.0 + github.com/grassrootseconomics/cic-go v1.2.0 github.com/hibiken/asynq v0.23.0 - github.com/jackc/pgx/v4 v4.16.0 + github.com/jackc/pgx/v4 v4.16.1 github.com/knadh/koanf v1.4.1 github.com/lmittmann/w3 v0.7.0 github.com/nleof/goyesql v1.0.1 @@ -17,20 +17,20 @@ require ( require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.1.2 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-ole/go-ole v1.2.1 // indirect - github.com/go-redis/redis/v8 v8.11.5 // indirect + github.com/go-redis/redis/v8 v8.11.2 // indirect github.com/go-stack/stack v1.8.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/uuid v1.2.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.12.0 // indirect + github.com/jackc/pgconn v1.12.1 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.0 // indirect @@ -38,18 +38,18 @@ require ( github.com/jackc/pgtype v1.11.0 // indirect github.com/jackc/puddle v1.2.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pelletier/go-toml v1.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect - google.golang.org/protobuf v1.28.0 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + google.golang.org/protobuf v1.25.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect ) diff --git a/go.sum b/go.sum index 748991d..3b04071 100644 --- a/go.sum +++ b/go.sum @@ -63,17 +63,17 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd/btcec/v2 v2.1.2 h1:YoYoC9J0jwfukodSBMzZYUVQ8PTiYg4BnOWiJVzTmLs= github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -149,9 +149,8 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-redis/redis/v8 v8.11.2 h1:WqlSpAwz8mxDSMCvbyz1Mkiqe0LE5OY4j3lgkvu1Ts0= github.com/go-redis/redis/v8 v8.11.2/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M= -github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -186,10 +185,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -206,7 +203,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -215,9 +211,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -225,8 +220,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grassrootseconomics/cic_go v0.1.0 h1:Z7HL3QuEGeUVo/9rNbpweIqolpQ0wHJfGGDa/x49diQ= -github.com/grassrootseconomics/cic_go v0.1.0/go.mod h1:7jATeNdmgfuai59yXcVrh8wOTJJJmR44ynftOm7PISA= +github.com/grassrootseconomics/cic-go v1.2.0 h1:edZ9/ugvorSlPK2BbACVcb4q08pcWZhcRFVpp3O0WYc= +github.com/grassrootseconomics/cic-go v1.2.0/go.mod h1:cQcLMsuhCirTVO5ccG37S4pGS1vnkSepoi+eYZvdOEY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -284,8 +279,8 @@ github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpT github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0= -github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= +github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -325,8 +320,8 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk= -github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= +github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= +github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -414,9 +409,8 @@ github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go. github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -431,8 +425,8 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo github.com/nleof/goyesql v1.0.1 h1:AynOKT0zEDdE3rouyUYlmnLSwJ1pjwOL0gmvjLkKI1o= github.com/nleof/goyesql v1.0.1/go.mod h1:tpOzlGZT8etKGAEp2dlYX2CFeVHQiC9VMYY1kaeHrC4= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -440,12 +434,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -511,9 +505,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -717,9 +710,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= -golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -811,11 +803,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/syncer/cache.go b/internal/syncer/cache.go new file mode 100644 index 0000000..6ed83ab --- /dev/null +++ b/internal/syncer/cache.go @@ -0,0 +1,30 @@ +package syncer + +import ( + "context" + "github.com/georgysavva/scany/pgxscan" + "github.com/hibiken/asynq" + "github.com/rs/zerolog/log" +) + +type tableCount struct { + Count int `db:"count"` +} + +func (s *Syncer) CacheSyncer(ctx context.Context, t *asynq.Task) error { + _, err := s.db.Exec(ctx, s.queries["cache-syncer"]) + if err != nil { + log.Err(err).Msg("cache syncer task failed") + return asynq.SkipRetry + } + + var table tableCount + if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from transactions"); err != nil { + log.Err(err).Msg("cache syncer task failed") + return asynq.SkipRetry + } + + log.Info().Msgf("=> %d transactions synced", table.Count) + + return nil +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go new file mode 100644 index 0000000..970ab2d --- /dev/null +++ b/internal/syncer/syncer.go @@ -0,0 +1,24 @@ +package syncer + +import ( + cic_net "github.com/grassrootseconomics/cic-go/net" + "github.com/hibiken/asynq" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/nleof/goyesql" +) + +type Syncer struct { + db *pgxpool.Pool + rClient asynq.RedisConnOpt + cicnetClient *cic_net.CicNet + queries goyesql.Queries +} + +func New(db *pgxpool.Pool, rClient asynq.RedisConnOpt, cicnetClient *cic_net.CicNet, queries goyesql.Queries) *Syncer { + return &Syncer{ + db: db, + rClient: rClient, + cicnetClient: cicnetClient, + queries: queries, + } +} diff --git a/internal/syncer/token.go b/internal/syncer/token.go new file mode 100644 index 0000000..3c68bdb --- /dev/null +++ b/internal/syncer/token.go @@ -0,0 +1,78 @@ +package syncer + +import ( + "context" + "github.com/georgysavva/scany/pgxscan" + "github.com/hibiken/asynq" + "github.com/jackc/pgx/v4" + "github.com/lmittmann/w3" + "github.com/rs/zerolog/log" + "math/big" + "strconv" +) + +type tokenCursor struct { + cursorPos string `db:"cursor_pos"` +} + +func (s *Syncer) TokenSyncer(ctx context.Context, t *asynq.Task) error { + var lastCursor tokenCursor + + if err := pgxscan.Get(ctx, s.db, &lastCursor, s.queries["cursor-pos"], 3); err != nil { + return err + } + latestChainIdx, err := s.cicnetClient.EntryCount(ctx) + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + + lastCursorPos, err := strconv.ParseInt(lastCursor.cursorPos, 10, 64) + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + + latestChainPos := latestChainIdx.Int64() - 1 + log.Info().Msgf("=> %d tokens synced", lastCursorPos) + if latestChainPos >= lastCursorPos { + batch := &pgx.Batch{} + + for i := lastCursorPos; i <= latestChainPos; i++ { + nextTokenAddress, err := s.cicnetClient.AddressAtIndex(ctx, big.NewInt(i)) + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + tokenInfo, err := s.cicnetClient.ERC20TokenInfo(ctx, w3.A(nextTokenAddress)) + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + + batch.Queue(s.queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64()) + } + + res := s.db.SendBatch(ctx, batch) + for i := 0; i < batch.Len(); i++ { + _, err := res.Exec() + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + } + err := res.Close() + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + + _, err = s.db.Exec(ctx, s.queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3) + if err != nil { + log.Err(err).Msg("token syncer task failed") + return err + } + } + + return nil +} diff --git a/internal/syncer/ussd.go b/internal/syncer/ussd.go new file mode 100644 index 0000000..a83a8ba --- /dev/null +++ b/internal/syncer/ussd.go @@ -0,0 +1,26 @@ +package syncer + +import ( + "context" + "github.com/georgysavva/scany/pgxscan" + "github.com/hibiken/asynq" + "github.com/rs/zerolog/log" +) + +func (s *Syncer) UssdSyncer(ctx context.Context, t *asynq.Task) error { + _, err := s.db.Exec(ctx, s.queries["ussd-syncer"]) + if err != nil { + log.Err(err).Msg("ussd syncer task failed") + return asynq.SkipRetry + } + + var table tableCount + if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from users"); err != nil { + log.Err(err).Msg("ussd syncer task failed") + return asynq.SkipRetry + } + + log.Info().Msgf("=> %d users synced", table.Count) + + return nil +} diff --git a/queries.sql b/queries.sql index 946ec86..e69de29 100644 --- a/queries.sql +++ b/queries.sql @@ -1,41 +0,0 @@ --- syncers - --- name: ussd-syncer --- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor --- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential -WITH current_ussd_cursor AS ( - SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1) -) - -INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status) -SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status -FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300; - -UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1; - --- name: cache-syncer --- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor --- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential -WITH current_cache_cursor AS ( - SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2) -) - -INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success) -SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success -FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id -WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300; - -UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2; - --- name: cursor-pos --- Generic cursor query -SELECT cursor_pos from cursors WHERE id = $1; - --- name: insert-token-data --- Insert new token -INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES -(LOWER($1), $2, $3, $4); - --- name: update-cursor --- Generic cursor update -UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2; \ No newline at end of file diff --git a/queries/core.sql b/queries/core.sql new file mode 100644 index 0000000..e6254e5 --- /dev/null +++ b/queries/core.sql @@ -0,0 +1,41 @@ +-- syncer + +-- name: ussd-syncer +-- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor +-- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential +WITH current_ussd_cursor AS ( + SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1) +) + +INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status) +SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status +FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300; + +UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1; + +-- name: cache-syncer +-- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor +-- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential +WITH current_cache_cursor AS ( + SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2) +) + +INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success) +SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success +FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id +WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300; + +UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2; + +-- name: cursor-pos +-- Generic cursor query +SELECT cursor_pos from cursors WHERE id = $1; + +-- name: insert-token-data +-- Insert new token +INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES + (LOWER($1), $2, $3, $4); + +-- name: update-cursor +-- Generic cursor update +UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2; \ No newline at end of file diff --git a/queries/dashboard.sql b/queries/dashboard.sql new file mode 100644 index 0000000..e69de29