sohail/update deps structure (#5)
* refactor: syncer structure - move syncer jobs to internal dir * refactor: queries struct and pkg updates - update cic-go to latest - separate sql queries by logic * ci: add dependabot
This commit is contained in:
GitHub
parent
9882a9ddc6
commit
6b71657e5b
|
|
@ -0,0 +1,6 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type tableCount struct {
|
||||
Count int `db:"count"`
|
||||
}
|
||||
|
||||
func cacheSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
_, err := db.Exec(ctx, queries["cache-syncer"])
|
||||
if err != nil {
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
var count tableCount
|
||||
if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from transactions"); err != nil {
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
log.Info().Msgf("=> %d transactions synced", count.Count)
|
||||
|
||||
return nil
|
||||
}
|
||||
55
cmd/init.go
55
cmd/init.go
|
|
@ -2,8 +2,9 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/grassrootseconomics/cic_go/cic_net"
|
||||
cic_net "github.com/grassrootseconomics/cic-go/net"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/knadh/koanf"
|
||||
|
|
@ -11,7 +12,6 @@ import (
|
|||
"github.com/knadh/koanf/providers/env"
|
||||
"github.com/knadh/koanf/providers/file"
|
||||
"github.com/nleof/goyesql"
|
||||
"github.com/rs/zerolog/log"
|
||||
"strings"
|
||||
)
|
||||
|
||||
|
|
@ -27,6 +27,11 @@ type config struct {
|
|||
Syncers map[string]string `koanf:"syncers"`
|
||||
}
|
||||
|
||||
type queries struct {
|
||||
core goyesql.Queries
|
||||
dashboard goyesql.Queries
|
||||
}
|
||||
|
||||
func loadConfig(configFilePath string, k *koanf.Koanf) error {
|
||||
confFile := file.Provider(configFilePath)
|
||||
if err := k.Load(confFile, toml.Parser()); err != nil {
|
||||
|
|
@ -77,45 +82,21 @@ func connectCicNet(rpcProvider string, tokenIndex common.Address) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func loadQueries(sqlFile string) error {
|
||||
var err error
|
||||
queries, err = goyesql.ParseFile(sqlFile)
|
||||
func loadQueries(sqlFilesPath string) error {
|
||||
coreQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/core.sql", sqlFilesPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) {
|
||||
scheduler := asynq.NewScheduler(redis, nil)
|
||||
|
||||
for k, v := range conf.Syncers {
|
||||
task := asynq.NewTask(k, nil)
|
||||
|
||||
_, err := scheduler.Register(v, task)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Info().Msgf("successfully registered %s syncer", k)
|
||||
dashboardQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/dashboard.sql", sqlFilesPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return scheduler, nil
|
||||
}
|
||||
|
||||
func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) {
|
||||
processorServer := asynq.NewServer(
|
||||
redis,
|
||||
asynq.Config{
|
||||
Concurrency: 5,
|
||||
},
|
||||
)
|
||||
|
||||
mux := asynq.NewServeMux()
|
||||
mux.HandleFunc("token", tokenSyncer)
|
||||
mux.HandleFunc("cache", cacheSyncer)
|
||||
mux.HandleFunc("ussd", ussdSyncer)
|
||||
|
||||
return processorServer, mux
|
||||
preparedQueries = &queries{
|
||||
core: coreQueries,
|
||||
dashboard: dashboardQueries,
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
13
cmd/main.go
13
cmd/main.go
|
|
@ -1,11 +1,10 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/grassrootseconomics/cic_go/cic_net"
|
||||
cic_net "github.com/grassrootseconomics/cic-go/net"
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/knadh/koanf"
|
||||
"github.com/lmittmann/w3"
|
||||
"github.com/nleof/goyesql"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/sys/unix"
|
||||
|
|
@ -16,10 +15,10 @@ import (
|
|||
var (
|
||||
k = koanf.New(".")
|
||||
|
||||
queries goyesql.Queries
|
||||
conf config
|
||||
db *pgxpool.Pool
|
||||
cicnetClient *cic_net.CicNet
|
||||
preparedQueries *queries
|
||||
conf config
|
||||
db *pgxpool.Pool
|
||||
cicnetClient *cic_net.CicNet
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
@ -29,7 +28,7 @@ func init() {
|
|||
log.Fatal().Err(err).Msg("failed to load config")
|
||||
}
|
||||
|
||||
if err := loadQueries("queries.sql"); err != nil {
|
||||
if err := loadQueries("queries"); err != nil {
|
||||
log.Fatal().Err(err).Msg("failed to load sql file")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"cic-dw/internal/syncer"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) {
|
||||
scheduler := asynq.NewScheduler(redis, nil)
|
||||
|
||||
for k, v := range conf.Syncers {
|
||||
task := asynq.NewTask(k, nil)
|
||||
|
||||
_, err := scheduler.Register(v, task)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Info().Msgf("successfully registered %s syncer", k)
|
||||
}
|
||||
|
||||
return scheduler, nil
|
||||
}
|
||||
|
||||
func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) {
|
||||
processorServer := asynq.NewServer(
|
||||
redis,
|
||||
asynq.Config{
|
||||
Concurrency: 5,
|
||||
},
|
||||
)
|
||||
|
||||
syncer := syncer.New(db, redis, cicnetClient, preparedQueries.core)
|
||||
|
||||
mux := asynq.NewServeMux()
|
||||
mux.HandleFunc("token", syncer.TokenSyncer)
|
||||
mux.HandleFunc("cache", syncer.CacheSyncer)
|
||||
mux.HandleFunc("ussd", syncer.UssdSyncer)
|
||||
|
||||
return processorServer, mux
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/jackc/pgx/v4"
|
||||
"github.com/lmittmann/w3"
|
||||
"github.com/rs/zerolog/log"
|
||||
"math/big"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type tokenCursor struct {
|
||||
CursorPos string `db:"cursor_pos"`
|
||||
}
|
||||
|
||||
func tokenSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
var lastCursor tokenCursor
|
||||
|
||||
if err := pgxscan.Get(ctx, db, &lastCursor, queries["cursor-pos"], 3); err != nil {
|
||||
return err
|
||||
}
|
||||
latestChainIdx, err := cicnetClient.EntryCount(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lastCursorPos, err := strconv.ParseInt(lastCursor.CursorPos, 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
latestChainPos := latestChainIdx.Int64() - 1
|
||||
log.Info().Msgf("=> %d tokens synced", lastCursorPos)
|
||||
if latestChainPos >= lastCursorPos {
|
||||
batch := &pgx.Batch{}
|
||||
|
||||
for i := lastCursorPos; i <= latestChainPos; i++ {
|
||||
nextTokenAddress, err := cicnetClient.AddressAtIndex(ctx, big.NewInt(i))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tokenInfo, err := cicnetClient.TokenInfo(ctx, w3.A(nextTokenAddress))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
batch.Queue(queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64())
|
||||
}
|
||||
|
||||
res := db.SendBatch(ctx, batch)
|
||||
for i := 0; i < batch.Len(); i++ {
|
||||
_, err := res.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err := res.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(ctx, queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func ussdSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
_, err := db.Exec(ctx, queries["ussd-syncer"])
|
||||
if err != nil {
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
var count tableCount
|
||||
if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from users"); err != nil {
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
log.Info().Msgf("=> %d users synced", count.Count)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -21,4 +21,4 @@ services:
|
|||
|
||||
volumes:
|
||||
cic-dw-db:
|
||||
driver: local
|
||||
driver: local
|
||||
|
|
|
|||
24
go.mod
24
go.mod
|
|
@ -5,9 +5,9 @@ go 1.18
|
|||
require (
|
||||
github.com/ethereum/go-ethereum v1.10.17
|
||||
github.com/georgysavva/scany v0.3.0
|
||||
github.com/grassrootseconomics/cic_go v0.1.0
|
||||
github.com/grassrootseconomics/cic-go v1.2.0
|
||||
github.com/hibiken/asynq v0.23.0
|
||||
github.com/jackc/pgx/v4 v4.16.0
|
||||
github.com/jackc/pgx/v4 v4.16.1
|
||||
github.com/knadh/koanf v1.4.1
|
||||
github.com/lmittmann/w3 v0.7.0
|
||||
github.com/nleof/goyesql v1.0.1
|
||||
|
|
@ -17,20 +17,20 @@ require (
|
|||
|
||||
require (
|
||||
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.1.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
||||
github.com/deckarep/golang-set v1.8.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/go-ole/go-ole v1.2.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.2 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/golang/protobuf v1.4.3 // indirect
|
||||
github.com/google/uuid v1.2.0 // indirect
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.12.0 // indirect
|
||||
github.com/jackc/pgconn v1.12.1 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
|
||||
|
|
@ -38,18 +38,18 @@ require (
|
|||
github.com/jackc/pgtype v1.11.0 // indirect
|
||||
github.com/jackc/puddle v1.2.1 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml v1.7.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
github.com/spf13/cast v1.3.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.5 // indirect
|
||||
github.com/tklauser/numcpus v0.2.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
|
||||
google.golang.org/protobuf v1.25.0 // indirect
|
||||
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
|
||||
)
|
||||
|
|
|
|||
49
go.sum
49
go.sum
|
|
@ -63,17 +63,17 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
|
|||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
|
||||
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.1.2 h1:YoYoC9J0jwfukodSBMzZYUVQ8PTiYg4BnOWiJVzTmLs=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
|
||||
github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
|
|
@ -149,9 +149,8 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
|
|||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-redis/redis/v8 v8.11.2 h1:WqlSpAwz8mxDSMCvbyz1Mkiqe0LE5OY4j3lgkvu1Ts0=
|
||||
github.com/go-redis/redis/v8 v8.11.2/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
|
|
@ -186,10 +185,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
|
|||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
|
|
@ -206,7 +203,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
|
||||
github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
|
|
@ -215,9 +211,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
|
|||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
|
||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
|
|
@ -225,8 +220,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
|
|||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/grassrootseconomics/cic_go v0.1.0 h1:Z7HL3QuEGeUVo/9rNbpweIqolpQ0wHJfGGDa/x49diQ=
|
||||
github.com/grassrootseconomics/cic_go v0.1.0/go.mod h1:7jATeNdmgfuai59yXcVrh8wOTJJJmR44ynftOm7PISA=
|
||||
github.com/grassrootseconomics/cic-go v1.2.0 h1:edZ9/ugvorSlPK2BbACVcb4q08pcWZhcRFVpp3O0WYc=
|
||||
github.com/grassrootseconomics/cic-go v1.2.0/go.mod h1:cQcLMsuhCirTVO5ccG37S4pGS1vnkSepoi+eYZvdOEY=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
|
|
@ -284,8 +279,8 @@ github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpT
|
|||
github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
|
||||
github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
|
||||
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
|
||||
github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0=
|
||||
github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono=
|
||||
github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8=
|
||||
github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono=
|
||||
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
|
||||
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
|
||||
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
|
||||
|
|
@ -325,8 +320,8 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6
|
|||
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
|
||||
github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA=
|
||||
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
|
||||
github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk=
|
||||
github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI=
|
||||
github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y=
|
||||
github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ=
|
||||
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
|
|
@ -414,9 +409,8 @@ github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.
|
|||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
|
||||
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
|
||||
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
||||
|
|
@ -431,8 +425,8 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo
|
|||
github.com/nleof/goyesql v1.0.1 h1:AynOKT0zEDdE3rouyUYlmnLSwJ1pjwOL0gmvjLkKI1o=
|
||||
github.com/nleof/goyesql v1.0.1/go.mod h1:tpOzlGZT8etKGAEp2dlYX2CFeVHQiC9VMYY1kaeHrC4=
|
||||
github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||
|
|
@ -440,12 +434,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
|
|||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
|
||||
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
|
||||
github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
|
|
@ -511,9 +505,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
|
|||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
|
||||
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
|
||||
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
|
|
@ -717,9 +710,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
|
||||
golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
@ -811,11 +803,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
|
|||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
package syncer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type tableCount struct {
|
||||
Count int `db:"count"`
|
||||
}
|
||||
|
||||
func (s *Syncer) CacheSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
_, err := s.db.Exec(ctx, s.queries["cache-syncer"])
|
||||
if err != nil {
|
||||
log.Err(err).Msg("cache syncer task failed")
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
var table tableCount
|
||||
if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from transactions"); err != nil {
|
||||
log.Err(err).Msg("cache syncer task failed")
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
log.Info().Msgf("=> %d transactions synced", table.Count)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package syncer
|
||||
|
||||
import (
|
||||
cic_net "github.com/grassrootseconomics/cic-go/net"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/nleof/goyesql"
|
||||
)
|
||||
|
||||
type Syncer struct {
|
||||
db *pgxpool.Pool
|
||||
rClient asynq.RedisConnOpt
|
||||
cicnetClient *cic_net.CicNet
|
||||
queries goyesql.Queries
|
||||
}
|
||||
|
||||
func New(db *pgxpool.Pool, rClient asynq.RedisConnOpt, cicnetClient *cic_net.CicNet, queries goyesql.Queries) *Syncer {
|
||||
return &Syncer{
|
||||
db: db,
|
||||
rClient: rClient,
|
||||
cicnetClient: cicnetClient,
|
||||
queries: queries,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package syncer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/jackc/pgx/v4"
|
||||
"github.com/lmittmann/w3"
|
||||
"github.com/rs/zerolog/log"
|
||||
"math/big"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type tokenCursor struct {
|
||||
cursorPos string `db:"cursor_pos"`
|
||||
}
|
||||
|
||||
func (s *Syncer) TokenSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
var lastCursor tokenCursor
|
||||
|
||||
if err := pgxscan.Get(ctx, s.db, &lastCursor, s.queries["cursor-pos"], 3); err != nil {
|
||||
return err
|
||||
}
|
||||
latestChainIdx, err := s.cicnetClient.EntryCount(ctx)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
|
||||
lastCursorPos, err := strconv.ParseInt(lastCursor.cursorPos, 10, 64)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
|
||||
latestChainPos := latestChainIdx.Int64() - 1
|
||||
log.Info().Msgf("=> %d tokens synced", lastCursorPos)
|
||||
if latestChainPos >= lastCursorPos {
|
||||
batch := &pgx.Batch{}
|
||||
|
||||
for i := lastCursorPos; i <= latestChainPos; i++ {
|
||||
nextTokenAddress, err := s.cicnetClient.AddressAtIndex(ctx, big.NewInt(i))
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
tokenInfo, err := s.cicnetClient.ERC20TokenInfo(ctx, w3.A(nextTokenAddress))
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
|
||||
batch.Queue(s.queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64())
|
||||
}
|
||||
|
||||
res := s.db.SendBatch(ctx, batch)
|
||||
for i := 0; i < batch.Len(); i++ {
|
||||
_, err := res.Exec()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
}
|
||||
err := res.Close()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = s.db.Exec(ctx, s.queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("token syncer task failed")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package syncer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/georgysavva/scany/pgxscan"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func (s *Syncer) UssdSyncer(ctx context.Context, t *asynq.Task) error {
|
||||
_, err := s.db.Exec(ctx, s.queries["ussd-syncer"])
|
||||
if err != nil {
|
||||
log.Err(err).Msg("ussd syncer task failed")
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
var table tableCount
|
||||
if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from users"); err != nil {
|
||||
log.Err(err).Msg("ussd syncer task failed")
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
|
||||
log.Info().Msgf("=> %d users synced", table.Count)
|
||||
|
||||
return nil
|
||||
}
|
||||
41
queries.sql
41
queries.sql
|
|
@ -1,41 +0,0 @@
|
|||
-- syncers
|
||||
|
||||
-- name: ussd-syncer
|
||||
-- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor
|
||||
-- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
|
||||
WITH current_ussd_cursor AS (
|
||||
SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1)
|
||||
)
|
||||
|
||||
INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status)
|
||||
SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status
|
||||
FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300;
|
||||
|
||||
UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1;
|
||||
|
||||
-- name: cache-syncer
|
||||
-- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor
|
||||
-- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
|
||||
WITH current_cache_cursor AS (
|
||||
SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2)
|
||||
)
|
||||
|
||||
INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success)
|
||||
SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success
|
||||
FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id
|
||||
WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300;
|
||||
|
||||
UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2;
|
||||
|
||||
-- name: cursor-pos
|
||||
-- Generic cursor query
|
||||
SELECT cursor_pos from cursors WHERE id = $1;
|
||||
|
||||
-- name: insert-token-data
|
||||
-- Insert new token
|
||||
INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES
|
||||
(LOWER($1), $2, $3, $4);
|
||||
|
||||
-- name: update-cursor
|
||||
-- Generic cursor update
|
||||
UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2;
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
-- syncer
|
||||
|
||||
-- name: ussd-syncer
|
||||
-- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor
|
||||
-- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
|
||||
WITH current_ussd_cursor AS (
|
||||
SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1)
|
||||
)
|
||||
|
||||
INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status)
|
||||
SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status
|
||||
FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300;
|
||||
|
||||
UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1;
|
||||
|
||||
-- name: cache-syncer
|
||||
-- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor
|
||||
-- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
|
||||
WITH current_cache_cursor AS (
|
||||
SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2)
|
||||
)
|
||||
|
||||
INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success)
|
||||
SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success
|
||||
FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id
|
||||
WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300;
|
||||
|
||||
UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2;
|
||||
|
||||
-- name: cursor-pos
|
||||
-- Generic cursor query
|
||||
SELECT cursor_pos from cursors WHERE id = $1;
|
||||
|
||||
-- name: insert-token-data
|
||||
-- Insert new token
|
||||
INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES
|
||||
(LOWER($1), $2, $3, $4);
|
||||
|
||||
-- name: update-cursor
|
||||
-- Generic cursor update
|
||||
UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2;
|
||||
Loading…
Reference in New Issue