sohail/update deps structure (#5)

* refactor: syncer structure

- move syncer jobs to internal dir

* refactor: queries struct and pkg updates

- update cic-go to latest
- separate sql queries by logic

* ci: add dependabot

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"

cmd/cache_syncer.go

@@ -1,28 +0,0 @@
-package main
-
-import (
-	"context"
-	"github.com/georgysavva/scany/pgxscan"
-	"github.com/hibiken/asynq"
-	"github.com/rs/zerolog/log"
-)
-
-type tableCount struct {
-	Count int `db:"count"`
-}
-
-func cacheSyncer(ctx context.Context, t *asynq.Task) error {
-	_, err := db.Exec(ctx, queries["cache-syncer"])
-	if err != nil {
-		return asynq.SkipRetry
-	}
-
-	var count tableCount
-	if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from transactions"); err != nil {
-		return asynq.SkipRetry
-	}
-
-	log.Info().Msgf("=> %d transactions synced", count.Count)
-
-	return nil
-}

cmd/init.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/grassrootseconomics/cic_go/cic_net"
+	cic_net "github.com/grassrootseconomics/cic-go/net"
 	"github.com/hibiken/asynq"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/knadh/koanf"
@@ -11,7 +12,6 @@ import (
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"
 	"github.com/nleof/goyesql"
-	"github.com/rs/zerolog/log"
 	"strings"
 )
 
@@ -27,6 +27,11 @@ type config struct {
 	Syncers map[string]string `koanf:"syncers"`
 }
 
+type queries struct {
+	core      goyesql.Queries
+	dashboard goyesql.Queries
+}
+
 func loadConfig(configFilePath string, k *koanf.Koanf) error {
 	confFile := file.Provider(configFilePath)
 	if err := k.Load(confFile, toml.Parser()); err != nil {
@@ -77,45 +82,21 @@ func connectCicNet(rpcProvider string, tokenIndex common.Address) error {
 	return nil
 }
 
-func loadQueries(sqlFile string) error {
+func loadQueries(sqlFilesPath string) error {
-	var err error
+	coreQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/core.sql", sqlFilesPath))
-	queries, err = goyesql.ParseFile(sqlFile)
 	if err != nil {
 		return err
 	}
 
-	return nil
+	dashboardQueries, err := goyesql.ParseFile(fmt.Sprintf("%s/dashboard.sql", sqlFilesPath))
-}
+	if err != nil {
-
+		return err
-func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) {
-	scheduler := asynq.NewScheduler(redis, nil)
-
-	for k, v := range conf.Syncers {
-		task := asynq.NewTask(k, nil)
-
-		_, err := scheduler.Register(v, task)
-		if err != nil {
-			return nil, err
-		}
-
-		log.Info().Msgf("successfully registered %s syncer", k)
 	}
 
-	return scheduler, nil
+	preparedQueries = &queries{
-}
+		core:      coreQueries,
-
+		dashboard: dashboardQueries,
-func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) {
+	}
-	processorServer := asynq.NewServer(
+
-		redis,
+	return nil
-		asynq.Config{
-			Concurrency: 5,
-		},
-	)
-
-	mux := asynq.NewServeMux()
-	mux.HandleFunc("token", tokenSyncer)
-	mux.HandleFunc("cache", cacheSyncer)
-	mux.HandleFunc("ussd", ussdSyncer)
-
-	return processorServer, mux
 }

cmd/main.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"github.com/grassrootseconomics/cic_go/cic_net"
+	cic_net "github.com/grassrootseconomics/cic-go/net"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/knadh/koanf"
 	"github.com/lmittmann/w3"
-	"github.com/nleof/goyesql"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sys/unix"
@@ -16,10 +15,10 @@ import (
 var (
 	k = koanf.New(".")
 
-	queries      goyesql.Queries
+	preparedQueries *queries
-	conf         config
+	conf            config
-	db           *pgxpool.Pool
+	db              *pgxpool.Pool
-	cicnetClient *cic_net.CicNet
+	cicnetClient    *cic_net.CicNet
 )
 
 func init() {
@@ -29,7 +28,7 @@ func init() {
 		log.Fatal().Err(err).Msg("failed to load config")
 	}
 
-	if err := loadQueries("queries.sql"); err != nil {
+	if err := loadQueries("queries"); err != nil {
 		log.Fatal().Err(err).Msg("failed to load sql file")
 	}
 

cmd/syncer.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"cic-dw/internal/syncer"
+	"github.com/hibiken/asynq"
+	"github.com/rs/zerolog/log"
+)
+
+func bootstrapScheduler(redis asynq.RedisConnOpt) (*asynq.Scheduler, error) {
+	scheduler := asynq.NewScheduler(redis, nil)
+
+	for k, v := range conf.Syncers {
+		task := asynq.NewTask(k, nil)
+
+		_, err := scheduler.Register(v, task)
+		if err != nil {
+			return nil, err
+		}
+
+		log.Info().Msgf("successfully registered %s syncer", k)
+	}
+
+	return scheduler, nil
+}
+
+func bootstrapProcessor(redis asynq.RedisConnOpt) (*asynq.Server, *asynq.ServeMux) {
+	processorServer := asynq.NewServer(
+		redis,
+		asynq.Config{
+			Concurrency: 5,
+		},
+	)
+
+	syncer := syncer.New(db, redis, cicnetClient, preparedQueries.core)
+
+	mux := asynq.NewServeMux()
+	mux.HandleFunc("token", syncer.TokenSyncer)
+	mux.HandleFunc("cache", syncer.CacheSyncer)
+	mux.HandleFunc("ussd", syncer.UssdSyncer)
+
+	return processorServer, mux
+}

cmd/token_syncer.go

@@ -1,71 +0,0 @@
-package main
-
-import (
-	"context"
-	"github.com/georgysavva/scany/pgxscan"
-	"github.com/hibiken/asynq"
-	"github.com/jackc/pgx/v4"
-	"github.com/lmittmann/w3"
-	"github.com/rs/zerolog/log"
-	"math/big"
-	"strconv"
-)
-
-type tokenCursor struct {
-	CursorPos string `db:"cursor_pos"`
-}
-
-func tokenSyncer(ctx context.Context, t *asynq.Task) error {
-	var lastCursor tokenCursor
-
-	if err := pgxscan.Get(ctx, db, &lastCursor, queries["cursor-pos"], 3); err != nil {
-		return err
-	}
-	latestChainIdx, err := cicnetClient.EntryCount(ctx)
-	if err != nil {
-		return err
-	}
-
-	lastCursorPos, err := strconv.ParseInt(lastCursor.CursorPos, 10, 64)
-	if err != nil {
-		return err
-	}
-
-	latestChainPos := latestChainIdx.Int64() - 1
-	log.Info().Msgf("=> %d tokens synced", lastCursorPos)
-	if latestChainPos >= lastCursorPos {
-		batch := &pgx.Batch{}
-
-		for i := lastCursorPos; i <= latestChainPos; i++ {
-			nextTokenAddress, err := cicnetClient.AddressAtIndex(ctx, big.NewInt(i))
-			if err != nil {
-				return err
-			}
-			tokenInfo, err := cicnetClient.TokenInfo(ctx, w3.A(nextTokenAddress))
-			if err != nil {
-				return err
-			}
-
-			batch.Queue(queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64())
-		}
-
-		res := db.SendBatch(ctx, batch)
-		for i := 0; i < batch.Len(); i++ {
-			_, err := res.Exec()
-			if err != nil {
-				return err
-			}
-		}
-		err := res.Close()
-		if err != nil {
-			return err
-		}
-
-		_, err = db.Exec(ctx, queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/ussd_syncer.go

@@ -1,24 +0,0 @@
-package main
-
-import (
-	"context"
-	"github.com/georgysavva/scany/pgxscan"
-	"github.com/hibiken/asynq"
-	"github.com/rs/zerolog/log"
-)
-
-func ussdSyncer(ctx context.Context, t *asynq.Task) error {
-	_, err := db.Exec(ctx, queries["ussd-syncer"])
-	if err != nil {
-		return asynq.SkipRetry
-	}
-
-	var count tableCount
-	if err := pgxscan.Get(ctx, db, &count, "SELECT COUNT(*) from users"); err != nil {
-		return asynq.SkipRetry
-	}
-
-	log.Info().Msgf("=> %d users synced", count.Count)
-
-	return nil
-}

go.mod

@@ -5,9 +5,9 @@ go 1.18
 require (
 	github.com/ethereum/go-ethereum v1.10.17
 	github.com/georgysavva/scany v0.3.0
-	github.com/grassrootseconomics/cic_go v0.1.0
+	github.com/grassrootseconomics/cic-go v1.2.0
 	github.com/hibiken/asynq v0.23.0
-	github.com/jackc/pgx/v4 v4.16.0
+	github.com/jackc/pgx/v4 v4.16.1
 	github.com/knadh/koanf v1.4.1
 	github.com/lmittmann/w3 v0.7.0
 	github.com/nleof/goyesql v1.0.1
@@ -17,20 +17,20 @@ require (
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.1.2 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-ole/go-ole v1.2.1 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/go-redis/redis/v8 v8.11.2 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.12.0 // indirect
+	github.com/jackc/pgconn v1.12.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
@@ -38,18 +38,18 @@ require (
 	github.com/jackc/pgtype v1.11.0 // indirect
 	github.com/jackc/puddle v1.2.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.7.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
-	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )

go.sum

@@ -63,17 +63,17 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/btcsuite/btcd/btcec/v2 v2.1.2 h1:YoYoC9J0jwfukodSBMzZYUVQ8PTiYg4BnOWiJVzTmLs=
 github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
+github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -149,9 +149,8 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-redis/redis/v8 v8.11.2 h1:WqlSpAwz8mxDSMCvbyz1Mkiqe0LE5OY4j3lgkvu1Ts0=
 github.com/go-redis/redis/v8 v8.11.2/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M=
-github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
-github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -186,10 +185,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -206,7 +203,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -215,9 +211,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -225,8 +220,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
-github.com/grassrootseconomics/cic_go v0.1.0 h1:Z7HL3QuEGeUVo/9rNbpweIqolpQ0wHJfGGDa/x49diQ=
+github.com/grassrootseconomics/cic-go v1.2.0 h1:edZ9/ugvorSlPK2BbACVcb4q08pcWZhcRFVpp3O0WYc=
-github.com/grassrootseconomics/cic_go v0.1.0/go.mod h1:7jATeNdmgfuai59yXcVrh8wOTJJJmR44ynftOm7PISA=
+github.com/grassrootseconomics/cic-go v1.2.0/go.mod h1:cQcLMsuhCirTVO5ccG37S4pGS1vnkSepoi+eYZvdOEY=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -284,8 +279,8 @@ github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpT
 github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
 github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
-github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0=
+github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8=
-github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono=
+github.com/jackc/pgconn v1.12.1/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono=
 github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@@ -325,8 +320,8 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6
 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
 github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA=
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
-github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk=
+github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y=
-github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI=
+github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
@@ -414,9 +409,8 @@ github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
@@ -431,8 +425,8 @@ github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo
 github.com/nleof/goyesql v1.0.1 h1:AynOKT0zEDdE3rouyUYlmnLSwJ1pjwOL0gmvjLkKI1o=
 github.com/nleof/goyesql v1.0.1/go.mod h1:tpOzlGZT8etKGAEp2dlYX2CFeVHQiC9VMYY1kaeHrC4=
 github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -440,12 +434,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
 github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
 github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
-github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -511,9 +505,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
-github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -717,9 +710,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w=
-golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -811,11 +803,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/syncer/cache.go

@@ -0,0 +1,30 @@
+package syncer
+
+import (
+	"context"
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/hibiken/asynq"
+	"github.com/rs/zerolog/log"
+)
+
+type tableCount struct {
+	Count int `db:"count"`
+}
+
+func (s *Syncer) CacheSyncer(ctx context.Context, t *asynq.Task) error {
+	_, err := s.db.Exec(ctx, s.queries["cache-syncer"])
+	if err != nil {
+		log.Err(err).Msg("cache syncer task failed")
+		return asynq.SkipRetry
+	}
+
+	var table tableCount
+	if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from transactions"); err != nil {
+		log.Err(err).Msg("cache syncer task failed")
+		return asynq.SkipRetry
+	}
+
+	log.Info().Msgf("=> %d transactions synced", table.Count)
+
+	return nil
+}

internal/syncer/syncer.go

@@ -0,0 +1,24 @@
+package syncer
+
+import (
+	cic_net "github.com/grassrootseconomics/cic-go/net"
+	"github.com/hibiken/asynq"
+	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/nleof/goyesql"
+)
+
+type Syncer struct {
+	db           *pgxpool.Pool
+	rClient      asynq.RedisConnOpt
+	cicnetClient *cic_net.CicNet
+	queries      goyesql.Queries
+}
+
+func New(db *pgxpool.Pool, rClient asynq.RedisConnOpt, cicnetClient *cic_net.CicNet, queries goyesql.Queries) *Syncer {
+	return &Syncer{
+		db:           db,
+		rClient:      rClient,
+		cicnetClient: cicnetClient,
+		queries:      queries,
+	}
+}

internal/syncer/token.go

@@ -0,0 +1,78 @@
+package syncer
+
+import (
+	"context"
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/hibiken/asynq"
+	"github.com/jackc/pgx/v4"
+	"github.com/lmittmann/w3"
+	"github.com/rs/zerolog/log"
+	"math/big"
+	"strconv"
+)
+
+type tokenCursor struct {
+	cursorPos string `db:"cursor_pos"`
+}
+
+func (s *Syncer) TokenSyncer(ctx context.Context, t *asynq.Task) error {
+	var lastCursor tokenCursor
+
+	if err := pgxscan.Get(ctx, s.db, &lastCursor, s.queries["cursor-pos"], 3); err != nil {
+		return err
+	}
+	latestChainIdx, err := s.cicnetClient.EntryCount(ctx)
+	if err != nil {
+		log.Err(err).Msg("token syncer task failed")
+		return err
+	}
+
+	lastCursorPos, err := strconv.ParseInt(lastCursor.cursorPos, 10, 64)
+	if err != nil {
+		log.Err(err).Msg("token syncer task failed")
+		return err
+	}
+
+	latestChainPos := latestChainIdx.Int64() - 1
+	log.Info().Msgf("=> %d tokens synced", lastCursorPos)
+	if latestChainPos >= lastCursorPos {
+		batch := &pgx.Batch{}
+
+		for i := lastCursorPos; i <= latestChainPos; i++ {
+			nextTokenAddress, err := s.cicnetClient.AddressAtIndex(ctx, big.NewInt(i))
+			if err != nil {
+				log.Err(err).Msg("token syncer task failed")
+				return err
+			}
+			tokenInfo, err := s.cicnetClient.ERC20TokenInfo(ctx, w3.A(nextTokenAddress))
+			if err != nil {
+				log.Err(err).Msg("token syncer task failed")
+				return err
+			}
+
+			batch.Queue(s.queries["insert-token-data"], nextTokenAddress[2:], tokenInfo.Name, tokenInfo.Symbol, tokenInfo.Decimals.Int64())
+		}
+
+		res := s.db.SendBatch(ctx, batch)
+		for i := 0; i < batch.Len(); i++ {
+			_, err := res.Exec()
+			if err != nil {
+				log.Err(err).Msg("token syncer task failed")
+				return err
+			}
+		}
+		err := res.Close()
+		if err != nil {
+			log.Err(err).Msg("token syncer task failed")
+			return err
+		}
+
+		_, err = s.db.Exec(ctx, s.queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3)
+		if err != nil {
+			log.Err(err).Msg("token syncer task failed")
+			return err
+		}
+	}
+
+	return nil
+}

internal/syncer/ussd.go

@@ -0,0 +1,26 @@
+package syncer
+
+import (
+	"context"
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/hibiken/asynq"
+	"github.com/rs/zerolog/log"
+)
+
+func (s *Syncer) UssdSyncer(ctx context.Context, t *asynq.Task) error {
+	_, err := s.db.Exec(ctx, s.queries["ussd-syncer"])
+	if err != nil {
+		log.Err(err).Msg("ussd syncer task failed")
+		return asynq.SkipRetry
+	}
+
+	var table tableCount
+	if err := pgxscan.Get(ctx, s.db, &table, "SELECT COUNT(*) from users"); err != nil {
+		log.Err(err).Msg("ussd syncer task failed")
+		return asynq.SkipRetry
+	}
+
+	log.Info().Msgf("=> %d users synced", table.Count)
+
+	return nil
+}

queries.sql

@@ -1,41 +0,0 @@
--- syncers
-
--- name: ussd-syncer
--- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor
--- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
-WITH current_ussd_cursor AS (
-    SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1)
-)
-
-INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status)
-SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status
-FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300;
-
-UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1;
-
--- name: cache-syncer
--- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor
--- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
-WITH current_cache_cursor AS (
-    SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2)
-)
-
-INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success)
-SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success
-FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id
-WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300;
-
-UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2;
-
--- name: cursor-pos
--- Generic cursor query
-SELECT cursor_pos from cursors WHERE id = $1;
-
--- name: insert-token-data
--- Insert new token
-INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES
-(LOWER($1), $2, $3, $4);
-
--- name: update-cursor
--- Generic cursor update
-UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2;

queries/core.sql

@@ -0,0 +1,41 @@
+-- syncer
+
+-- name: ussd-syncer
+-- This db transaction will auto scroll the cic_ussd remote adding values as per the limit and auto-updating the cursor
+-- The blockchain_address is used as a cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
+WITH current_ussd_cursor AS (
+    SELECT id FROM cic_ussd.account WHERE blockchain_address = (SELECT cursor_pos FROM cursors WHERE id = 1)
+)
+
+INSERT INTO users (phone_number, blockchain_address, date_registered, failed_pin_attempts, ussd_account_status)
+SELECT cic_ussd.account.phone_number, cic_ussd.account.blockchain_address, cic_ussd.account.created, cic_ussd.account.failed_pin_attempts, cic_ussd.account.status
+FROM cic_ussd.account WHERE cic_ussd.account.id > (SELECT id FROM current_ussd_cursor) ORDER BY cic_ussd.account.id ASC LIMIT 300;
+
+UPDATE cursors SET cursor_pos = (SELECT blockchain_address FROM users ORDER BY id DESC LIMIT 1) WHERE cursors.id = 1;
+
+-- name: cache-syncer
+-- This db transaction will auto scroll the cic_cache remote adding values as per the limit and auto-updating the cursor
+-- The tx_hash is used as the cursor to retrieve the corresponding id since the id is not guaranteed to be sequential
+WITH current_cache_cursor AS (
+    SELECT id FROM cic_cache.tx WHERE LOWER(tx_hash) = (SELECT cursor_pos FROM cursors WHERE id = 2)
+)
+
+INSERT INTO transactions (tx_hash, block_number, tx_index, token_address, sender_address, recipient_address, tx_value, date_block, tx_type, success)
+SELECT cic_cache.tx.tx_hash, cic_cache.tx.block_number, cic_cache.tx.tx_index, LOWER(cic_cache.tx.source_token), LOWER(cic_cache.tx.sender), LOWER(cic_cache.tx.recipient), cic_cache.tx.from_value, cic_cache.tx.date_block, concat(cic_cache.tag.domain, '_', cic_cache.tag.value) AS tx_type, cic_cache.tx.success
+FROM cic_cache.tx INNER JOIN cic_cache.tag_tx_link ON cic_cache.tx.id = cic_cache.tag_tx_link.tx_id INNER JOIN cic_cache.tag ON cic_cache.tag_tx_link.tag_id = cic_cache.tag.id
+WHERE cic_cache.tx.id > (SELECT id FROM current_cache_cursor) ORDER BY cic_cache.tx.id ASC LIMIT 300;
+
+UPDATE cursors SET cursor_pos = (SELECT tx_hash FROM transactions ORDER BY id DESC LIMIT 1) WHERE cursors.id = 2;
+
+-- name: cursor-pos
+-- Generic cursor query
+SELECT cursor_pos from cursors WHERE id = $1;
+
+-- name: insert-token-data
+-- Insert new token
+INSERT INTO tokens (token_address, token_name, token_symbol, token_decimals) VALUES
+    (LOWER($1), $2, $3, $4);
+
+-- name: update-cursor
+-- Generic cursor update
+UPDATE cursors SET cursor_pos = $1 WHERE cursors.id = $2;