add: ussd and cache syncer tasks

- no repeat on failure, picked up on next schedule
- enforce uniq on users and tx table to prevent duplicates

cmd/cache_syncer.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"context"
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/hibiken/asynq"
+	"github.com/rs/zerolog/log"
+)
+
+type cacheSyncer struct {
+	app *App
+}
+
+type tableCount struct {
+	Count int `db:"count"`
+}
+
+func newCacheSyncer(app *App) *cacheSyncer {
+	return &cacheSyncer{
+		app: app,
+	}
+}
+
+func (s *cacheSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
+	_, err := s.app.db.Exec(ctx, s.app.queries["cache-syncer"])
+	if err != nil {
+		return asynq.SkipRetry
+	}
+
+	var count tableCount
+	if err := pgxscan.Get(ctx, s.app.db, &count, "SELECT COUNT(*) from transactions"); err != nil {
+		return asynq.SkipRetry
+	}
+
+	log.Info().Msgf("=> %d transactions synced", count.Count)
+
+	return nil
+}

cmd/init.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 )
 
+// TODO: Load into koanf struct
 func loadConfig(configFilePath string, envOverridePrefix string, conf *koanf.Koanf) error {
 	// assumed to always be at the root folder
 	confFile := file.Provider(configFilePath)

cmd/processor.go

@@ -15,6 +15,8 @@ func runProcessor(app *App) {
 
 	mux := asynq.NewServeMux()
 	mux.Handle("token:sync", newTokenSyncer(app))
+	mux.Handle("cache:sync", newCacheSyncer(app))
+	mux.Handle("ussd:sync", newUssdSyncer(app))
 
 	if err := processorServer.Run(mux); err != nil {
 		log.Fatal().Err(err).Msg("failed to start job processor")

cmd/scheduler.go

@@ -10,14 +10,29 @@ var scheduler *asynq.Scheduler
 func runScheduler(app *App) {
 	scheduler = asynq.NewScheduler(app.rClient, nil)
 
+	// TODO: Refactor boilerplate and pull enabled tasks from koanf
 	tokenTask := asynq.NewTask("token:sync", nil)
+	cacheTask := asynq.NewTask("cache:sync", nil)
+	ussdTask := asynq.NewTask("ussd:sync", nil)
 
-	_, err := scheduler.Register(conf.String("schedule.token"), tokenTask)
+	_, err := scheduler.Register(conf.String("token.schedule"), tokenTask)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to register token syncer")
 	}
 	log.Info().Msg("successfully registered token syncer")
 
+	_, err = scheduler.Register(conf.String("cache.schedule"), cacheTask)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to register cache syncer")
+	}
+	log.Info().Msg("successfully registered cache syncer")
+
+	_, err = scheduler.Register(conf.String("ussd.schedule"), ussdTask)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to register ussd syncer")
+	}
+	log.Info().Msg("successfully registered ussd syncer")
+
 	if err := scheduler.Run(); err != nil {
 		log.Fatal().Err(err).Msg("could not start asynq scheduler")
 	}

cmd/token_syncer.go

@@ -27,10 +27,9 @@ func newTokenSyncer(app *App) *tokenSyncer {
 }
 
 func (s *tokenSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
-	log.Info().Msgf("running task type: %s", t.Type())
 	var lastCursor tokenCursor
 
-	if err := pgxscan.Get(ctx, s.app.db, &lastCursor, s.app.queries["token-cursor-pos"]); err != nil {
+	if err := pgxscan.Get(ctx, s.app.db, &lastCursor, s.app.queries["cursor-pos"], 3); err != nil {
 		return err
 	}
 	latestChainIdx, err := s.app.cicnetClient.EntryCount(ctx)
@@ -44,7 +43,7 @@ func (s *tokenSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
 	}
 
 	latestChainPos := latestChainIdx.Int64() - 1
-	log.Info().Msgf("current db cursor: %s, latest chain pos: %d", lastCursor.CursorPos, latestChainPos)
+	log.Info().Msgf("=> %d tokens synced", lastCursorPos)
 	if latestChainPos >= lastCursorPos {
 		batch := &pgx.Batch{}
 
@@ -63,7 +62,6 @@ func (s *tokenSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
 		}
 
 		res := s.app.db.SendBatch(ctx, batch)
-		log.Info().Msgf("inserting %d new records", batch.Len())
 		for i := 0; i < batch.Len(); i++ {
 			_, err := res.Exec()
 			if err != nil {
@@ -75,7 +73,7 @@ func (s *tokenSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
 			return err
 		}
 
-		_, err = s.app.db.Exec(ctx, s.app.queries["update-token-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10))
+		_, err = s.app.db.Exec(ctx, s.app.queries["update-cursor"], strconv.FormatInt(latestChainIdx.Int64(), 10), 3)
 		if err != nil {
 			return err
 		}

cmd/ussd_syncer.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"context"
+	"github.com/georgysavva/scany/pgxscan"
+	"github.com/hibiken/asynq"
+	"github.com/rs/zerolog/log"
+)
+
+type ussdSyncer struct {
+	app *App
+}
+
+func newUssdSyncer(app *App) *ussdSyncer {
+	return &ussdSyncer{
+		app: app,
+	}
+}
+
+func (s *ussdSyncer) ProcessTask(ctx context.Context, t *asynq.Task) error {
+	_, err := s.app.db.Exec(ctx, s.app.queries["ussd-syncer"])
+	if err != nil {
+		return asynq.SkipRetry
+	}
+
+	var count tableCount
+	if err := pgxscan.Get(ctx, s.app.db, &count, "SELECT COUNT(*) from users"); err != nil {
+		return asynq.SkipRetry
+	}
+
+	log.Info().Msgf("=> %d users synced", count.Count)
+
+	return nil
+}