feat: add nonce bootstrapper

Useful for rebuilding the nonce cache automatically.

* uses otx_sign as the first source
* can fallback to chain nonce value of the 1st source is corrupted

cmd/service/init.go

@@ -110,9 +110,11 @@ func initCommonRedisPool() *redis.RedisPool {
 }
 
 // Load redis backed noncestore.
-func initRedisNoncestore(redisPool *redis.RedisPool) nonce.Noncestore {
+func initRedisNoncestore(redisPool *redis.RedisPool, chainProvider *celoutils.Provider, store store.Store) nonce.Noncestore {
 	return nonce.NewRedisNoncestore(nonce.Opts{
-		RedisPool: redisPool,
+		ChainProvider: chainProvider,
+		RedisPool:     redisPool,
+		Store:         store,
 	})
 }
 

cmd/service/main.go

@@ -51,7 +51,7 @@ func main() {
 	redisPool := initCommonRedisPool()
 
 	store := initPgStore()
-	redisNoncestore := initRedisNoncestore(redisPool)
+	redisNoncestore := initRedisNoncestore(redisPool, celoProvider, store)
 	lockProvider := initLockProvider(redisPool.Client)
 	taskerClient := initTaskerClient(asynqRedisPool)
 

internal/custodial/custodial.go

@@ -13,7 +13,6 @@ import (
 	"github.com/grassrootseconomics/cic-custodial/internal/tasker"
 	"github.com/grassrootseconomics/cic-custodial/pkg/util"
 	"github.com/grassrootseconomics/w3-celo-patch"
-	"github.com/grassrootseconomics/w3-celo-patch/module/eth"
 	"github.com/labstack/gommon/log"
 	"github.com/redis/go-redis/v9"
 )
@@ -56,22 +55,7 @@ func NewCustodial(o Opts) (*Custodial, error) {
 	}
 
 	_, err = o.Noncestore.Peek(ctx, o.SystemPublicKey)
-	if err == redis.Nil {
+	if err != nil {
-		// TODO: Bootsrap from Postgres first
-		var networkNonce uint64
-
-		err := o.CeloProvider.Client.CallCtx(
-			ctx,
-			eth.Nonce(celoutils.HexToAddress(o.SystemPublicKey), nil).Returns(&networkNonce),
-		)
-		if err != nil {
-			return nil, err
-		}
-
-		if err := o.Noncestore.SetAccountNonce(ctx, o.SystemPublicKey, networkNonce); err != nil {
-			return nil, err
-		}
-	} else if err != nil {
 		return nil, err
 	}
 

internal/nonce/redis.go

@@ -2,30 +2,47 @@ package nonce
 
 import (
 	"context"
+	"errors"
 
+	"github.com/grassrootseconomics/celoutils"
+	"github.com/grassrootseconomics/cic-custodial/internal/store"
 	redispool "github.com/grassrootseconomics/cic-custodial/pkg/redis"
+	"github.com/grassrootseconomics/w3-celo-patch/module/eth"
+	"github.com/jackc/pgx/v5"
+	"github.com/redis/go-redis/v9"
 )
 
 type (
 	Opts struct {
-		RedisPool *redispool.RedisPool
+		ChainProvider *celoutils.Provider
+		RedisPool     *redispool.RedisPool
+		Store         store.Store
 	}
 
 	// RedisNoncestore implements `Noncestore`
 	RedisNoncestore struct {
-		redis *redispool.RedisPool
+		chainProvider *celoutils.Provider
+		redis         *redispool.RedisPool
+		store         store.Store
 	}
 )
 
 func NewRedisNoncestore(o Opts) Noncestore {
 	return &RedisNoncestore{
-		redis: o.RedisPool,
+		chainProvider: o.ChainProvider,
+		redis:         o.RedisPool,
+		store:         o.Store,
 	}
 }
 
 func (n *RedisNoncestore) Peek(ctx context.Context, publicKey string) (uint64, error) {
 	nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64()
-	if err != nil {
+	if err == redis.Nil {
+		nonce, err = n.bootstrap(ctx, publicKey)
+		if err != nil {
+			return 0, err
+		}
+	} else if err != nil {
 		return 0, err
 	}
 
@@ -38,8 +55,13 @@ func (n *RedisNoncestore) Acquire(ctx context.Context, publicKey string) (uint64
 	)
 
 	nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64()
-	if err != nil {
+	if err == redis.Nil {
-		return 0, nil
+		nonce, err = n.bootstrap(ctx, publicKey)
+		if err != nil {
+			return 0, err
+		}
+	} else if err != nil {
+		return 0, err
 	}
 
 	err = n.redis.Client.Incr(ctx, publicKey).Err()
@@ -73,3 +95,30 @@ func (n *RedisNoncestore) SetAccountNonce(ctx context.Context, publicKey string,
 
 	return nil
 }
+
+// bootstrap can be used to restore a destroyed redis nonce cache automatically.
+// It first uses the otx_sign table as a source of nonce values.
+// If the otx_sign table is corrupted, it can fallback to the network nonce.
+// Ideally, the redis nonce cache should never be lost.
+func (n *RedisNoncestore) bootstrap(ctx context.Context, publicKey string) (uint64, error) {
+	lastDbNonce, err := n.store.GetNextNonce(ctx, publicKey)
+	if err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			err := n.chainProvider.Client.CallCtx(
+				ctx,
+				eth.Nonce(celoutils.HexToAddress(publicKey), nil).Returns(&lastDbNonce),
+			)
+			if err != nil {
+				return 0, err
+			}
+		} else {
+			return 0, err
+		}
+	}
+
+	if err := n.SetAccountNonce(ctx, publicKey, lastDbNonce); err != nil {
+		return 0, err
+	}
+
+	return lastDbNonce, nil
+}

internal/store/otx.go

@@ -61,6 +61,27 @@ func (s *PgStore) CreateOtx(
 	return id, nil
 }
 
+func (s *PgStore) GetNextNonce(
+	ctx context.Context,
+	publicAddress string,
+) (uint64, error) {
+	var (
+		lastNonce uint64
+	)
+
+	if err := s.db.QueryRow(
+		ctx,
+		s.queries.GetNextNonce,
+		publicAddress,
+	).Scan(
+		&lastNonce,
+	); err != nil {
+		return 0, err
+	}
+
+	return lastNonce, nil
+}
+
 func (s *PgStore) GetTxStatus(
 	ctx context.Context,
 	trackingId string,

internal/store/store.go

@@ -21,6 +21,7 @@ type (
 		WriteKeyPair(context.Context, keypair.Key) (uint, error)
 		// Otx related actions.
 		CreateOtx(context.Context, Otx) (uint, error)
+		GetNextNonce(context.Context, string) (uint64, error)
 		GetTxStatus(context.Context, string) (txStatus, error)
 		CreateDispatchStatus(context.Context, uint, enum.OtxStatus) error
 		UpdateDispatchStatus(context.Context, bool, string, uint64) error
@@ -49,6 +50,7 @@ type (
 		LoadKeyPair  string `query:"load-key-pair"`
 		// Otx related queries.
 		CreateOTX               string `query:"create-otx"`
+		GetNextNonce            string `query:"get-next-nonce"`
 		GetTxStatusByTrackingId string `query:"get-tx-status-by-tracking-id"`
 		CreateDispatchStatus    string `query:"create-dispatch-status"`
 		UpdateDispatchStatus    string `query:"update-dispatch-status"`

queries.sql

@@ -34,6 +34,11 @@ INSERT INTO otx_sign(
     nonce
 ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id
 
+--name: get-next-nonce
+-- Gets last nonce from the otx table for a particular address for bootstrapping purposes
+-- $1: public_key
+SELECT nonce + 1 AS nonce FROM otx_sign WHERE otx_sign.from = $1 ORDER BY created_at DESC LIMIT 1;
+
 --name: get-tx-status-by-tracking-id
 -- Gets tx status's from possible multiple txs with the same tracking_id
 -- $1: tracking_id