diff --git a/cmd/service/init.go b/cmd/service/init.go index c745855..d13ab8e 100644 --- a/cmd/service/init.go +++ b/cmd/service/init.go @@ -110,9 +110,11 @@ func initCommonRedisPool() *redis.RedisPool { } // Load redis backed noncestore. -func initRedisNoncestore(redisPool *redis.RedisPool) nonce.Noncestore { +func initRedisNoncestore(redisPool *redis.RedisPool, chainProvider *celoutils.Provider, store store.Store) nonce.Noncestore { return nonce.NewRedisNoncestore(nonce.Opts{ - RedisPool: redisPool, + ChainProvider: chainProvider, + RedisPool: redisPool, + Store: store, }) } diff --git a/cmd/service/main.go b/cmd/service/main.go index e233f9d..869d956 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -51,7 +51,7 @@ func main() { redisPool := initCommonRedisPool() store := initPgStore() - redisNoncestore := initRedisNoncestore(redisPool) + redisNoncestore := initRedisNoncestore(redisPool, celoProvider, store) lockProvider := initLockProvider(redisPool.Client) taskerClient := initTaskerClient(asynqRedisPool) diff --git a/internal/custodial/custodial.go b/internal/custodial/custodial.go index 3697cee..35ad03d 100644 --- a/internal/custodial/custodial.go +++ b/internal/custodial/custodial.go @@ -13,7 +13,6 @@ import ( "github.com/grassrootseconomics/cic-custodial/internal/tasker" "github.com/grassrootseconomics/cic-custodial/pkg/util" "github.com/grassrootseconomics/w3-celo-patch" - "github.com/grassrootseconomics/w3-celo-patch/module/eth" "github.com/labstack/gommon/log" "github.com/redis/go-redis/v9" ) @@ -56,22 +55,7 @@ func NewCustodial(o Opts) (*Custodial, error) { } _, err = o.Noncestore.Peek(ctx, o.SystemPublicKey) - if err == redis.Nil { - // TODO: Bootsrap from Postgres first - var networkNonce uint64 - - err := o.CeloProvider.Client.CallCtx( - ctx, - eth.Nonce(celoutils.HexToAddress(o.SystemPublicKey), nil).Returns(&networkNonce), - ) - if err != nil { - return nil, err - } - - if err := o.Noncestore.SetAccountNonce(ctx, o.SystemPublicKey, networkNonce); err != nil { - return nil, err - } - } else if err != nil { + if err != nil { return nil, err } diff --git a/internal/nonce/redis.go b/internal/nonce/redis.go index 16d8b11..0f3d7ec 100644 --- a/internal/nonce/redis.go +++ b/internal/nonce/redis.go @@ -2,30 +2,47 @@ package nonce import ( "context" + "errors" + "github.com/grassrootseconomics/celoutils" + "github.com/grassrootseconomics/cic-custodial/internal/store" redispool "github.com/grassrootseconomics/cic-custodial/pkg/redis" + "github.com/grassrootseconomics/w3-celo-patch/module/eth" + "github.com/jackc/pgx/v5" + "github.com/redis/go-redis/v9" ) type ( Opts struct { - RedisPool *redispool.RedisPool + ChainProvider *celoutils.Provider + RedisPool *redispool.RedisPool + Store store.Store } // RedisNoncestore implements `Noncestore` RedisNoncestore struct { - redis *redispool.RedisPool + chainProvider *celoutils.Provider + redis *redispool.RedisPool + store store.Store } ) func NewRedisNoncestore(o Opts) Noncestore { return &RedisNoncestore{ - redis: o.RedisPool, + chainProvider: o.ChainProvider, + redis: o.RedisPool, + store: o.Store, } } func (n *RedisNoncestore) Peek(ctx context.Context, publicKey string) (uint64, error) { nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64() - if err != nil { + if err == redis.Nil { + nonce, err = n.bootstrap(ctx, publicKey) + if err != nil { + return 0, err + } + } else if err != nil { return 0, err } @@ -38,8 +55,13 @@ func (n *RedisNoncestore) Acquire(ctx context.Context, publicKey string) (uint64 ) nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64() - if err != nil { - return 0, nil + if err == redis.Nil { + nonce, err = n.bootstrap(ctx, publicKey) + if err != nil { + return 0, err + } + } else if err != nil { + return 0, err } err = n.redis.Client.Incr(ctx, publicKey).Err() @@ -73,3 +95,30 @@ func (n *RedisNoncestore) SetAccountNonce(ctx context.Context, publicKey string, return nil } + +// bootstrap can be used to restore a destroyed redis nonce cache automatically. +// It first uses the otx_sign table as a source of nonce values. +// If the otx_sign table is corrupted, it can fallback to the network nonce. +// Ideally, the redis nonce cache should never be lost. +func (n *RedisNoncestore) bootstrap(ctx context.Context, publicKey string) (uint64, error) { + lastDbNonce, err := n.store.GetNextNonce(ctx, publicKey) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + err := n.chainProvider.Client.CallCtx( + ctx, + eth.Nonce(celoutils.HexToAddress(publicKey), nil).Returns(&lastDbNonce), + ) + if err != nil { + return 0, err + } + } else { + return 0, err + } + } + + if err := n.SetAccountNonce(ctx, publicKey, lastDbNonce); err != nil { + return 0, err + } + + return lastDbNonce, nil +} diff --git a/internal/store/otx.go b/internal/store/otx.go index 14fca24..722469a 100644 --- a/internal/store/otx.go +++ b/internal/store/otx.go @@ -61,6 +61,27 @@ func (s *PgStore) CreateOtx( return id, nil } +func (s *PgStore) GetNextNonce( + ctx context.Context, + publicAddress string, +) (uint64, error) { + var ( + lastNonce uint64 + ) + + if err := s.db.QueryRow( + ctx, + s.queries.GetNextNonce, + publicAddress, + ).Scan( + &lastNonce, + ); err != nil { + return 0, err + } + + return lastNonce, nil +} + func (s *PgStore) GetTxStatus( ctx context.Context, trackingId string, diff --git a/internal/store/store.go b/internal/store/store.go index 8cad093..5027193 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -21,6 +21,7 @@ type ( WriteKeyPair(context.Context, keypair.Key) (uint, error) // Otx related actions. CreateOtx(context.Context, Otx) (uint, error) + GetNextNonce(context.Context, string) (uint64, error) GetTxStatus(context.Context, string) (txStatus, error) CreateDispatchStatus(context.Context, uint, enum.OtxStatus) error UpdateDispatchStatus(context.Context, bool, string, uint64) error @@ -49,6 +50,7 @@ type ( LoadKeyPair string `query:"load-key-pair"` // Otx related queries. CreateOTX string `query:"create-otx"` + GetNextNonce string `query:"get-next-nonce"` GetTxStatusByTrackingId string `query:"get-tx-status-by-tracking-id"` CreateDispatchStatus string `query:"create-dispatch-status"` UpdateDispatchStatus string `query:"update-dispatch-status"` diff --git a/queries.sql b/queries.sql index 8933ddf..f652e89 100644 --- a/queries.sql +++ b/queries.sql @@ -34,6 +34,11 @@ INSERT INTO otx_sign( nonce ) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id +--name: get-next-nonce +-- Gets last nonce from the otx table for a particular address for bootstrapping purposes +-- $1: public_key +SELECT nonce + 1 AS nonce FROM otx_sign WHERE otx_sign.from = $1 ORDER BY created_at DESC LIMIT 1; + --name: get-tx-status-by-tracking-id -- Gets tx status's from possible multiple txs with the same tracking_id -- $1: tracking_id