diff --git a/cmd/service/main.go b/cmd/service/main.go index 869d956..4d66cf3 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -60,6 +60,7 @@ func main() { custodial, err := custodial.NewCustodial(custodial.Opts{ CeloProvider: celoProvider, LockProvider: lockProvider, + Logg: lo, Noncestore: redisNoncestore, Store: store, RedisClient: redisPool.Client, diff --git a/internal/api/sign.go b/internal/api/sign.go index 081d02c..7cea418 100644 --- a/internal/api/sign.go +++ b/internal/api/sign.go @@ -13,6 +13,7 @@ import ( ) // HandleSignTransfer godoc +// // @Summary Sign and dispatch transfer request. // @Description Sign and dispatch a transfer request. // @Tags network @@ -42,7 +43,7 @@ func HandleSignTransfer(cu *custodial.Custodial) func(echo.Context) error { return err } - accountActive, gasQuota, err := cu.Store.GetAccountStatus(c.Request().Context(), req.From) + accountActive, gasLock, err := cu.Store.GetAccountStatus(c.Request().Context(), req.From) if err != nil { return err } @@ -54,36 +55,15 @@ func HandleSignTransfer(cu *custodial.Custodial) func(echo.Context) error { }) } - trackingId := uuid.NewString() - - if gasQuota < 1 { - gasRefillPayload, err := json.Marshal(task.AccountPayload{ - PublicKey: req.From, - TrackingId: trackingId, - }) - if err != nil { - return err - } - - _, err = cu.TaskerClient.CreateTask( - c.Request().Context(), - tasker.AccountRefillGasTask, - tasker.DefaultPriority, - &tasker.Task{ - Id: trackingId, - Payload: gasRefillPayload, - }, - ) - if err != nil { - return err - } - + if gasLock { return c.JSON(http.StatusForbidden, ErrResp{ Ok: false, - Message: "Out of gas, refill pending. Try again later.", + Message: "Gas lock. Gas balance unavailable. Try again later.", }) } + trackingId := uuid.NewString() + taskPayload, err := json.Marshal(task.TransferPayload{ TrackingId: trackingId, From: req.From, diff --git a/internal/custodial/custodial.go b/internal/custodial/custodial.go index 35ad03d..7d335e4 100644 --- a/internal/custodial/custodial.go +++ b/internal/custodial/custodial.go @@ -15,12 +15,14 @@ import ( "github.com/grassrootseconomics/w3-celo-patch" "github.com/labstack/gommon/log" "github.com/redis/go-redis/v9" + "github.com/zerodha/logf" ) type ( Opts struct { CeloProvider *celoutils.Provider LockProvider *redislock.Client + Logg logf.Logger Noncestore nonce.Noncestore Store store.Store RedisClient *redis.Client @@ -54,11 +56,13 @@ func NewCustodial(o Opts) (*Custodial, error) { return nil, err } - _, err = o.Noncestore.Peek(ctx, o.SystemPublicKey) + systemNonce, err := o.Noncestore.Peek(ctx, o.SystemPublicKey) if err != nil { return nil, err } + o.Logg.Info("custodial: loaded_nonce", "system_nonce", systemNonce) + privateKey, err := eth_crypto.HexToECDSA(o.SystemPrivateKey) if err != nil { return nil, err diff --git a/internal/nonce/redis.go b/internal/nonce/redis.go index 0f3d7ec..aa7d768 100644 --- a/internal/nonce/redis.go +++ b/internal/nonce/redis.go @@ -37,13 +37,17 @@ func NewRedisNoncestore(o Opts) Noncestore { func (n *RedisNoncestore) Peek(ctx context.Context, publicKey string) (uint64, error) { nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64() - if err == redis.Nil { - nonce, err = n.bootstrap(ctx, publicKey) - if err != nil { + if err != nil { + if err == redis.Nil { + nonce, err = n.bootstrap(ctx, publicKey) + if err != nil { + return 0, err + } + + return nonce, nil + } else { return 0, err } - } else if err != nil { - return 0, err } return nonce, nil @@ -55,13 +59,15 @@ func (n *RedisNoncestore) Acquire(ctx context.Context, publicKey string) (uint64 ) nonce, err := n.redis.Client.Get(ctx, publicKey).Uint64() - if err == redis.Nil { - nonce, err = n.bootstrap(ctx, publicKey) - if err != nil { + if err != nil { + if err == redis.Nil { + nonce, err = n.bootstrap(ctx, publicKey) + if err != nil { + return 0, err + } + } else { return 0, err } - } else if err != nil { - return 0, err } err = n.redis.Client.Incr(ctx, publicKey).Err() diff --git a/internal/store/account.go b/internal/store/account.go index 8f68c74..2bdac26 100644 --- a/internal/store/account.go +++ b/internal/store/account.go @@ -22,10 +22,10 @@ func (s *PgStore) ActivateAccount( func (s *PgStore) GetAccountStatus( ctx context.Context, publicAddress string, -) (bool, int, error) { +) (bool, bool, error) { var ( accountActive bool - gasQuota int + gasLock bool ) if err := s.db.QueryRow( @@ -34,21 +34,21 @@ func (s *PgStore) GetAccountStatus( publicAddress, ).Scan( &accountActive, - &gasQuota, + &gasLock, ); err != nil { - return false, 0, err + return false, false, err } - return accountActive, gasQuota, nil + return accountActive, gasLock, nil } -func (s *PgStore) DecrGasQuota( +func (s *PgStore) GasLock( ctx context.Context, publicAddress string, ) error { if _, err := s.db.Exec( ctx, - s.queries.DecrGasQuota, + s.queries.GasLock, publicAddress, ); err != nil { return err @@ -57,13 +57,13 @@ func (s *PgStore) DecrGasQuota( return nil } -func (s *PgStore) ResetGasQuota( +func (s *PgStore) GasUnlock( ctx context.Context, publicAddress string, ) error { if _, err := s.db.Exec( ctx, - s.queries.ResetGasQuota, + s.queries.GasUnlock, publicAddress, ); err != nil { return err diff --git a/internal/store/store.go b/internal/store/store.go index f0f40a7..07aa25d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -27,10 +27,10 @@ type ( UpdateDispatchStatus(context.Context, bool, string, uint64) error // Account related actions. ActivateAccount(context.Context, string) error - GetAccountStatus(context.Context, string) (bool, int, error) + GetAccountStatus(context.Context, string) (bool, bool, error) // Gas quota related actions. - DecrGasQuota(context.Context, string) error - ResetGasQuota(context.Context, string) error + GasLock(context.Context, string) error + GasUnlock(context.Context, string) error } Opts struct { @@ -57,8 +57,8 @@ type ( // Account related queries. ActivateAccount string `query:"activate-account"` GetAccountStatus string `query:"get-account-status-by-address"` - DecrGasQuota string `query:"decr-gas-quota"` - ResetGasQuota string `query:"reset-gas-quota"` + GasLock string `query:"acc-gas-lock"` + GasUnlock string `query:"acc-gas-unlock"` } ) diff --git a/internal/sub/handler.go b/internal/sub/handler.go index 66d5a0b..b11c5b0 100644 --- a/internal/sub/handler.go +++ b/internal/sub/handler.go @@ -45,11 +45,11 @@ func (s *Sub) processEventHandler(ctx context.Context, msg *nats.Msg) error { return err } - if err := s.cu.Store.ResetGasQuota(ctx, chainEvent.To); err != nil { + if err := s.cu.Store.GasUnlock(ctx, chainEvent.To); err != nil { return err } case "CHAIN.gas": - if err := s.cu.Store.ResetGasQuota(ctx, chainEvent.To); err != nil { + if err := s.cu.Store.GasUnlock(ctx, chainEvent.To); err != nil { return err } } diff --git a/internal/tasker/task/account_refill_gas.go b/internal/tasker/task/account_refill_gas.go index ea6fb7a..53f32b5 100644 --- a/internal/tasker/task/account_refill_gas.go +++ b/internal/tasker/task/account_refill_gas.go @@ -35,16 +35,6 @@ func AccountRefillGasProcessor(cu *custodial.Custodial) func(context.Context, *a return err } - _, gasQuota, err := cu.Store.GetAccountStatus(ctx, payload.PublicKey) - if err != nil { - return err - } - - // The user has enough gas for atleast 5 more transactions. - if gasQuota > 5 { - return nil - } - if err := cu.CeloProvider.Client.CallCtx( ctx, eth.CallFunc( @@ -56,8 +46,8 @@ func AccountRefillGasProcessor(cu *custodial.Custodial) func(context.Context, *a return err } - // The user already requested funds, there is a cooldown applied. - // We can schedule an attempt after the cooldown period has passed. + // The user recently requested funds, there is a cooldown applied. + // We can schedule an attempt after the cooldown period has passed + 10 seconds. if nextTime.Int64() > time.Now().Unix() { _, err = cu.TaskerClient.CreateTask( ctx, @@ -66,7 +56,7 @@ func AccountRefillGasProcessor(cu *custodial.Custodial) func(context.Context, *a &tasker.Task{ Payload: t.Payload(), }, - asynq.ProcessAt(time.Unix(nextTime.Int64(), 0)), + asynq.ProcessAt(time.Unix(nextTime.Int64()+10, 0)), ) if err != nil { return err @@ -130,6 +120,7 @@ func AccountRefillGasProcessor(cu *custodial.Custodial) func(context.Context, *a InputData: input, GasFeeCap: celoutils.SafeGasFeeCap, GasTipCap: celoutils.SafeGasTipCap, + GasLimit: gasLimit, Nonce: nonce, }, ) diff --git a/internal/tasker/task/sign_transfer.go b/internal/tasker/task/sign_transfer.go index d836785..151c693 100644 --- a/internal/tasker/task/sign_transfer.go +++ b/internal/tasker/task/sign_transfer.go @@ -12,6 +12,7 @@ import ( "github.com/grassrootseconomics/cic-custodial/internal/store" "github.com/grassrootseconomics/cic-custodial/internal/tasker" "github.com/grassrootseconomics/cic-custodial/pkg/enum" + "github.com/grassrootseconomics/w3-celo-patch/module/eth" "github.com/hibiken/asynq" ) @@ -26,8 +27,9 @@ type TransferPayload struct { func SignTransfer(cu *custodial.Custodial) func(context.Context, *asynq.Task) error { return func(ctx context.Context, t *asynq.Task) error { var ( - err error - payload TransferPayload + err error + networkBalance big.Int + payload TransferPayload ) if err := json.Unmarshal(t.Payload(), &payload); err != nil { @@ -105,7 +107,10 @@ func SignTransfer(cu *custodial.Custodial) func(context.Context, *asynq.Task) er return err } - if err := cu.Store.DecrGasQuota(ctx, payload.From); err != nil { + if err := cu.CeloProvider.Client.CallCtx( + ctx, + eth.Balance(celoutils.HexToAddress(payload.From), nil).Returns(&networkBalance), + ); err != nil { return err } @@ -137,16 +142,22 @@ func SignTransfer(cu *custodial.Custodial) func(context.Context, *asynq.Task) er return err } - _, err = cu.TaskerClient.CreateTask( - ctx, - tasker.AccountRefillGasTask, - tasker.DefaultPriority, - &tasker.Task{ - Payload: gasRefillPayload, - }, - ) - if err != nil { - return err + if !balanceCheck(networkBalance) { + if err := cu.Store.GasLock(ctx, payload.From); err != nil { + return err + } + + _, err = cu.TaskerClient.CreateTask( + ctx, + tasker.AccountRefillGasTask, + tasker.DefaultPriority, + &tasker.Task{ + Payload: gasRefillPayload, + }, + ) + if err != nil { + return err + } } return nil diff --git a/internal/tasker/task/utils.go b/internal/tasker/task/utils.go index e3c7d77..b4ea683 100644 --- a/internal/tasker/task/utils.go +++ b/internal/tasker/task/utils.go @@ -1,6 +1,7 @@ package task import ( + "math/big" "time" "github.com/bsm/redislock" @@ -14,6 +15,15 @@ const ( lockTimeout = 1 * time.Second ) +var ( + // 20 gwei = max gas price we are willing to pay + // 250k = max gas limit + // minGasBalanceRequired is optimistic that the immidiate next transfer request will be successful + // but the subsequent one could fail (though low probability), therefore we can trigger a gas lock. + // Therefore our system wide threshold is 0.01 CELO or 10000000000000000 gas units + minGasBalanceRequired = big.NewInt(20000000000 * 250000 * 2) +) + // lockRetry will at most try to obtain the lock 20 times within ~0.5s. // it is expected to prevent immidiate requeue of the task at the expense of more redis calls. func lockRetry() redislock.RetryStrategy { @@ -22,3 +32,8 @@ func lockRetry() redislock.RetryStrategy { 20, ) } + +// balanceCheck compares the network balance with the system set min as threshold to execute a transfer. +func balanceCheck(networkBalance big.Int) bool { + return minGasBalanceRequired.Cmp(&networkBalance) < 0 +} diff --git a/migrations/005_remove_gas_quota.sql b/migrations/005_remove_gas_quota.sql new file mode 100644 index 0000000..49af156 --- /dev/null +++ b/migrations/005_remove_gas_quota.sql @@ -0,0 +1,34 @@ +-- Replace gas_quota with gas_lock which checks network balance threshold +DROP TRIGGER IF EXISTS update_gas_quota_timestamp ON gas_quota; +DROP TABLE IF EXISTS gas_quota_meta; +DROP TABLE IF EXISTS gas_quota; +DROP TRIGGER IF EXISTS insert_gas_quota ON keystore; +DROP FUNCTION IF EXISTS insert_gas_quota; + +-- Gas lock table +-- A gas_locked account indicates gas balance is below threshold awaiting next available top up +CREATE TABLE IF NOT EXISTS gas_lock ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + key_id INT REFERENCES keystore(id) NOT NULL, + lock BOOLEAN DEFAULT true, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +create function insert_gas_lock() + returns trigger +as $$ +begin + insert into gas_lock (key_id) values (new.id); + return new; +end; +$$ language plpgsql; + +create trigger insert_gas_lock + after insert on keystore +for each row +execute procedure insert_gas_lock(); + +create trigger update_gas_lock_timestamp + before update on gas_lock +for each row +execute procedure update_timestamp(); \ No newline at end of file diff --git a/queries.sql b/queries.sql index f652e89..2b6f6e6 100644 --- a/queries.sql +++ b/queries.sql @@ -73,27 +73,24 @@ UPDATE otx_dispatch SET "status" = $2, "block" = $3 WHERE otx_dispatch.id = ( UPDATE keystore SET active = true WHERE public_key=$1 --name: get-account-status-by-address --- Gets current gas quota for an individual account by address +-- Gets current gas lock and activation status for an individual account by address -- $1: public_key -SELECT keystore.active, gas_quota.quota FROM keystore -INNER JOIN gas_quota ON keystore.id = gas_quota.key_id +SELECT keystore.active, gas_lock.lock FROM keystore +INNER JOIN gas_lock ON keystore.id = gas_lock.key_id WHERE keystore.public_key=$1 ---name: decr-gas-quota --- Consumes a gas quota +--name: acc-gas-lock +-- Locks an account for gas reasons -- $1: public_key -UPDATE gas_quota SET quota = quota - 1 WHERE key_id = ( +UPDATE gas_lock SET lock = true WHERE key_id = ( SELECT id FROM keystore WHERE public_key=$1 ) ---name: reset-gas-quota --- Resets the gas quota --- 25 is the agreed upon quota +--name: acc-gas-unlock +-- Unlocks an account for gas reasons -- $1: public_key -UPDATE gas_quota SET quota = gas_quota_meta.default_quota -FROM gas_quota_meta -WHERE key_id = ( +UPDATE gas_lock SET lock = false WHERE key_id = ( SELECT id FROM keystore WHERE public_key=$1 -) +) \ No newline at end of file