diff --git a/cmd/service/custodial.go b/cmd/service/custodial.go index 18dea1e..68117a8 100644 --- a/cmd/service/custodial.go +++ b/cmd/service/custodial.go @@ -19,7 +19,7 @@ func initAbis() map[string]*w3.Func { // Keccak hash -> 0x449a52f8 "mintTo": w3.MustNewFunc("mintTo(address, uint256)", "bool"), // Keccak hash -> 0xa9059cbb - "transfer": w3.MustNewFunc("transfer(address,uint256)", "bool"), + "transfer": w3.MustNewFunc("transfer(address,uint256)", "bool"), // Keccak hash -> 0x23b872dd "transferFrom": w3.MustNewFunc("transferFrom(address, address, uint256)", "bool"), } @@ -43,7 +43,7 @@ func initSystemContainer(ctx context.Context, noncestore nonce.Noncestore) (*tas TokenTransferGasLimit: uint64(ko.MustInt64("system.token_transfer_gas_limit")), } // Check if system signer account nonce is present. - // If not, we bootstrap it from the network. + // If not (first boot), we bootstrap it from the network. currentSystemNonce, err := noncestore.Peek(ctx, ko.MustString("system.public_key")) lo.Info("custodial: loaded (noncestore) system nonce", "nonce", currentSystemNonce) if err == redis.Nil { diff --git a/cmd/service/init.go b/cmd/service/init.go index fd09cfb..3d936e3 100644 --- a/cmd/service/init.go +++ b/cmd/service/init.go @@ -8,11 +8,13 @@ import ( celo "github.com/grassrootseconomics/cic-celo-sdk" "github.com/grassrootseconomics/cic-custodial/internal/keystore" "github.com/grassrootseconomics/cic-custodial/internal/nonce" + "github.com/grassrootseconomics/cic-custodial/internal/queries" "github.com/grassrootseconomics/cic-custodial/internal/tasker" "github.com/grassrootseconomics/cic-custodial/pkg/logg" "github.com/grassrootseconomics/cic-custodial/pkg/postgres" "github.com/grassrootseconomics/cic-custodial/pkg/redis" "github.com/jackc/pgx/v5/pgxpool" + "github.com/knadh/goyesql/v2" "github.com/knadh/koanf" "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/env" @@ -93,7 +95,7 @@ func initPostgresPool() (*pgxpool.Pool, error) { func initAsynqRedisPool() (*redis.RedisPool, error) { poolOpts := redis.RedisPoolOpts{ DSN: ko.MustString("asynq.dsn"), - MinIdleConns: ko.MustInt("redis.minconn"), + MinIdleConns: ko.MustInt("redis.min_idle_conn"), } pool, err := redis.NewRedisPool(poolOpts) @@ -108,7 +110,7 @@ func initAsynqRedisPool() (*redis.RedisPool, error) { func initCommonRedisPool() (*redis.RedisPool, error) { poolOpts := redis.RedisPoolOpts{ DSN: ko.MustString("redis.dsn"), - MinIdleConns: ko.MustInt("redis.minconn"), + MinIdleConns: ko.MustInt("redis.min_idle_conn"), } pool, err := redis.NewRedisPool(poolOpts) @@ -121,14 +123,21 @@ func initCommonRedisPool() (*redis.RedisPool, error) { // Load postgres based keystore func initPostgresKeystore(postgresPool *pgxpool.Pool) (keystore.Keystore, error) { - keystore, err := keystore.NewPostgresKeytore(keystore.Opts{ - PostgresPool: postgresPool, - Logg: lo, - }) + parsedQueries, err := goyesql.ParseFile(queriesFlag) if err != nil { return nil, err } + loadedQueries, err := queries.LoadQueries(parsedQueries) + if err != nil { + return nil, err + } + + keystore := keystore.NewPostgresKeytore(keystore.Opts{ + PostgresPool: postgresPool, + Queries: loadedQueries, + }) + return keystore, nil } diff --git a/cmd/service/main.go b/cmd/service/main.go index db57e28..4b8b6a8 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -20,8 +20,9 @@ import ( ) var ( - confFlag string - debugFlag bool + confFlag string + debugFlag bool + queriesFlag string lo logf.Logger ko *koanf.Koanf @@ -39,6 +40,7 @@ type custodial struct { func init() { flag.StringVar(&confFlag, "config", "config.toml", "Config file location") flag.BoolVar(&debugFlag, "log", false, "Enable debug logging") + flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location") flag.Parse() lo = initLogger(debugFlag) @@ -77,7 +79,7 @@ func main() { postgresKeystore, err := initPostgresKeystore(postgresPool) if err != nil { - lo.Fatal("main: critical error loading postgres keystore", "error", err) + lo.Fatal("main: critical error loading keystore") } redisNoncestore := initRedisNoncestore(redisPool, celoProvider) diff --git a/cmd/service/tasker.go b/cmd/service/tasker.go index 73bb978..52e4343 100644 --- a/cmd/service/tasker.go +++ b/cmd/service/tasker.go @@ -7,7 +7,7 @@ import ( "github.com/hibiken/asynq" ) -// Load tasker handlers injecting necessary handler dependencies from the system container. +// Load tasker handlers, injecting any necessary handler dependencies from the system container. func initTasker(custodialContainer *custodial, redisPool *redis.RedisPool) *tasker.TaskerServer { lo.Debug("Bootstrapping tasker") @@ -50,14 +50,6 @@ func initTasker(custodialContainer *custodial, redisPool *redis.RedisPool) *task custodialContainer.systemContainer, custodialContainer.taskerClient, )) - taskerServer.RegisterHandlers(tasker.TransferTokenTask, task.TransferToken( - custodialContainer.celoProvider, - custodialContainer.noncestore, - custodialContainer.keystore, - custodialContainer.lockProvider, - custodialContainer.systemContainer, - custodialContainer.taskerClient, - )) taskerServer.RegisterHandlers(tasker.TxDispatchTask, task.TxDispatch( custodialContainer.celoProvider, )) diff --git a/config.toml b/config.toml index 9e2f6a6..1576ce1 100644 --- a/config.toml +++ b/config.toml @@ -41,5 +41,5 @@ min_idle_conn = 5 [asynq] worker_count = 15 debug = false -dsn = "redis://redis:6379/0" +dsn = "redis://localhost:6379/0" task_retention_hrs = 24 diff --git a/docker-compose.yaml b/docker-compose.dev.yaml similarity index 68% rename from docker-compose.yaml rename to docker-compose.dev.yaml index ebfc13e..1e55a5f 100644 --- a/docker-compose.yaml +++ b/docker-compose.dev.yaml @@ -1,4 +1,4 @@ -version: '3.9' +version: "3.9" services: redis: image: redis:6-alpine @@ -7,12 +7,12 @@ services: volumes: - cic-custodial-redis:/data ports: - - '6379:6379' + - "127.0.0.1:6379:6379" healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 10s timeout: 5s - retries: 5 + retries: 5 postgres: image: postgres:14-alpine restart: unless-stopped @@ -24,7 +24,7 @@ services: volumes: - cic-custodial-pg:/var/lib/postgresql/data ports: - - '5432:5432' + - "127.0.0.1:5432:5432" healthcheck: test: ["CMD-SHELL", "pg_isready"] interval: 10s @@ -36,24 +36,12 @@ services: environment: - REDIS_ADDR=redis:6379 ports: - - '8080:8080' + - "127.0.0.1:8080:8080" depends_on: - redis: - condition: service_healthy - cic-custodial: - image: ghcr.io/grassrootseconomics/cic-custodial/cic-custodial:latest - restart: unless-stopped - depends_on: - postgres: + redis: condition: service_healthy - redis: - condition: service_healthy - env_file: - - .env - ports: - - '5000:5000' volumes: cic-custodial-pg: driver: local cic-custodial-redis: - driver: local \ No newline at end of file + driver: local diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml deleted file mode 100644 index 21b770e..0000000 --- a/docker-compose.test.yaml +++ /dev/null @@ -1,29 +0,0 @@ -version: '3.9' -services: - redis: - image: redis:6-alpine - restart: unless-stopped - command: redis-server --save 60 1 --loglevel warning - ports: - - '6379:6379' - healthcheck: - test: ["CMD-SHELL", "redis-cli ping | grep PONG"] - interval: 10s - timeout: 5s - retries: 5 - postgres: - image: postgres:14-alpine - restart: unless-stopped - user: postgres - environment: - - POSTGRES_PASSWORD=postgres - - POSTGRES_USER=postgres - - POSTGRES_DB=cic_custodial - ports: - - '5432:5432' - healthcheck: - test: ["CMD-SHELL", "pg_isready"] - interval: 10s - timeout: 5s - retries: 5 - \ No newline at end of file diff --git a/go.mod b/go.mod index 22d0629..6a6ed4d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/grassrootseconomics/cic-custodial go 1.19 require ( - github.com/arl/statsviz v0.5.1 + github.com/VictoriaMetrics/metrics v1.23.1 github.com/bsm/redislock v0.7.2 github.com/celo-org/celo-blockchain v1.6.1 github.com/go-playground/validator v9.31.0+incompatible @@ -13,16 +13,15 @@ require ( github.com/grassrootseconomics/w3-celo-patch v0.1.0 github.com/hibiken/asynq v0.24.0 github.com/jackc/pgx/v5 v5.2.0 + github.com/knadh/goyesql/v2 v2.2.0 github.com/knadh/koanf v1.4.5 github.com/labstack/echo/v4 v4.10.0 - github.com/stretchr/testify v1.8.1 github.com/zerodha/logf v0.5.5 ) require ( filippo.io/edwards25519 v1.0.0 // indirect github.com/VictoriaMetrics/fastcache v1.12.0 // indirect - github.com/VictoriaMetrics/metrics v1.23.1 // indirect github.com/btcsuite/btcd v0.20.1-beta // indirect github.com/celo-org/celo-bls-go v0.6.4 // indirect github.com/celo-org/celo-bls-go-android v0.6.3 // indirect @@ -32,7 +31,6 @@ require ( github.com/celo-org/celo-bls-go-other v0.6.3 // indirect github.com/celo-org/celo-bls-go-windows v0.6.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set v1.8.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -65,10 +63,10 @@ require ( github.com/onsi/gomega v1.24.1 // indirect github.com/pelletier/go-toml v1.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/tsdb v0.10.0 // indirect github.com/rivo/uniseg v0.4.3 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spf13/cast v1.3.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect diff --git a/go.sum b/go.sum index 5af756e..e7dc595 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/arl/statsviz v0.5.1 h1:3HY0ZEB738JtguWsD1Tf1pFJZiCcWUmYRq/3OTYKaSI= -github.com/arl/statsviz v0.5.1/go.mod h1:zDnjgRblGm1Dyd7J5YlbH7gM1/+HRC+SfkhZhQb5AnM= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -205,6 +203,7 @@ github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Px github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -280,8 +279,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/grassrootseconomics/cic-celo-sdk v0.3.0 h1:uqYlad/sL4nWzExSE3ecfGQNdY0Gs6pqpkez1vX5XnI= -github.com/grassrootseconomics/cic-celo-sdk v0.3.0/go.mod h1:EiR6d03GYu6jlVKNL1MbTAw/bqAW2WP3J/lkrZxPMdU= github.com/grassrootseconomics/cic-celo-sdk v0.3.1 h1:SzmMFrqxSIdgePqwbUdoS3PNP82MFnlOecycVk2ZYWg= github.com/grassrootseconomics/cic-celo-sdk v0.3.1/go.mod h1:EiR6d03GYu6jlVKNL1MbTAw/bqAW2WP3J/lkrZxPMdU= github.com/grassrootseconomics/w3-celo-patch v0.1.0 h1:0fev2hYkGEyFX2D4oUG8yy4jXhtHv7qUtLLboXL5ycw= @@ -370,6 +367,7 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -394,6 +392,8 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knadh/goyesql/v2 v2.2.0 h1:DNQIzgITmMTXA+z+jDzbXCpgr7fGD6Hp0AJ7ZLEAem4= +github.com/knadh/goyesql/v2 v2.2.0/go.mod h1:is+wK/XQBukYK3DdKfpJRyDH9U/ZTMyX2u6DFijjRnI= github.com/knadh/koanf v1.4.5 h1:yKWFswTrqFc0u7jBAoERUz30+N1b1yPXU01gAPr8IrY= github.com/knadh/koanf v1.4.5/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -445,6 +445,7 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -512,6 +513,7 @@ github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHu github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -552,6 +554,7 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -581,8 +584,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -590,10 +591,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= diff --git a/internal/api/account.go b/internal/api/account.go index 7d687c8..703880c 100644 --- a/internal/api/account.go +++ b/internal/api/account.go @@ -9,6 +9,9 @@ import ( "github.com/labstack/echo/v4" ) +// CreateAccountHandler route. +// POST: /api/account/create. +// Returns the public key and tasker account prep receipt. func CreateAccountHandler( taskerClient *tasker.TaskerClient, keystore keystore.Keystore, @@ -22,7 +25,8 @@ func CreateAccountHandler( }) } - if err := keystore.WriteKeyPair(c.Request().Context(), generatedKeyPair); err != nil { + id, err := keystore.WriteKeyPair(c.Request().Context(), generatedKeyPair) + if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, errResp{ Ok: false, Code: INTERNAL_ERROR, @@ -33,11 +37,16 @@ func CreateAccountHandler( Ok: true, Result: H{ "publicKey": generatedKeyPair.Public, + "keyId": id, }, }) } } +// AccountStatusHandler route. +// GET: /api/account/status. +// Check if an account is ready to be used. +// Returns the status as a bool. func AccountStatusHandler() func(echo.Context) error { return func(c echo.Context) error { return c.JSON(http.StatusOK, okResp{ diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 671ad4e..609723f 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -7,7 +7,8 @@ import ( "github.com/grassrootseconomics/cic-custodial/pkg/keypair" ) +// Keystore defines how keypairs should be stored and accessed from a storage backend. type Keystore interface { - WriteKeyPair(context.Context, keypair.Key) error + WriteKeyPair(context.Context, keypair.Key) (uint, error) LoadPrivateKey(context.Context, string) (*ecdsa.PrivateKey, error) } diff --git a/internal/keystore/keystore_pg_test.go b/internal/keystore/keystore_pg_test.go deleted file mode 100644 index 94e9a1a..0000000 --- a/internal/keystore/keystore_pg_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package keystore - -import ( - "context" - "testing" - - "github.com/grassrootseconomics/cic-custodial/pkg/keypair" - "github.com/grassrootseconomics/cic-custodial/pkg/logg" - "github.com/grassrootseconomics/cic-custodial/pkg/postgres" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/stretchr/testify/suite" - "github.com/zerodha/logf" -) - -const ( - testDsn = "postgres://postgres:postgres@localhost:5432/cic_custodial" -) - -type itKeystoreSuite struct { - suite.Suite - keystore Keystore - pgPool *pgxpool.Pool - logg logf.Logger -} - -func TestItKeystoreSuite(t *testing.T) { - suite.Run(t, new(itKeystoreSuite)) -} - -func (s *itKeystoreSuite) SetupSuite() { - logg := logg.NewLogg(logg.LoggOpts{ - Debug: true, - Caller: true, - }) - - pgPool, err := postgres.NewPostgresPool(postgres.PostgresPoolOpts{ - DSN: testDsn, - }) - s.Require().NoError(err) - s.pgPool = pgPool - s.logg = logg - - s.keystore, err = NewPostgresKeytore(Opts{ - PostgresPool: pgPool, - Logg: logg, - }) - s.Require().NoError(err) -} - -func (s *itKeystoreSuite) TearDownSuite() { - _, err := s.pgPool.Exec(context.Background(), "DROP TABLE IF EXISTS keystore") - s.Require().NoError(err) -} - -func (s *itKeystoreSuite) Test_Write_And_Load_KeyPair() { - ctx := context.Background() - keypair, err := keypair.Generate() - s.NoError(err) - - err = s.keystore.WriteKeyPair(ctx, keypair) - s.NoError(err) - - _, err = s.keystore.LoadPrivateKey(ctx, keypair.Public) - s.NoError(err) -} diff --git a/internal/keystore/migrations.go b/internal/keystore/migrations.go deleted file mode 100644 index c183e01..0000000 --- a/internal/keystore/migrations.go +++ /dev/null @@ -1,27 +0,0 @@ -package keystore - -import ( - "context" - "time" - - "github.com/jackc/pgx/v5/pgxpool" -) - -func applyMigration(dbPool *pgxpool.Pool) error { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - _, err := dbPool.Exec(ctx, ` - CREATE TABLE IF NOT EXISTS keystore ( - id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - public_key TEXT NOT NULL, - private_key TEXT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - `) - if err != nil { - return err - } - - return nil -} diff --git a/internal/keystore/postgres.go b/internal/keystore/postgres.go index f8f57b7..72308d0 100644 --- a/internal/keystore/postgres.go +++ b/internal/keystore/postgres.go @@ -3,49 +3,48 @@ package keystore import ( "context" "crypto/ecdsa" - "fmt" eth_crypto "github.com/celo-org/celo-blockchain/crypto" + "github.com/grassrootseconomics/cic-custodial/internal/queries" "github.com/grassrootseconomics/cic-custodial/pkg/keypair" "github.com/jackc/pgx/v5/pgxpool" - "github.com/zerodha/logf" ) -type Opts struct { - PostgresPool *pgxpool.Pool - Logg logf.Logger -} - -type PostgresKeystore struct { - db *pgxpool.Pool -} - -func NewPostgresKeytore(o Opts) (Keystore, error) { - if err := applyMigration(o.PostgresPool); err != nil { - return nil, fmt.Errorf("keystore migration failed %v", err) +type ( + Opts struct { + PostgresPool *pgxpool.Pool + Queries *queries.Queries } - o.Logg.Info("Successfully ran keystore migrations") + PostgresKeystore struct { + db *pgxpool.Pool + queries *queries.Queries + } +) + +func NewPostgresKeytore(o Opts) Keystore { return &PostgresKeystore{ - db: o.PostgresPool, - }, nil + db: o.PostgresPool, + queries: o.Queries, + } } -func (ks *PostgresKeystore) WriteKeyPair(ctx context.Context, keypair keypair.Key) error { - _, err := ks.db.Exec(ctx, "INSERT INTO keystore(public_key, private_key) VALUES($1, $2)", keypair.Public, keypair.Private) - if err != nil { - return err +// WriteKeyPair inserts a keypair into the db and returns the linked id. +func (ks *PostgresKeystore) WriteKeyPair(ctx context.Context, keypair keypair.Key) (uint, error) { + var id uint + + if err := ks.db.QueryRow(ctx, ks.queries.WriteKeyPair, keypair.Public, keypair.Private).Scan(&id); err != nil { + return 0, err } - return nil + return id, nil } +// LoadPrivateKey loads a private key as a crypto primitive for direct use. An id is used to search for the private key. func (ks *PostgresKeystore) LoadPrivateKey(ctx context.Context, publicKey string) (*ecdsa.PrivateKey, error) { - var ( - privateKeyString string - ) + var privateKeyString string - if err := ks.db.QueryRow(ctx, "SELECT private_key FROM keystore WHERE public_key=$1", publicKey).Scan(&privateKeyString); err != nil { + if err := ks.db.QueryRow(ctx, ks.queries.LoadKeyPair, publicKey).Scan(&privateKeyString); err != nil { return nil, err } diff --git a/internal/nonce/nonce.go b/internal/nonce/nonce.go index e163354..9817ae2 100644 --- a/internal/nonce/nonce.go +++ b/internal/nonce/nonce.go @@ -2,6 +2,7 @@ package nonce import "context" +// Noncestore defines how a nonce store should be implemented for any storage backend. type Noncestore interface { Peek(context.Context, string) (uint64, error) Acquire(context.Context, string) (uint64, error) diff --git a/internal/queries/queries.go b/internal/queries/queries.go new file mode 100644 index 0000000..ffe33e7 --- /dev/null +++ b/internal/queries/queries.go @@ -0,0 +1,24 @@ +package queries + +import ( + "fmt" + + "github.com/knadh/goyesql/v2" +) + +type Queries struct { + // Keystore + WriteKeyPair string `query:"write-key-pair"` + LoadKeyPair string `query:"load-key-pair"` + // OTX +} + +func LoadQueries(q goyesql.Queries) (*Queries, error) { + loadedQueries := &Queries{} + + if err := goyesql.ScanToStruct(loadedQueries, q, nil); err != nil { + return nil, fmt.Errorf("failed to scan queries %v", err) + } + + return loadedQueries, nil +} diff --git a/internal/tasker/server.go b/internal/tasker/server.go index 85bf91e..4009f04 100644 --- a/internal/tasker/server.go +++ b/internal/tasker/server.go @@ -73,6 +73,7 @@ func (ts *TaskerServer) Stop() { func expectedFailures(err error) bool { switch err { + // Ignore lock contention errors; retry until lock obtain. case redislock.ErrNotObtained: return false default: @@ -80,6 +81,7 @@ func expectedFailures(err error) bool { } } +// Immidiatel func retryDelay(count int, err error, task *asynq.Task) time.Duration { if count < fixedRetryCount { return fixedRetryPeriod diff --git a/internal/tasker/task/dispatch.go b/internal/tasker/task/dispatch.go index a358f2c..1d6ef89 100644 --- a/internal/tasker/task/dispatch.go +++ b/internal/tasker/task/dispatch.go @@ -29,6 +29,7 @@ func TxDispatch( return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry) } + // TODO: Handle all fail cases if err := celoProvider.Client.CallCtx( ctx, eth.SendTx(p.Tx).Returns(&txHash), diff --git a/internal/tasker/task/system.go b/internal/tasker/task/system.go index 682fbec..c5c1bb2 100644 --- a/internal/tasker/task/system.go +++ b/internal/tasker/task/system.go @@ -191,6 +191,7 @@ func GiftTokenProcessor( } } +// TODO: https://github.com/grassrootseconomics/cic-custodial/issues/43 func RefillGasProcessor( celoProvider *celo.Provider, nonceProvider nonce.Noncestore, diff --git a/internal/tasker/task/transfer.go b/internal/tasker/task/transfer.go deleted file mode 100644 index 333a558..0000000 --- a/internal/tasker/task/transfer.go +++ /dev/null @@ -1,124 +0,0 @@ -package task - -import ( - "context" - "encoding/json" - "fmt" - "math" - "math/big" - "strconv" - - "github.com/bsm/redislock" - celo "github.com/grassrootseconomics/cic-celo-sdk" - "github.com/grassrootseconomics/cic-custodial/internal/keystore" - "github.com/grassrootseconomics/cic-custodial/internal/nonce" - "github.com/grassrootseconomics/cic-custodial/internal/tasker" - "github.com/grassrootseconomics/w3-celo-patch" - "github.com/hibiken/asynq" -) - -type TransferPayload struct { - From string `json:"from"` - To string `json:"to"` - VoucherAddress string `json:"voucherAddress"` - Amount string `json:"amount"` -} - -func TransferToken( - celoProvider *celo.Provider, - nonceProvider nonce.Noncestore, - keystoreProvider keystore.Keystore, - lockProvider *redislock.Client, - system *tasker.SystemContainer, - taskerClient *tasker.TaskerClient, -) func(context.Context, *asynq.Task) error { - return func(ctx context.Context, t *asynq.Task) error { - var p TransferPayload - - if err := json.Unmarshal(t.Payload(), &p); err != nil { - return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry) - } - - lock, err := lockProvider.Obtain(ctx, system.LockPrefix+p.From, system.LockTimeout, nil) - if err != nil { - return err - } - defer lock.Release(ctx) - - nonce, err := nonceProvider.Acquire(ctx, p.From) - if err != nil { - return err - } - - key, err := keystoreProvider.LoadPrivateKey(ctx, p.From) - if err != nil { - return err - } - - input, err := system.Abis["transfer"].EncodeArgs(w3.A(p.To), parseTransferValue(p.Amount, system.TokenDecimals)) - if err != nil { - return fmt.Errorf("ABI encode failed %v: %w", err, asynq.SkipRetry) - } - - builtTx, err := celoProvider.SignContractExecutionTx( - key, - celo.ContractExecutionTxOpts{ - ContractAddress: system.GiftableToken, - InputData: input, - GasPrice: celo.FixedMinGas, - GasLimit: system.TokenTransferGasLimit, - Nonce: nonce, - }, - ) - if err != nil { - if err := nonceProvider.Return(ctx, p.From); err != nil { - return err - } - return fmt.Errorf("nonce.Return failed: %v: %w", err, asynq.SkipRetry) - } - - disptachJobPayload, err := json.Marshal(TxPayload{ - Tx: builtTx, - }) - if err != nil { - return fmt.Errorf("json.Marshal failed: %v: %w", err, asynq.SkipRetry) - } - - _, err = taskerClient.CreateTask( - tasker.TxDispatchTask, - tasker.HighPriority, - &tasker.Task{ - Payload: disptachJobPayload, - }, - ) - if err != nil { - return err - } - - gasRefillPayload, err := json.Marshal(SystemPayload{ - PublicKey: p.From, - }) - if err != nil { - return fmt.Errorf("json.Marshal failed: %v: %w", err, asynq.SkipRetry) - } - - _, err = taskerClient.CreateTask( - tasker.RefillGasTask, - tasker.DefaultPriority, - &tasker.Task{ - Payload: gasRefillPayload, - }, - ) - if err != nil { - return err - } - - return nil - } -} - -func parseTransferValue(value string, tokenDecimals int) *big.Int { - floatValue, _ := strconv.ParseFloat(value, 64) - - return big.NewInt(int64(floatValue * math.Pow10(tokenDecimals))) -} diff --git a/internal/tasker/task/transfer_test.go b/internal/tasker/task/transfer_test.go deleted file mode 100644 index 853b463..0000000 --- a/internal/tasker/task/transfer_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package task - -import ( - "math/big" - "reflect" - "testing" -) - -func Test_parseTransferValue(t *testing.T) { - type args struct { - value string - tokenDecimals int - } - tests := []struct { - name string - args args - want *big.Int - }{ - { - name: "zero value string", - args: args{ - value: "0", - tokenDecimals: 6, - }, - want: big.NewInt(0), - }, - { - name: "fixed value string", - args: args{ - value: "2", - tokenDecimals: 6, - }, - want: big.NewInt(2000000), - }, - { - name: "float (2 d.p) value string", - args: args{ - value: "2.19", - tokenDecimals: 6, - }, - want: big.NewInt(2190000), - }, - { - name: "float (6 d.p) value string", - args: args{ - value: "2.123456", - tokenDecimals: 6, - }, - want: big.NewInt(2123456), - }, - { - name: "float (10 d.p) value string", - args: args{ - value: "2.1234567891", - tokenDecimals: 6, - }, - want: big.NewInt(2123456), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := parseTransferValue(tt.args.value, tt.args.tokenDecimals) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseValue() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/migrations/001_keystore.sql b/migrations/001_keystore.sql new file mode 100644 index 0000000..13b349c --- /dev/null +++ b/migrations/001_keystore.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS keystore ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + public_key TEXT NOT NULL, + private_key TEXT NOT NULL, + active BOOLEAN DEFAULT true, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +) \ No newline at end of file diff --git a/migrations/tern.conf b/migrations/tern.conf new file mode 100644 index 0000000..d651776 --- /dev/null +++ b/migrations/tern.conf @@ -0,0 +1,7 @@ +[database] +host = {{env "PG_HOST"}} +port = {{env "PG_PORT"}} +database = {{env "PG_DB"}} +user = {{env "PG_USER"}} +password = {{env "PG_PASSWORD"}} +sslmode = prefer \ No newline at end of file diff --git a/queries.sql b/queries.sql new file mode 100644 index 0000000..8e3acdd --- /dev/null +++ b/queries.sql @@ -0,0 +1,14 @@ +-- Keystore queries + +--name: write-key-pair +-- Save hex encoded private key +-- $1: public_key +-- $2: private_key +INSERT INTO keystore(public_key, private_key) VALUES($1, $2) RETURNING id + +--name: load-key-pair +-- Load saved key pair +-- $1: public_key +SELECT private_key FROM keystore WHERE id=$1 + +-- OTX queries