From 05523697b6cf8c35666734dc8c804cfb9f632715 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Thu, 31 Oct 2024 10:40:44 +0300 Subject: [PATCH] feat (breaking): switch to self contained auto bootstrapper --- Dockerfile | 1 - Makefile | 7 +- cmd/bootstrap/main.go | 251 ------------------------------------------ go.mod | 2 +- go.sum | 4 +- internal/api/api.go | 15 ++- queries.sql | 22 ---- 7 files changed, 11 insertions(+), 291 deletions(-) delete mode 100644 cmd/bootstrap/main.go diff --git a/Dockerfile b/Dockerfile index e538bbc..46d5cb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ WORKDIR /build COPY . . RUN go mod download -RUN go build -o eth-indexer-bootstrap -ldflags="-X main.build=${BUILD} -s -w" cmd/bootstrap/main.go RUN go build -o eth-indexer -ldflags="-X main.build=${BUILD} -s -w" cmd/service/*.go FROM debian:bookworm-slim diff --git a/Makefile b/Makefile index 7cc494f..ac32730 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ BIN := eth-indexer -BOOTSTRAP_BIN := eth-indexer-cache-bootstrap DB_FILE := tracker_db BUILD_CONF := CGO_ENABLED=1 GOOS=linux GOARCH=amd64 BUILD_COMMIT := $(shell git rev-parse --short HEAD 2> /dev/null) @@ -8,17 +7,13 @@ DEBUG := DEV=true .PHONY: build run run-bootstrap clean clean-debug clean: - rm ${BIN} ${BOOTSTRAP_BIN} + rm ${BIN} clean-db: rm ${DB_FILE} build: - ${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BOOTSTRAP_BIN} cmd/bootstrap/main.go ${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BIN} cmd/service/*.go -run-bootstrap: - ${BUILD_CONF} ${DEBUG} go run cmd/bootstrap/main.go - run: ${BUILD_CONF} ${DEBUG} go run cmd/service/*.go \ No newline at end of file diff --git a/cmd/bootstrap/main.go b/cmd/bootstrap/main.go deleted file mode 100644 index d0c6736..0000000 --- a/cmd/bootstrap/main.go +++ /dev/null @@ -1,251 +0,0 @@ -package main - -import ( - "context" - "flag" - "log/slog" - "os" - "time" - - "github.com/grassrootseconomics/eth-indexer/internal/store" - "github.com/grassrootseconomics/eth-indexer/internal/util" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/knadh/koanf/v2" - - "github.com/ethereum/go-ethereum/common" - "github.com/grassrootseconomics/ethutils" - "github.com/lmittmann/w3" - "github.com/lmittmann/w3/module/eth" -) - -type TokenArgs struct { - ContractAddress string - TokenName string - TokenSymbol string - TokenDecimals uint8 -} - -const ( - insertTokenQuery = `INSERT INTO tokens( - contract_address, - token_name, - token_symbol, - token_decimals - ) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING` -) - -var ( - build = "dev" - - confFlag string - migrationsFolderFlag string - queriesFlag string - - lo *slog.Logger - ko *koanf.Koanf - - dbPool *pgxpool.Pool -) - -func init() { - flag.StringVar(&confFlag, "config", "config.toml", "Config file location") - flag.StringVar(&migrationsFolderFlag, "migrations", "migrations/", "Migrations folder location") - flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location") - flag.Parse() - - lo = util.InitLogger() - ko = util.InitConfig(lo, confFlag) - - lo.Info("starting GE indexer token bootstrapper", "build", build) -} - -func main() { - var ( - tokenRegistryGetter = w3.MustNewFunc("tokenRegistry()", "address") - nameGetter = w3.MustNewFunc("name()", "string") - symbolGetter = w3.MustNewFunc("symbol()", "string") - decimalsGetter = w3.MustNewFunc("decimals()", "uint8") - ) - - chainProvider := ethutils.NewProvider(ko.MustString("chain.rpc_endpoint"), ko.MustInt64("chain.chainid")) - - var err error - dbPool, err = newPgStore() - if err != nil { - lo.Error("could not initialize postgres store", "error", err) - os.Exit(1) - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() - - for _, registry := range ko.MustStrings("bootstrap.ge_registries") { - registryMap, err := chainProvider.RegistryMap(ctx, ethutils.HexToAddress(registry)) - if err != nil { - lo.Error("could not fetch registry", "error", err) - os.Exit(1) - } - - if tokenIndex := registryMap[ethutils.TokenIndex]; tokenIndex != ethutils.ZeroAddress { - tokenIndexIter, err := chainProvider.NewBatchIterator(ctx, tokenIndex) - if err != nil { - lo.Error("could not create token index iter", "error", err) - os.Exit(1) - } - - for { - batch, err := tokenIndexIter.Next(ctx) - if err != nil { - lo.Error("error fetching next token index batch", "error", err) - os.Exit(1) - } - if batch == nil { - break - } - lo.Debug("index batch", "index", tokenIndex.Hex(), "size", len(batch)) - for _, address := range batch { - if address != ethutils.ZeroAddress { - var ( - tokenName string - tokenSymbol string - tokenDecimals uint8 - ) - - err := chainProvider.Client.CallCtx( - ctx, - eth.CallFunc(address, nameGetter).Returns(&tokenName), - eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol), - eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals), - ) - if err != nil { - lo.Error("error fetching token details", "error", err) - os.Exit(1) - } - - if err := insertToken(ctx, TokenArgs{ - ContractAddress: address.Hex(), - TokenName: tokenName, - TokenSymbol: tokenSymbol, - TokenDecimals: tokenDecimals, - }); err != nil { - lo.Error("pg insert error", "error", err) - os.Exit(1) - } - } - } - } - } - - if poolIndex := registryMap[ethutils.PoolIndex]; poolIndex != ethutils.ZeroAddress { - poolIndexIter, err := chainProvider.NewBatchIterator(ctx, poolIndex) - if err != nil { - lo.Error("cache could create pool index iter", "error", err) - os.Exit(1) - } - - for { - batch, err := poolIndexIter.Next(ctx) - if err != nil { - lo.Error("error fetching next pool index batch", "error", err) - os.Exit(1) - } - if batch == nil { - break - } - lo.Debug("index batch", "index", poolIndex.Hex(), "size", len(batch)) - for _, address := range batch { - var poolTokenIndex common.Address - err := chainProvider.Client.CallCtx( - ctx, - eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex), - ) - if err != nil { - lo.Error("error fetching pool token index and/or quoter", "error", err) - os.Exit(1) - } - if poolTokenIndex != ethutils.ZeroAddress { - poolTokenIndexIter, err := chainProvider.NewBatchIterator(ctx, poolTokenIndex) - if err != nil { - lo.Error("error creating pool token index iter", "error", err) - os.Exit(1) - } - - for { - batch, err := poolTokenIndexIter.Next(ctx) - if err != nil { - lo.Error("error fetching next pool token index batch", "error", err) - os.Exit(1) - } - if batch == nil { - break - } - lo.Debug("index batch", "index", poolTokenIndex.Hex(), "size", len(batch)) - for _, address := range batch { - if address != ethutils.ZeroAddress { - var ( - tokenName string - tokenSymbol string - tokenDecimals uint8 - ) - - err := chainProvider.Client.CallCtx( - ctx, - eth.CallFunc(address, nameGetter).Returns(&tokenName), - eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol), - eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals), - ) - if err != nil { - lo.Error("error fetching token details", "error", err) - os.Exit(1) - } - - if err := insertToken(ctx, TokenArgs{ - ContractAddress: address.Hex(), - TokenName: tokenName, - TokenSymbol: tokenSymbol, - TokenDecimals: tokenDecimals, - }); err != nil { - lo.Error("pg insert error", "error", err) - os.Exit(1) - } - } - } - } - } - } - } - } - } - lo.Info("tokens bootstrap complete") -} - -func newPgStore() (*pgxpool.Pool, error) { - store, err := store.NewPgStore(store.PgOpts{ - Logg: lo, - DSN: ko.MustString("postgres.dsn"), - MigrationsFolderPath: migrationsFolderFlag, - QueriesFolderPath: queriesFlag, - }) - if err != nil { - lo.Error("could not initialize postgres store", "error", err) - os.Exit(1) - } - - return store.Pool(), nil -} - -func insertToken(ctx context.Context, insertArgs TokenArgs) error { - _, err := dbPool.Exec( - ctx, - insertTokenQuery, - insertArgs.ContractAddress, - insertArgs.TokenName, - insertArgs.TokenSymbol, - insertArgs.TokenDecimals, - ) - if err != nil { - return err - } - - return nil -} diff --git a/go.mod b/go.mod index b97f358..d82587e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 require ( github.com/VictoriaMetrics/metrics v1.35.1 github.com/ethereum/go-ethereum v1.14.8 + github.com/go-chi/chi/v5 v5.1.0 github.com/grassrootseconomics/eth-tracker v1.2.2-rc github.com/grassrootseconomics/ethutils v1.3.0 github.com/jackc/pgx/v5 v5.6.0 @@ -19,7 +20,6 @@ require ( github.com/nats-io/nats.go v1.37.0 github.com/puzpuzpuz/xsync/v3 v3.4.0 github.com/sourcegraph/conc v0.3.0 - github.com/uptrace/bunrouter v1.0.22 ) require ( diff --git a/go.sum b/go.sum index 14da223..1627af9 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -204,8 +206,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/uptrace/bunrouter v1.0.22 h1:634bRGogHxjMaSqc5a3MjM/sisS/MkfXhWJ/WZXrktc= -github.com/uptrace/bunrouter v1.0.22/go.mod h1:O3jAcl+5qgnF+ejhgkmbceEk0E/mqaK+ADOocdNpY8M= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= diff --git a/internal/api/api.go b/internal/api/api.go index ea47162..29d774e 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -4,20 +4,19 @@ import ( "net/http" "github.com/VictoriaMetrics/metrics" - "github.com/uptrace/bunrouter" + "github.com/go-chi/chi/v5" ) -func New() *bunrouter.Router { - router := bunrouter.New() +func New() *chi.Mux { + r := chi.NewRouter() - router.GET("/metrics", metricsHandler()) + r.Get("/metrics", metricsHandler()) - return router + return r } -func metricsHandler() bunrouter.HandlerFunc { - return func(w http.ResponseWriter, _ bunrouter.Request) error { +func metricsHandler() http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { metrics.WritePrometheus(w, true) - return nil } } diff --git a/queries.sql b/queries.sql index 842b213..ab21ed1 100644 --- a/queries.sql +++ b/queries.sql @@ -127,25 +127,3 @@ INSERT INTO tokens( pool_name, pool_symbol ) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING - ---name: last-10-tx --- Fetches an account's last 10 transfers --- $1: public_key -SELECT -token_transfer.sender_address AS sender, token_transfer.recipient_address AS recipient, token_transfer.transfer_value, token_transfer.contract_address, -tx.tx_hash, tx.date_block, -tokens.token_symbol, tokens.token_decimals -FROM token_transfer -INNER JOIN tx ON token_transfer.tx_id = tx.id -INNER JOIN tokens ON token_transfer.contract_address = tokens.contract_address -WHERE token_transfer.sender_address = $1 OR token_transfer.recipient_address = $1 -ORDER BY tx.date_block DESC -LIMIT 10; - ---name: token-holdings --- Fetches an account's token holdings --- $1: public_key -SELECT DISTINCT tokens.token_symbol, tokens.contract_address, tokens.token_decimals FROM tokens -INNER JOIN token_transfer on tokens.contract_address = token_transfer.contract_address -WHERE token_transfer.sender_address = $1 -OR token_transfer.recipient_address = $1; \ No newline at end of file