feat (breaking): switch to self contained auto bootstrapper

Dockerfile

@@ -11,7 +11,6 @@ WORKDIR /build
 
 COPY . .
 RUN go mod download
-RUN go build -o eth-indexer-bootstrap -ldflags="-X main.build=${BUILD} -s -w" cmd/bootstrap/main.go
 RUN go build -o eth-indexer -ldflags="-X main.build=${BUILD} -s -w" cmd/service/*.go
 
 FROM debian:bookworm-slim

Makefile

@@ -1,5 +1,4 @@
 BIN := eth-indexer
-BOOTSTRAP_BIN := eth-indexer-cache-bootstrap
 DB_FILE := tracker_db 
 BUILD_CONF := CGO_ENABLED=1 GOOS=linux GOARCH=amd64
 BUILD_COMMIT := $(shell git rev-parse --short HEAD 2> /dev/null)
@@ -8,17 +7,13 @@ DEBUG := DEV=true
 .PHONY: build run run-bootstrap clean clean-debug
 
 clean:
-	rm ${BIN} ${BOOTSTRAP_BIN}
+	rm ${BIN}
 
 clean-db:
 	rm ${DB_FILE}
 
 build:
-	${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BOOTSTRAP_BIN} cmd/bootstrap/main.go
 	${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BIN} cmd/service/*.go
 
-run-bootstrap:
-	${BUILD_CONF} ${DEBUG} go run cmd/bootstrap/main.go
-
 run:
 	${BUILD_CONF} ${DEBUG} go run cmd/service/*.go

cmd/bootstrap/main.go

@@ -1,251 +0,0 @@
-package main
-
-import (
-	"context"
-	"flag"
-	"log/slog"
-	"os"
-	"time"
-
-	"github.com/grassrootseconomics/eth-indexer/internal/store"
-	"github.com/grassrootseconomics/eth-indexer/internal/util"
-	"github.com/jackc/pgx/v5/pgxpool"
-	"github.com/knadh/koanf/v2"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/grassrootseconomics/ethutils"
-	"github.com/lmittmann/w3"
-	"github.com/lmittmann/w3/module/eth"
-)
-
-type TokenArgs struct {
-	ContractAddress string
-	TokenName       string
-	TokenSymbol     string
-	TokenDecimals   uint8
-}
-
-const (
-	insertTokenQuery = `INSERT INTO tokens(
-		contract_address,
-		token_name,
-		token_symbol,
-		token_decimals
-	) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`
-)
-
-var (
-	build = "dev"
-
-	confFlag             string
-	migrationsFolderFlag string
-	queriesFlag          string
-
-	lo *slog.Logger
-	ko *koanf.Koanf
-
-	dbPool *pgxpool.Pool
-)
-
-func init() {
-	flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
-	flag.StringVar(&migrationsFolderFlag, "migrations", "migrations/", "Migrations folder location")
-	flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location")
-	flag.Parse()
-
-	lo = util.InitLogger()
-	ko = util.InitConfig(lo, confFlag)
-
-	lo.Info("starting GE indexer token bootstrapper", "build", build)
-}
-
-func main() {
-	var (
-		tokenRegistryGetter = w3.MustNewFunc("tokenRegistry()", "address")
-		nameGetter          = w3.MustNewFunc("name()", "string")
-		symbolGetter        = w3.MustNewFunc("symbol()", "string")
-		decimalsGetter      = w3.MustNewFunc("decimals()", "uint8")
-	)
-
-	chainProvider := ethutils.NewProvider(ko.MustString("chain.rpc_endpoint"), ko.MustInt64("chain.chainid"))
-
-	var err error
-	dbPool, err = newPgStore()
-	if err != nil {
-		lo.Error("could not initialize postgres store", "error", err)
-		os.Exit(1)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
-	defer cancel()
-
-	for _, registry := range ko.MustStrings("bootstrap.ge_registries") {
-		registryMap, err := chainProvider.RegistryMap(ctx, ethutils.HexToAddress(registry))
-		if err != nil {
-			lo.Error("could not fetch registry", "error", err)
-			os.Exit(1)
-		}
-
-		if tokenIndex := registryMap[ethutils.TokenIndex]; tokenIndex != ethutils.ZeroAddress {
-			tokenIndexIter, err := chainProvider.NewBatchIterator(ctx, tokenIndex)
-			if err != nil {
-				lo.Error("could not create token index iter", "error", err)
-				os.Exit(1)
-			}
-
-			for {
-				batch, err := tokenIndexIter.Next(ctx)
-				if err != nil {
-					lo.Error("error fetching next token index batch", "error", err)
-					os.Exit(1)
-				}
-				if batch == nil {
-					break
-				}
-				lo.Debug("index batch", "index", tokenIndex.Hex(), "size", len(batch))
-				for _, address := range batch {
-					if address != ethutils.ZeroAddress {
-						var (
-							tokenName     string
-							tokenSymbol   string
-							tokenDecimals uint8
-						)
-
-						err := chainProvider.Client.CallCtx(
-							ctx,
-							eth.CallFunc(address, nameGetter).Returns(&tokenName),
-							eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol),
-							eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals),
-						)
-						if err != nil {
-							lo.Error("error fetching token details", "error", err)
-							os.Exit(1)
-						}
-
-						if err := insertToken(ctx, TokenArgs{
-							ContractAddress: address.Hex(),
-							TokenName:       tokenName,
-							TokenSymbol:     tokenSymbol,
-							TokenDecimals:   tokenDecimals,
-						}); err != nil {
-							lo.Error("pg insert error", "error", err)
-							os.Exit(1)
-						}
-					}
-				}
-			}
-		}
-
-		if poolIndex := registryMap[ethutils.PoolIndex]; poolIndex != ethutils.ZeroAddress {
-			poolIndexIter, err := chainProvider.NewBatchIterator(ctx, poolIndex)
-			if err != nil {
-				lo.Error("cache could create pool index iter", "error", err)
-				os.Exit(1)
-			}
-
-			for {
-				batch, err := poolIndexIter.Next(ctx)
-				if err != nil {
-					lo.Error("error fetching next pool index batch", "error", err)
-					os.Exit(1)
-				}
-				if batch == nil {
-					break
-				}
-				lo.Debug("index batch", "index", poolIndex.Hex(), "size", len(batch))
-				for _, address := range batch {
-					var poolTokenIndex common.Address
-					err := chainProvider.Client.CallCtx(
-						ctx,
-						eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex),
-					)
-					if err != nil {
-						lo.Error("error fetching pool token index and/or quoter", "error", err)
-						os.Exit(1)
-					}
-					if poolTokenIndex != ethutils.ZeroAddress {
-						poolTokenIndexIter, err := chainProvider.NewBatchIterator(ctx, poolTokenIndex)
-						if err != nil {
-							lo.Error("error creating pool token index iter", "error", err)
-							os.Exit(1)
-						}
-
-						for {
-							batch, err := poolTokenIndexIter.Next(ctx)
-							if err != nil {
-								lo.Error("error fetching next pool token index batch", "error", err)
-								os.Exit(1)
-							}
-							if batch == nil {
-								break
-							}
-							lo.Debug("index batch", "index", poolTokenIndex.Hex(), "size", len(batch))
-							for _, address := range batch {
-								if address != ethutils.ZeroAddress {
-									var (
-										tokenName     string
-										tokenSymbol   string
-										tokenDecimals uint8
-									)
-
-									err := chainProvider.Client.CallCtx(
-										ctx,
-										eth.CallFunc(address, nameGetter).Returns(&tokenName),
-										eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol),
-										eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals),
-									)
-									if err != nil {
-										lo.Error("error fetching token details", "error", err)
-										os.Exit(1)
-									}
-
-									if err := insertToken(ctx, TokenArgs{
-										ContractAddress: address.Hex(),
-										TokenName:       tokenName,
-										TokenSymbol:     tokenSymbol,
-										TokenDecimals:   tokenDecimals,
-									}); err != nil {
-										lo.Error("pg insert error", "error", err)
-										os.Exit(1)
-									}
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	lo.Info("tokens bootstrap complete")
-}
-
-func newPgStore() (*pgxpool.Pool, error) {
-	store, err := store.NewPgStore(store.PgOpts{
-		Logg:                 lo,
-		DSN:                  ko.MustString("postgres.dsn"),
-		MigrationsFolderPath: migrationsFolderFlag,
-		QueriesFolderPath:    queriesFlag,
-	})
-	if err != nil {
-		lo.Error("could not initialize postgres store", "error", err)
-		os.Exit(1)
-	}
-
-	return store.Pool(), nil
-}
-
-func insertToken(ctx context.Context, insertArgs TokenArgs) error {
-	_, err := dbPool.Exec(
-		ctx,
-		insertTokenQuery,
-		insertArgs.ContractAddress,
-		insertArgs.TokenName,
-		insertArgs.TokenSymbol,
-		insertArgs.TokenDecimals,
-	)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

go.mod

@@ -5,6 +5,7 @@ go 1.23.0
 require (
 	github.com/VictoriaMetrics/metrics v1.35.1
 	github.com/ethereum/go-ethereum v1.14.8
+	github.com/go-chi/chi/v5 v5.1.0
 	github.com/grassrootseconomics/eth-tracker v1.2.2-rc
 	github.com/grassrootseconomics/ethutils v1.3.0
 	github.com/jackc/pgx/v5 v5.6.0
@@ -19,7 +20,6 @@ require (
 	github.com/nats-io/nats.go v1.37.0
 	github.com/puzpuzpuz/xsync/v3 v3.4.0
 	github.com/sourcegraph/conc v0.3.0
-	github.com/uptrace/bunrouter v1.0.22
 )
 
 require (

go.sum

@@ -68,6 +68,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
 github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
+github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
@@ -204,8 +206,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
-github.com/uptrace/bunrouter v1.0.22 h1:634bRGogHxjMaSqc5a3MjM/sisS/MkfXhWJ/WZXrktc=
-github.com/uptrace/bunrouter v1.0.22/go.mod h1:O3jAcl+5qgnF+ejhgkmbceEk0E/mqaK+ADOocdNpY8M=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=

internal/api/api.go

@@ -4,20 +4,19 @@ import (
 	"net/http"
 
 	"github.com/VictoriaMetrics/metrics"
-	"github.com/uptrace/bunrouter"
+	"github.com/go-chi/chi/v5"
 )
 
-func New() *bunrouter.Router {
+func New() *chi.Mux {
-	router := bunrouter.New()
+	r := chi.NewRouter()
 
-	router.GET("/metrics", metricsHandler())
+	r.Get("/metrics", metricsHandler())
 
-	return router
+	return r
 }
 
-func metricsHandler() bunrouter.HandlerFunc {
+func metricsHandler() http.HandlerFunc {
-	return func(w http.ResponseWriter, _ bunrouter.Request) error {
+	return func(w http.ResponseWriter, _ *http.Request) {
 		metrics.WritePrometheus(w, true)
-		return nil
 	}
 }

queries.sql

@@ -127,25 +127,3 @@ INSERT INTO tokens(
     pool_name,
     pool_symbol
 ) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING
-
---name: last-10-tx
--- Fetches an account's last 10 transfers
--- $1: public_key
-SELECT
-token_transfer.sender_address AS sender, token_transfer.recipient_address AS recipient, token_transfer.transfer_value, token_transfer.contract_address,
-tx.tx_hash, tx.date_block,
-tokens.token_symbol, tokens.token_decimals
-FROM token_transfer
-INNER JOIN tx ON token_transfer.tx_id = tx.id
-INNER JOIN tokens ON token_transfer.contract_address = tokens.contract_address
-WHERE token_transfer.sender_address = $1 OR token_transfer.recipient_address = $1
-ORDER BY tx.date_block DESC
-LIMIT 10;
-
---name: token-holdings
--- Fetches an account's token holdings
--- $1: public_key
-SELECT DISTINCT tokens.token_symbol, tokens.contract_address, tokens.token_decimals FROM tokens
-INNER JOIN token_transfer on tokens.contract_address = token_transfer.contract_address
-WHERE token_transfer.sender_address = $1
-OR token_transfer.recipient_address = $1;