grassrootseconomics/eth-indexer
7
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat (breaking): switch to self contained auto bootstrapper

Browse Source
This commit is contained in:
Mohamed Sohail 2024-10-31 10:40:44 +03:00
parent 4ba4b72ec6
commit 05523697b6
Signed by: kamikazechaser
GPG Key ID: 7DD45520C01CD85D
7 changed files with 11 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -11,7 +11,6 @@ WORKDIR /build
COPY . .
RUN go mod download
RUN go build -o eth-indexer-bootstrap -ldflags="-X main.build=${BUILD} -s -w" cmd/bootstrap/main.go
RUN go build -o eth-indexer -ldflags="-X main.build=${BUILD} -s -w" cmd/service/*.go
FROM debian:bookworm-slim

7
Makefile
View File

@ -1,5 +1,4 @@
BIN := eth-indexer
BOOTSTRAP_BIN := eth-indexer-cache-bootstrap
DB_FILE := tracker_db
BUILD_CONF := CGO_ENABLED=1 GOOS=linux GOARCH=amd64
BUILD_COMMIT := $(shell git rev-parse --short HEAD 2> /dev/null)
@ -8,17 +7,13 @@ DEBUG := DEV=true
.PHONY: build run run-bootstrap clean clean-debug
clean:
rm ${BIN} ${BOOTSTRAP_BIN}
rm ${BIN}
clean-db:
rm ${DB_FILE}
build:
${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BOOTSTRAP_BIN} cmd/bootstrap/main.go
${BUILD_CONF} go build -ldflags="-X main.build=${BUILD_COMMIT} -s -w" -o ${BIN} cmd/service/*.go
run-bootstrap:
${BUILD_CONF} ${DEBUG} go run cmd/bootstrap/main.go
run:
${BUILD_CONF} ${DEBUG} go run cmd/service/*.go

251
cmd/bootstrap/main.go
View File

@ -1,251 +0,0 @@
package main
import (
"context"
"flag"
"log/slog"
"os"
"time"
"github.com/grassrootseconomics/eth-indexer/internal/store"
"github.com/grassrootseconomics/eth-indexer/internal/util"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/knadh/koanf/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/grassrootseconomics/ethutils"
"github.com/lmittmann/w3"
"github.com/lmittmann/w3/module/eth"
)
type TokenArgs struct {
ContractAddress string
TokenName string
TokenSymbol string
TokenDecimals uint8
}
const (
insertTokenQuery = `INSERT INTO tokens(
contract_address,
token_name,
token_symbol,
token_decimals
) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`
)
var (
build = "dev"
confFlag string
migrationsFolderFlag string
queriesFlag string
lo *slog.Logger
ko *koanf.Koanf
dbPool *pgxpool.Pool
)
func init() {
flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
flag.StringVar(&migrationsFolderFlag, "migrations", "migrations/", "Migrations folder location")
flag.StringVar(&queriesFlag, "queries", "queries.sql", "Queries file location")
flag.Parse()
lo = util.InitLogger()
ko = util.InitConfig(lo, confFlag)
lo.Info("starting GE indexer token bootstrapper", "build", build)
}
func main() {
var (
tokenRegistryGetter = w3.MustNewFunc("tokenRegistry()", "address")
nameGetter = w3.MustNewFunc("name()", "string")
symbolGetter = w3.MustNewFunc("symbol()", "string")
decimalsGetter = w3.MustNewFunc("decimals()", "uint8")
)
chainProvider := ethutils.NewProvider(ko.MustString("chain.rpc_endpoint"), ko.MustInt64("chain.chainid"))
var err error
dbPool, err = newPgStore()
if err != nil {
lo.Error("could not initialize postgres store", "error", err)
os.Exit(1)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
defer cancel()
for _, registry := range ko.MustStrings("bootstrap.ge_registries") {
registryMap, err := chainProvider.RegistryMap(ctx, ethutils.HexToAddress(registry))
if err != nil {
lo.Error("could not fetch registry", "error", err)
os.Exit(1)
}
if tokenIndex := registryMap[ethutils.TokenIndex]; tokenIndex != ethutils.ZeroAddress {
tokenIndexIter, err := chainProvider.NewBatchIterator(ctx, tokenIndex)
if err != nil {
lo.Error("could not create token index iter", "error", err)
os.Exit(1)
}
for {
batch, err := tokenIndexIter.Next(ctx)
if err != nil {
lo.Error("error fetching next token index batch", "error", err)
os.Exit(1)
}
if batch == nil {
break
}
lo.Debug("index batch", "index", tokenIndex.Hex(), "size", len(batch))
for _, address := range batch {
if address != ethutils.ZeroAddress {
var (
tokenName string
tokenSymbol string
tokenDecimals uint8
)
err := chainProvider.Client.CallCtx(
ctx,
eth.CallFunc(address, nameGetter).Returns(&tokenName),
eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol),
eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals),
)
if err != nil {
lo.Error("error fetching token details", "error", err)
os.Exit(1)
}
if err := insertToken(ctx, TokenArgs{
ContractAddress: address.Hex(),
TokenName: tokenName,
TokenSymbol: tokenSymbol,
TokenDecimals: tokenDecimals,
}); err != nil {
lo.Error("pg insert error", "error", err)
os.Exit(1)
}
}
}
}
}
if poolIndex := registryMap[ethutils.PoolIndex]; poolIndex != ethutils.ZeroAddress {
poolIndexIter, err := chainProvider.NewBatchIterator(ctx, poolIndex)
if err != nil {
lo.Error("cache could create pool index iter", "error", err)
os.Exit(1)
}
for {
batch, err := poolIndexIter.Next(ctx)
if err != nil {
lo.Error("error fetching next pool index batch", "error", err)
os.Exit(1)
}
if batch == nil {
break
}
lo.Debug("index batch", "index", poolIndex.Hex(), "size", len(batch))
for _, address := range batch {
var poolTokenIndex common.Address
err := chainProvider.Client.CallCtx(
ctx,
eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex),
)
if err != nil {
lo.Error("error fetching pool token index and/or quoter", "error", err)
os.Exit(1)
}
if poolTokenIndex != ethutils.ZeroAddress {
poolTokenIndexIter, err := chainProvider.NewBatchIterator(ctx, poolTokenIndex)
if err != nil {
lo.Error("error creating pool token index iter", "error", err)
os.Exit(1)
}
for {
batch, err := poolTokenIndexIter.Next(ctx)
if err != nil {
lo.Error("error fetching next pool token index batch", "error", err)
os.Exit(1)
}
if batch == nil {
break
}
lo.Debug("index batch", "index", poolTokenIndex.Hex(), "size", len(batch))
for _, address := range batch {
if address != ethutils.ZeroAddress {
var (
tokenName string
tokenSymbol string
tokenDecimals uint8
)
err := chainProvider.Client.CallCtx(
ctx,
eth.CallFunc(address, nameGetter).Returns(&tokenName),
eth.CallFunc(address, symbolGetter).Returns(&tokenSymbol),
eth.CallFunc(address, decimalsGetter).Returns(&tokenDecimals),
)
if err != nil {
lo.Error("error fetching token details", "error", err)
os.Exit(1)
}
if err := insertToken(ctx, TokenArgs{
ContractAddress: address.Hex(),
TokenName: tokenName,
TokenSymbol: tokenSymbol,
TokenDecimals: tokenDecimals,
}); err != nil {
lo.Error("pg insert error", "error", err)
os.Exit(1)
}
}
}
}
}
}
}
}
}
lo.Info("tokens bootstrap complete")
}
func newPgStore() (*pgxpool.Pool, error) {
store, err := store.NewPgStore(store.PgOpts{
Logg: lo,
DSN: ko.MustString("postgres.dsn"),
MigrationsFolderPath: migrationsFolderFlag,
QueriesFolderPath: queriesFlag,
})
if err != nil {
lo.Error("could not initialize postgres store", "error", err)
os.Exit(1)
}
return store.Pool(), nil
}
func insertToken(ctx context.Context, insertArgs TokenArgs) error {
_, err := dbPool.Exec(
ctx,
insertTokenQuery,
insertArgs.ContractAddress,
insertArgs.TokenName,
insertArgs.TokenSymbol,
insertArgs.TokenDecimals,
)
if err != nil {
return err
}
return nil
}

2
go.mod
View File

@ -5,6 +5,7 @@ go 1.23.0
require (
github.com/VictoriaMetrics/metrics v1.35.1
github.com/ethereum/go-ethereum v1.14.8
github.com/go-chi/chi/v5 v5.1.0
github.com/grassrootseconomics/eth-tracker v1.2.2-rc
github.com/grassrootseconomics/ethutils v1.3.0
github.com/jackc/pgx/v5 v5.6.0
@ -19,7 +20,6 @@ require (
github.com/nats-io/nats.go v1.37.0
github.com/puzpuzpuz/xsync/v3 v3.4.0
github.com/sourcegraph/conc v0.3.0
github.com/uptrace/bunrouter v1.0.22
)
require (

4
go.sum
View File

@ -68,6 +68,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
@ -204,8 +206,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/uptrace/bunrouter v1.0.22 h1:634bRGogHxjMaSqc5a3MjM/sisS/MkfXhWJ/WZXrktc=
github.com/uptrace/bunrouter v1.0.22/go.mod h1:O3jAcl+5qgnF+ejhgkmbceEk0E/mqaK+ADOocdNpY8M=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=

15
internal/api/api.go
View File

@ -4,20 +4,19 @@ import (
"net/http"
"github.com/VictoriaMetrics/metrics"
"github.com/uptrace/bunrouter"
"github.com/go-chi/chi/v5"
)
func New() *bunrouter.Router {
router := bunrouter.New()
func New() *chi.Mux {
r := chi.NewRouter()
router.GET("/metrics", metricsHandler())
r.Get("/metrics", metricsHandler())
return router
return r
}
func metricsHandler() bunrouter.HandlerFunc {
return func(w http.ResponseWriter, _ bunrouter.Request) error {
func metricsHandler() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
metrics.WritePrometheus(w, true)
return nil
}
}

22
queries.sql
View File

@ -127,25 +127,3 @@ INSERT INTO tokens(
pool_name,
pool_symbol
) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING
--name: last-10-tx
-- Fetches an account's last 10 transfers
-- $1: public_key
SELECT
token_transfer.sender_address AS sender, token_transfer.recipient_address AS recipient, token_transfer.transfer_value, token_transfer.contract_address,
tx.tx_hash, tx.date_block,
tokens.token_symbol, tokens.token_decimals
FROM token_transfer
INNER JOIN tx ON token_transfer.tx_id = tx.id
INNER JOIN tokens ON token_transfer.contract_address = tokens.contract_address
WHERE token_transfer.sender_address = $1 OR token_transfer.recipient_address = $1
ORDER BY tx.date_block DESC
LIMIT 10;
--name: token-holdings
-- Fetches an account's token holdings
-- $1: public_key
SELECT DISTINCT tokens.token_symbol, tokens.contract_address, tokens.token_decimals FROM tokens
INNER JOIN token_transfer on tokens.contract_address = token_transfer.contract_address
WHERE token_transfer.sender_address = $1
OR token_transfer.recipient_address = $1;