mirror of
https://github.com/grassrootseconomics/eth-tracker.git
synced 2026-05-15 09:47:13 +02:00
release: v1.0.0-rc
This commit is contained in:
236
cmd/bootstrap/main.go
Normal file
236
cmd/bootstrap/main.go
Normal file
@@ -0,0 +1,236 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/celo-org/celo-blockchain/common"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/cache"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/chain"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/util"
|
||||
"github.com/grassrootseconomics/celoutils/v3"
|
||||
"github.com/grassrootseconomics/w3-celo"
|
||||
"github.com/grassrootseconomics/w3-celo/module/eth"
|
||||
"github.com/knadh/koanf/v2"
|
||||
)
|
||||
|
||||
const cacheType = "redis"
|
||||
|
||||
var (
|
||||
build = "dev"
|
||||
|
||||
confFlag string
|
||||
|
||||
lo *slog.Logger
|
||||
ko *koanf.Koanf
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
|
||||
flag.Parse()
|
||||
|
||||
lo = util.InitLogger()
|
||||
ko = util.InitConfig(lo, confFlag)
|
||||
|
||||
lo.Info("starting GE redis cache bootstrapper", "build", build)
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
tokenRegistryGetter = w3.MustNewFunc("tokenRegistry()", "address")
|
||||
quoterGetter = w3.MustNewFunc("quoter()", "address")
|
||||
)
|
||||
|
||||
chain, err := chain.NewRPCFetcher(chain.CeloRPCOpts{
|
||||
RPCEndpoint: ko.MustString("chain.rpc_endpoint"),
|
||||
ChainID: ko.MustInt64("chain.chainid"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize chain client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cache, err := cache.New(cache.CacheOpts{
|
||||
Logg: lo,
|
||||
CacheType: cacheType,
|
||||
RedisDSN: ko.MustString("redis.dsn"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize cache", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
|
||||
defer cancel()
|
||||
|
||||
for _, registry := range ko.MustStrings("bootstrap.ge_registries") {
|
||||
registryMap, err := chain.Provider().RegistryMap(ctx, celoutils.HexToAddress(registry))
|
||||
if err != nil {
|
||||
lo.Error("could not fetch registry", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if tokenIndex := registryMap[celoutils.TokenIndex]; tokenIndex != celoutils.ZeroAddress {
|
||||
tokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, tokenIndex)
|
||||
if err != nil {
|
||||
lo.Error("could not create token index iter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := tokenIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next token index batch", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", tokenIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if poolIndex := registryMap[celoutils.PoolIndex]; poolIndex != celoutils.ZeroAddress {
|
||||
poolIndexIter, err := chain.Provider().NewBatchIterator(ctx, poolIndex)
|
||||
if err != nil {
|
||||
lo.Error("cache could create pool index iter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := poolIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next pool index batch", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", poolIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err, "address", address.Hex())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var poolTokenIndex, priceQuoter common.Address
|
||||
err := chain.Provider().Client.CallCtx(
|
||||
ctx,
|
||||
eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex),
|
||||
eth.CallFunc(address, quoterGetter).Returns(&priceQuoter),
|
||||
)
|
||||
if err != nil {
|
||||
lo.Error("error fetching pool token index and/or quoter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if priceQuoter != celoutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, priceQuoter.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if poolTokenIndex != celoutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, poolTokenIndex.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
poolTokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, poolTokenIndex)
|
||||
if err != nil {
|
||||
lo.Error("error creating pool token index iter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := poolTokenIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next pool token index batch", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", poolTokenIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if address != celoutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if accountsIndex := registryMap[celoutils.AccountIndex]; accountsIndex != celoutils.ZeroAddress {
|
||||
accountsIndexIter, err := chain.Provider().NewBatchIterator(ctx, accountsIndex)
|
||||
if err != nil {
|
||||
lo.Error("could not create accounts index iter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := accountsIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next accounts index batch", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", accountsIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range registryMap {
|
||||
if err := cache.Add(ctx, v.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, address := range ko.MustStrings("bootstrap.watchlist") {
|
||||
if err := cache.Add(ctx, address); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
for _, address := range ko.MustStrings("bootstrap.blacklist") {
|
||||
if err := cache.Remove(ctx, address); err != nil {
|
||||
lo.Error("redis error deleting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cache.Remove(ctx, celoutils.ZeroAddress.Hex()); err != nil {
|
||||
lo.Error("redis error deleting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cacheSize, err := cache.Size(ctx)
|
||||
if err != nil {
|
||||
lo.Warn("error fetching cache size")
|
||||
}
|
||||
|
||||
lo.Info("redis cache bootstrap complete", "addresses_cached", cacheSize)
|
||||
}
|
||||
209
cmd/service/main.go
Normal file
209
cmd/service/main.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/grassrootseconomics/celo-tracker/db"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/api"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/backfill"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/cache"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/chain"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/pool"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/processor"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/pub"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/router"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/stats"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/syncer"
|
||||
"github.com/grassrootseconomics/celo-tracker/internal/util"
|
||||
"github.com/knadh/koanf/v2"
|
||||
)
|
||||
|
||||
const defaultGracefulShutdownPeriod = time.Second * 30
|
||||
|
||||
var (
|
||||
build = "dev"
|
||||
|
||||
confFlag string
|
||||
|
||||
lo *slog.Logger
|
||||
ko *koanf.Koanf
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&confFlag, "config", "config.toml", "Config file location")
|
||||
flag.Parse()
|
||||
|
||||
lo = util.InitLogger()
|
||||
ko = util.InitConfig(lo, confFlag)
|
||||
|
||||
lo.Info("starting celo tracker", "build", build)
|
||||
}
|
||||
|
||||
func main() {
|
||||
var wg sync.WaitGroup
|
||||
ctx, stop := notifyShutdown()
|
||||
|
||||
chain, err := chain.NewRPCFetcher(chain.CeloRPCOpts{
|
||||
RPCEndpoint: ko.MustString("chain.rpc_endpoint"),
|
||||
ChainID: ko.MustInt64("chain.chainid"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize chain client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
db, err := db.New(db.DBOpts{
|
||||
Logg: lo,
|
||||
DBType: ko.MustString("core.db_type"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize blocks db", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cache, err := cache.New(cache.CacheOpts{
|
||||
Logg: lo,
|
||||
CacheType: ko.MustString("core.cache_type"),
|
||||
RedisDSN: ko.MustString("redis.dsn"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize cache", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
jetStreamPub, err := pub.NewJetStreamPub(pub.JetStreamOpts{
|
||||
Endpoint: ko.MustString("jetstream.endpoint"),
|
||||
PersistDuration: time.Duration(ko.MustInt("jetstream.persist_duration_hrs")) * time.Hour,
|
||||
DedupDuration: time.Duration(ko.MustInt("jetstream.dedup_duration_hrs")) * time.Hour,
|
||||
Logg: lo,
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize jetstream pub", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
router := router.New(router.RouterOpts{
|
||||
Pub: jetStreamPub,
|
||||
Cache: cache,
|
||||
})
|
||||
|
||||
blockProcessor := processor.NewProcessor(processor.ProcessorOpts{
|
||||
Cache: cache,
|
||||
Chain: chain,
|
||||
DB: db,
|
||||
Router: router,
|
||||
Logg: lo,
|
||||
})
|
||||
|
||||
poolOpts := pool.PoolOpts{
|
||||
Logg: lo,
|
||||
WorkerCount: ko.Int("core.pool_size"),
|
||||
Processor: blockProcessor,
|
||||
}
|
||||
if ko.Int("core.pool_size") <= 0 {
|
||||
poolOpts.WorkerCount = runtime.NumCPU() * 3
|
||||
}
|
||||
workerPool := pool.New(poolOpts)
|
||||
|
||||
stats := stats.New(stats.StatsOpts{
|
||||
Cache: cache,
|
||||
Logg: lo,
|
||||
Pool: workerPool,
|
||||
})
|
||||
|
||||
chainSyncer, err := syncer.New(syncer.SyncerOpts{
|
||||
DB: db,
|
||||
Chain: chain,
|
||||
Logg: lo,
|
||||
Pool: workerPool,
|
||||
Stats: stats,
|
||||
StartBlock: ko.Int64("chain.start_block"),
|
||||
WebSocketEndpoint: ko.MustString("chain.ws_endpoint"),
|
||||
})
|
||||
if err != nil {
|
||||
lo.Error("could not initialize chain syncer", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
backfill := backfill.New(backfill.BackfillOpts{
|
||||
DB: db,
|
||||
Logg: lo,
|
||||
Pool: workerPool,
|
||||
})
|
||||
|
||||
apiServer := &http.Server{
|
||||
Addr: ko.MustString("api.address"),
|
||||
Handler: api.New(),
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
chainSyncer.Start()
|
||||
}()
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if err := backfill.Run(false); err != nil {
|
||||
lo.Error("backfiller initial run error", "error", err)
|
||||
}
|
||||
backfill.Start()
|
||||
}()
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
lo.Info("metrics and stats server starting", "address", ko.MustString("api.address"))
|
||||
if err := apiServer.ListenAndServe(); err != http.ErrServerClosed {
|
||||
lo.Error("failed to start API server", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
lo.Info("shutdown signal received")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), defaultGracefulShutdownPeriod)
|
||||
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
chainSyncer.Stop()
|
||||
backfill.Stop()
|
||||
workerPool.Stop()
|
||||
jetStreamPub.Close()
|
||||
db.Cleanup()
|
||||
db.Close()
|
||||
apiServer.Shutdown(shutdownCtx)
|
||||
lo.Info("graceful shutdown routine complete")
|
||||
}()
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
stop()
|
||||
cancel()
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
<-shutdownCtx.Done()
|
||||
if errors.Is(shutdownCtx.Err(), context.DeadlineExceeded) {
|
||||
stop()
|
||||
cancel()
|
||||
lo.Error("graceful shutdown period exceeded, forcefully shutting down")
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func notifyShutdown() (context.Context, context.CancelFunc) {
|
||||
return signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
|
||||
}
|
||||
Reference in New Issue
Block a user