mirror of
https://github.com/grassrootseconomics/eth-tracker.git
synced 2025-02-22 16:00:37 +01:00
fix: rewrite bootstrap script from scratch to verify correctness/fix inconsistency
* same size when using map and redis * same size when tested against different RPC nodes * closes #38
This commit is contained in:
parent
6057208b50
commit
1c7c53a12f
GPG Key ID:
7DD45520C01CD85D
@ -17,8 +17,6 @@ import (
|
||||
"github.com/lmittmann/w3/module/eth"
|
||||
)
|
||||
|
||||
const cacheType = "redis"
|
||||
|
||||
var (
|
||||
build = "dev"
|
||||
|
||||
@ -39,6 +37,13 @@ func init() {
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := bootstrapCache(); err != nil {
|
||||
lo.Error("critical error bootstrapping cache", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func bootstrapCache() error {
|
||||
var (
|
||||
tokenRegistryGetter = w3.MustNewFunc("tokenRegistry()", "address")
|
||||
quoterGetter = w3.MustNewFunc("quoter()", "address")
|
||||
@ -55,7 +60,7 @@ func main() {
|
||||
|
||||
cache, err := cache.New(cache.CacheOpts{
|
||||
Logg: lo,
|
||||
CacheType: cacheType,
|
||||
CacheType: ko.MustString("core.cache_type"),
|
||||
RedisDSN: ko.MustString("redis.dsn"),
|
||||
})
|
||||
if err != nil {
|
||||
@ -73,164 +78,176 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if tokenIndex := registryMap[ethutils.TokenIndex]; tokenIndex != ethutils.ZeroAddress {
|
||||
tokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, tokenIndex)
|
||||
if err != nil {
|
||||
lo.Error("could not create token index iter", "error", err)
|
||||
os.Exit(1)
|
||||
for k, v := range registryMap {
|
||||
if v != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, v.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lo.Debug("cached registry entry", "type", k, "address", v.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
if accountIndex := registryMap[ethutils.AccountIndex]; accountIndex != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, accountIndex.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lo.Debug("cached account index", "address", accountIndex.Hex())
|
||||
|
||||
accountIndexIter, err := chain.Provider().NewBatchIterator(ctx, accountIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
batch, err := tokenIndexIter.Next(ctx)
|
||||
accountIndexBatch, err := accountIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next token index batch", "error", err)
|
||||
os.Exit(1)
|
||||
return err
|
||||
}
|
||||
if batch == nil {
|
||||
if accountIndexBatch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", tokenIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
|
||||
for _, address := range accountIndexBatch {
|
||||
if address != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
lo.Debug("cached account index batch", "batch_size", len(accountIndexBatch))
|
||||
}
|
||||
}
|
||||
|
||||
if tokenIndex := registryMap[ethutils.TokenIndex]; tokenIndex != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, tokenIndex.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lo.Debug("cached token index", "address", tokenIndex.Hex())
|
||||
|
||||
tokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, tokenIndex)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
tokenIndexBatch, err := tokenIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tokenIndexBatch == nil {
|
||||
break
|
||||
}
|
||||
|
||||
for _, address := range tokenIndexBatch {
|
||||
if address != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
lo.Debug("cached token index batch", "batch_size", len(tokenIndexBatch))
|
||||
}
|
||||
}
|
||||
|
||||
if poolIndex := registryMap[ethutils.PoolIndex]; poolIndex != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, poolIndex.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lo.Debug("cached pool index", "address", poolIndex.Hex())
|
||||
|
||||
poolIndexIter, err := chain.Provider().NewBatchIterator(ctx, poolIndex)
|
||||
if err != nil {
|
||||
lo.Error("cache could create pool index iter", "error", err)
|
||||
os.Exit(1)
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := poolIndexIter.Next(ctx)
|
||||
poolIndexBatch, err := poolIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next pool index batch", "error", err)
|
||||
os.Exit(1)
|
||||
return err
|
||||
}
|
||||
if batch == nil {
|
||||
if poolIndexBatch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", poolIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err, "address", address.Hex())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
var poolTokenIndex, priceQuoter common.Address
|
||||
err := chain.Provider().Client.CallCtx(
|
||||
ctx,
|
||||
eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex),
|
||||
eth.CallFunc(address, quoterGetter).Returns(&priceQuoter),
|
||||
)
|
||||
if err != nil {
|
||||
lo.Error("error fetching pool token index and/or quoter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if priceQuoter != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, priceQuoter.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if poolTokenIndex != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, poolTokenIndex.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
for _, address := range poolIndexBatch {
|
||||
if address != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
poolTokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, poolTokenIndex)
|
||||
var poolTokenIndex, priceQuoter common.Address
|
||||
err := chain.Provider().Client.CallCtx(
|
||||
ctx,
|
||||
eth.CallFunc(address, tokenRegistryGetter).Returns(&poolTokenIndex),
|
||||
eth.CallFunc(address, quoterGetter).Returns(&priceQuoter),
|
||||
)
|
||||
if err != nil {
|
||||
lo.Error("error creating pool token index iter", "error", err)
|
||||
os.Exit(1)
|
||||
return err
|
||||
}
|
||||
if priceQuoter != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, priceQuoter.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := poolTokenIndexIter.Next(ctx)
|
||||
lo.Debug("cached pool index quoter", "pool", poolIndex.Hex(), "address", priceQuoter.Hex())
|
||||
}
|
||||
if poolTokenIndex != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, poolTokenIndex.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
lo.Debug("cached pool index token index", "pool", poolIndex.Hex(), "address", poolTokenIndex.Hex())
|
||||
|
||||
poolTokenIndexIter, err := chain.Provider().NewBatchIterator(ctx, poolTokenIndex)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next pool token index batch", "error", err)
|
||||
os.Exit(1)
|
||||
return err
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", poolTokenIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if address != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
for {
|
||||
poolTokenIndexBatch, err := poolTokenIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if poolTokenIndexBatch == nil {
|
||||
break
|
||||
}
|
||||
for _, address := range poolTokenIndexBatch {
|
||||
if address != ethutils.ZeroAddress {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
lo.Debug("cached pool token index batch", "batch_size", len(poolTokenIndexBatch))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lo.Debug("cached pool index batch", "batch_size", len(poolIndexBatch))
|
||||
}
|
||||
}
|
||||
|
||||
if accountsIndex := registryMap[ethutils.AccountIndex]; accountsIndex != ethutils.ZeroAddress {
|
||||
accountsIndexIter, err := chain.Provider().NewBatchIterator(ctx, accountsIndex)
|
||||
if err != nil {
|
||||
lo.Error("could not create accounts index iter", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for {
|
||||
batch, err := accountsIndexIter.Next(ctx)
|
||||
if err != nil {
|
||||
lo.Error("error fetching next accounts index batch", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if batch == nil {
|
||||
break
|
||||
}
|
||||
lo.Debug("index batch", "index", accountsIndex.Hex(), "size", len(batch))
|
||||
for _, address := range batch {
|
||||
if err := cache.Add(ctx, address.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
for _, address := range ko.MustStrings("bootstrap.watchlist") {
|
||||
if err := cache.Add(ctx, address); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range registryMap {
|
||||
if err := cache.Add(ctx, v.Hex()); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
for _, address := range ko.MustStrings("bootstrap.blacklist") {
|
||||
if err := cache.Remove(ctx, address); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, address := range ko.MustStrings("bootstrap.watchlist") {
|
||||
if err := cache.Add(ctx, address); err != nil {
|
||||
lo.Error("redis error setting key", "error", err)
|
||||
os.Exit(1)
|
||||
if err := cache.Remove(ctx, ethutils.ZeroAddress.Hex()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, address := range ko.MustStrings("bootstrap.blacklist") {
|
||||
if err := cache.Remove(ctx, address); err != nil {
|
||||
lo.Error("redis error deleting key", "error", err)
|
||||
os.Exit(1)
|
||||
cacheSize, err := cache.Size(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lo.Info("registry bootstrap complete", "registry", registry, "current_cache_size", cacheSize)
|
||||
}
|
||||
|
||||
if err := cache.Remove(ctx, ethutils.ZeroAddress.Hex()); err != nil {
|
||||
lo.Error("redis error deleting key", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cacheSize, err := cache.Size(ctx)
|
||||
if err != nil {
|
||||
lo.Warn("error fetching cache size")
|
||||
}
|
||||
|
||||
lo.Info("redis cache bootstrap complete", "addresses_cached", cacheSize)
|
||||
return nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user