2021-06-26 14:04:34 +02:00
|
|
|
|
# standard imports
|
|
|
|
|
import sys
|
|
|
|
|
import signal
|
|
|
|
|
import argparse
|
|
|
|
|
import confini
|
|
|
|
|
import logging
|
|
|
|
|
import os
|
2022-01-30 20:44:03 +01:00
|
|
|
|
import importlib
|
2022-03-30 10:11:46 +02:00
|
|
|
|
import uuid
|
2022-03-31 19:32:25 +02:00
|
|
|
|
import datetime
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
|
|
|
|
# external imports
|
2022-05-10 13:07:25 +02:00
|
|
|
|
import chainlib.cli
|
|
|
|
|
import chainsyncer.cli
|
2021-06-26 14:04:34 +02:00
|
|
|
|
from chainlib.chain import ChainSpec
|
|
|
|
|
from chainlib.eth.connection import EthHTTPConnection
|
|
|
|
|
from chainlib.eth.block import block_latest
|
|
|
|
|
from hexathon import (
|
|
|
|
|
strip_0x,
|
|
|
|
|
add_0x,
|
|
|
|
|
)
|
2022-04-20 19:42:11 +02:00
|
|
|
|
#from chainsyncer.store.fs import SyncFsStore
|
2022-03-30 10:11:46 +02:00
|
|
|
|
from chainsyncer.driver.chain_interface import ChainInterfaceDriver
|
|
|
|
|
from chainsyncer.error import SyncDone
|
|
|
|
|
|
2022-03-01 13:10:35 +01:00
|
|
|
|
from eth_cache.rpc import CacheRPC
|
|
|
|
|
from eth_cache.store.file import FileStore
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
|
|
|
|
# local imports
|
2022-01-23 23:07:59 +01:00
|
|
|
|
from eth_monitor.filters.cache import Filter as CacheFilter
|
2022-02-27 12:48:15 +01:00
|
|
|
|
from eth_monitor.rules import (
|
|
|
|
|
AddressRules,
|
|
|
|
|
RuleSimple,
|
2022-04-05 13:44:15 +02:00
|
|
|
|
RuleMethod,
|
2022-04-05 16:02:18 +02:00
|
|
|
|
RuleData,
|
2022-02-27 12:48:15 +01:00
|
|
|
|
)
|
2022-01-23 18:49:04 +01:00
|
|
|
|
from eth_monitor.filters import RuledFilter
|
2022-01-30 15:43:53 +01:00
|
|
|
|
from eth_monitor.filters.out import OutFilter
|
2022-03-06 20:42:01 +01:00
|
|
|
|
from eth_monitor.config import override, list_from_prefix
|
2022-05-10 16:21:49 +02:00
|
|
|
|
from eth_monitor.callback import (
|
|
|
|
|
BlockCallbackFilter,
|
|
|
|
|
pre_callback,
|
|
|
|
|
post_callback,
|
|
|
|
|
)
|
2022-05-10 13:07:25 +02:00
|
|
|
|
from eth_monitor.settings import EthMonitorSettings
|
|
|
|
|
import eth_monitor.cli
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2022-04-02 13:29:12 +02:00
|
|
|
|
logging.STATETRACE = 5
|
2021-06-26 14:04:34 +02:00
|
|
|
|
logging.basicConfig(level=logging.WARNING)
|
|
|
|
|
logg = logging.getLogger()
|
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
#default_eth_provider = os.environ.get('RPC_PROVIDER')
|
|
|
|
|
#if default_eth_provider == None:
|
|
|
|
|
# default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
#exec_dir = os.path.realpath(os.getcwd())
|
2022-01-30 20:44:03 +01:00
|
|
|
|
#default_config_dir = os.environ.get('CONFINI_DIR', os.path.join(exec_dir, 'config'))
|
2022-05-10 13:07:25 +02:00
|
|
|
|
script_dir = os.path.realpath(os.path.dirname(__file__))
|
|
|
|
|
config_dir = os.path.join(script_dir, '..', 'data', 'config')
|
|
|
|
|
|
|
|
|
|
arg_flags = chainlib.cli.argflag_std_base | chainlib.cli.Flag.CHAIN_SPEC | chainlib.cli.Flag.PROVIDER
|
|
|
|
|
argparser = chainlib.cli.ArgumentParser(arg_flags)
|
|
|
|
|
eth_monitor.cli.process_flags(argparser, 0)
|
|
|
|
|
|
2022-01-24 01:17:27 +01:00
|
|
|
|
argparser.add_argument('--store-tx-data', dest='store_tx_data', action='store_true', help='Include all transaction data objects by default')
|
|
|
|
|
argparser.add_argument('--store-block-data', dest='store_block_data', action='store_true', help='Include all block data objects by default')
|
2022-02-27 14:52:05 +01:00
|
|
|
|
argparser.add_argument('--fresh', action='store_true', help='Do not read block and tx data from cache, even if available')
|
2022-04-20 19:42:11 +02:00
|
|
|
|
argparser.add_argument('--list-backends', dest='list_backends', action='store_true', help='List built-in store backends')
|
2022-02-27 12:48:15 +01:00
|
|
|
|
argparser.add_argument('-vvv', action='store_true', help='Be incredibly verbose')
|
2022-05-10 13:07:25 +02:00
|
|
|
|
|
|
|
|
|
sync_flags = chainsyncer.cli.SyncFlag.RANGE | chainsyncer.cli.SyncFlag.HEAD
|
|
|
|
|
chainsyncer.cli.process_flags(argparser, sync_flags)
|
|
|
|
|
|
2021-06-26 14:04:34 +02:00
|
|
|
|
args = argparser.parse_args(sys.argv[1:])
|
|
|
|
|
|
2022-04-20 19:42:11 +02:00
|
|
|
|
if args.list_backends:
|
|
|
|
|
for v in [
|
|
|
|
|
'fs',
|
|
|
|
|
'rocksdb',
|
2022-04-26 10:32:56 +02:00
|
|
|
|
'mem',
|
2022-04-20 19:42:11 +02:00
|
|
|
|
]:
|
|
|
|
|
print(v)
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
2022-02-27 12:48:15 +01:00
|
|
|
|
if args.vvv:
|
2022-04-02 13:29:12 +02:00
|
|
|
|
logg.setLevel(logging.STATETRACE)
|
2022-02-27 12:48:15 +01:00
|
|
|
|
else:
|
|
|
|
|
logging.getLogger('chainlib.connection').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainlib.eth.tx').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainsyncer.driver.history').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainsyncer.driver.head').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainsyncer.backend.file').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainsyncer.backend.sql').setLevel(logging.WARNING)
|
|
|
|
|
logging.getLogger('chainsyncer.filter').setLevel(logging.WARNING)
|
|
|
|
|
|
|
|
|
|
if args.vv:
|
|
|
|
|
logg.setLevel(logging.DEBUG)
|
|
|
|
|
elif args.v:
|
|
|
|
|
logg.setLevel(logging.INFO)
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
base_config_dir = [
|
|
|
|
|
chainsyncer.cli.config_dir,
|
|
|
|
|
config_dir,
|
|
|
|
|
]
|
|
|
|
|
config = chainlib.cli.Config.from_args(args, arg_flags, base_config_dir=base_config_dir)
|
|
|
|
|
config = chainsyncer.cli.process_config(config, args, sync_flags)
|
|
|
|
|
config = eth_monitor.cli.process_config(config, args, 0)
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
settings = EthMonitorSettings()
|
|
|
|
|
settings.process(config)
|
|
|
|
|
logg.debug('loaded settings:\n{}'.format(settings))
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
#rpc_id_generator = None
|
|
|
|
|
#if args.seq:
|
|
|
|
|
# rpc_id_generator = IntSequenceGenerator()
|
|
|
|
|
|
|
|
|
|
#auth = None
|
|
|
|
|
#if os.environ.get('RPC_AUTHENTICATION') == 'basic':
|
|
|
|
|
# from chainlib.auth import BasicAuth
|
|
|
|
|
# auth = BasicAuth(os.environ['RPC_USERNAME'], os.environ['RPC_PASSWORD'])
|
|
|
|
|
#rpc = EthHTTPConnection(args.p)
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
|
|
|
|
|
2022-04-05 13:44:15 +02:00
|
|
|
|
|
|
|
|
|
|
2022-01-24 01:17:27 +01:00
|
|
|
|
def setup_filter(chain_spec, cache_dir, include_tx_data, include_block_data):
|
2022-01-30 15:11:54 +01:00
|
|
|
|
store = None
|
2022-01-23 18:28:25 +01:00
|
|
|
|
if cache_dir == None:
|
2022-01-30 15:11:54 +01:00
|
|
|
|
logg.warning('no cache dir specified, will discard everything!!')
|
2022-03-05 08:01:04 +01:00
|
|
|
|
from eth_cache.store.null import NullStore
|
2022-01-30 15:11:54 +01:00
|
|
|
|
store = NullStore()
|
|
|
|
|
else:
|
|
|
|
|
store = FileStore(chain_spec, cache_dir)
|
|
|
|
|
cache_dir = os.path.realpath(cache_dir)
|
|
|
|
|
if cache_dir == None:
|
|
|
|
|
import tempfile
|
|
|
|
|
cache_dir = tempfile.mkdtemp()
|
|
|
|
|
logg.info('using chain spec {} and store {}'.format(chain_spec, store))
|
2022-01-24 01:17:27 +01:00
|
|
|
|
RuledFilter.init(store, include_tx_data=include_tx_data, include_block_data=include_block_data)
|
2022-01-23 18:28:25 +01:00
|
|
|
|
|
2022-02-27 14:52:05 +01:00
|
|
|
|
return store
|
|
|
|
|
|
2022-01-23 18:49:04 +01:00
|
|
|
|
|
|
|
|
|
def setup_cache_filter(rules_filter=None):
|
|
|
|
|
return CacheFilter(rules_filter=rules_filter)
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2021-06-27 11:01:31 +02:00
|
|
|
|
|
2022-03-30 10:11:46 +02:00
|
|
|
|
def block_callback(block, tx):
|
2022-03-31 19:32:25 +02:00
|
|
|
|
logg.info('processing {} {}'.format(block, datetime.datetime.fromtimestamp(block.timestamp)))
|
2022-01-30 15:43:53 +01:00
|
|
|
|
|
|
|
|
|
|
2022-01-30 20:44:03 +01:00
|
|
|
|
def main():
|
2022-05-10 13:07:25 +02:00
|
|
|
|
rpc = settings.get('RPC')
|
2022-03-30 10:11:46 +02:00
|
|
|
|
o = block_latest()
|
|
|
|
|
r = rpc.do(o)
|
|
|
|
|
block_offset = int(strip_0x(r), 16) + 1
|
|
|
|
|
logg.info('network block height is {}'.format(block_offset))
|
|
|
|
|
|
2022-02-27 14:52:05 +01:00
|
|
|
|
store = setup_filter(
|
2022-05-10 13:07:25 +02:00
|
|
|
|
settings.get('CHAIN_SPEC'),
|
2022-03-30 10:11:46 +02:00
|
|
|
|
config.get('_CACHE_DIR'),
|
2022-01-24 01:17:27 +01:00
|
|
|
|
bool(args.store_tx_data),
|
|
|
|
|
bool(args.store_block_data),
|
2022-01-23 18:49:04 +01:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
cache_filter = setup_cache_filter(
|
2022-05-10 16:21:49 +02:00
|
|
|
|
rules_filter=settings.get('RULES'), #address_rules,
|
2022-01-23 18:28:25 +01:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
filters = [
|
|
|
|
|
cache_filter,
|
|
|
|
|
]
|
|
|
|
|
|
2022-03-06 20:42:01 +01:00
|
|
|
|
for fltr in list_from_prefix(config, 'filter'):
|
|
|
|
|
m = importlib.import_module(fltr)
|
2022-05-10 16:21:49 +02:00
|
|
|
|
fltr_object = m.Filter(rules_filter=settings.get('RULES'))
|
2022-03-06 20:42:01 +01:00
|
|
|
|
filters.append(fltr_object)
|
|
|
|
|
logg.info('using filter module {}'.format(fltr))
|
|
|
|
|
|
|
|
|
|
renderers_mods = []
|
|
|
|
|
for renderer in list_from_prefix(config, 'renderer'):
|
|
|
|
|
m = importlib.import_module(renderer)
|
|
|
|
|
renderers_mods.append(m)
|
|
|
|
|
logg.info('using renderer module {}'.format(renderer))
|
|
|
|
|
|
2022-05-07 13:33:35 +02:00
|
|
|
|
block_filter_handler = BlockCallbackFilter()
|
|
|
|
|
for block_filter in list_from_prefix(config, 'block_filter'):
|
|
|
|
|
m = importlib.import_module(block_filter)
|
|
|
|
|
block_filter_handler.register(m)
|
|
|
|
|
logg.info('using block filter module {}'.format(block_filter))
|
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
out_filter = OutFilter(
|
|
|
|
|
settings.get('CHAIN_SPEC'),
|
2022-05-10 16:21:49 +02:00
|
|
|
|
rules_filter=settings.get('RULES'),
|
|
|
|
|
renderers=renderers_mods,
|
2022-05-10 13:07:25 +02:00
|
|
|
|
)
|
2022-01-30 15:43:53 +01:00
|
|
|
|
filters.append(out_filter)
|
2022-03-30 10:11:46 +02:00
|
|
|
|
|
2022-05-10 13:07:25 +02:00
|
|
|
|
logg.info('session is {}'.format(settings.get('SESSION_ID')))
|
|
|
|
|
|
2022-03-30 10:11:46 +02:00
|
|
|
|
for fltr in filters:
|
2022-05-10 16:21:49 +02:00
|
|
|
|
settings.get('SYNC_STORE').register(fltr)
|
|
|
|
|
drv = ChainInterfaceDriver(
|
|
|
|
|
settings.get('SYNC_STORE'),
|
|
|
|
|
settings.get('SYNCER_INTERFACE'),
|
|
|
|
|
offset=settings.get('SYNCER_OFFSET'),
|
|
|
|
|
target=settings.get('SYNCER_LIMIT'),
|
|
|
|
|
pre_callback=pre_callback,
|
|
|
|
|
post_callback=post_callback,
|
|
|
|
|
block_callback=block_filter_handler.filter,
|
|
|
|
|
)
|
2022-02-27 15:21:09 +01:00
|
|
|
|
|
|
|
|
|
use_rpc = rpc
|
|
|
|
|
if not args.fresh:
|
|
|
|
|
use_rpc = CacheRPC(rpc, store)
|
2022-03-06 20:42:01 +01:00
|
|
|
|
|
2021-06-26 14:04:34 +02:00
|
|
|
|
i = 0
|
2022-03-30 10:11:46 +02:00
|
|
|
|
try:
|
|
|
|
|
r = drv.run(use_rpc)
|
|
|
|
|
except SyncDone as e:
|
|
|
|
|
sys.stderr.write("sync {} done at block {}\n".format(drv, e))
|
2021-06-26 14:04:34 +02:00
|
|
|
|
|
2022-03-30 10:11:46 +02:00
|
|
|
|
i += 1
|
2022-01-30 20:44:03 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|