From 5853f6bb9ac5f7f769a7fd468b0e1ccbb95e4461 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 20 Apr 2022 17:42:11 +0000 Subject: [PATCH] Add rocksdb backend --- CHANGELOG | 2 ++ eth_monitor/data/config/syncer.ini | 1 + eth_monitor/runnable/sync.py | 30 ++++++++++++++++++++++++++++-- requirements.txt | 6 +++--- setup.cfg | 4 ++-- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e0863be..a25f4c7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +- 0.4.0 + * Add rocksdb backend - 0.3.3 * Upgrade chainsyncer to shep 0.2.0 - 0.3.2 diff --git a/eth_monitor/data/config/syncer.ini b/eth_monitor/data/config/syncer.ini index 42e7cd8..e0fbf04 100644 --- a/eth_monitor/data/config/syncer.ini +++ b/eth_monitor/data/config/syncer.ini @@ -1,2 +1,3 @@ [syncer] loop_interval = 5 +backend = fs diff --git a/eth_monitor/runnable/sync.py b/eth_monitor/runnable/sync.py index 6af61a2..778de15 100644 --- a/eth_monitor/runnable/sync.py +++ b/eth_monitor/runnable/sync.py @@ -17,7 +17,7 @@ from hexathon import ( strip_0x, add_0x, ) -from chainsyncer.store.fs import SyncFsStore +#from chainsyncer.store.fs import SyncFsStore from chainsyncer.driver.chain_interface import ChainInterfaceDriver from chainsyncer.error import SyncDone @@ -84,11 +84,21 @@ argparser.add_argument('--state-dir', dest='state_dir', default=exec_dir, type=s argparser.add_argument('--fresh', action='store_true', help='Do not read block and tx data from cache, even if available') argparser.add_argument('--single', action='store_true', help='Execute a single sync, regardless of previous states') argparser.add_argument('--session-id', dest='session_id', type=str, help='Use state from specified session id') +argparser.add_argument('--backend', type=str, help='State store backend') +argparser.add_argument('--list-backends', dest='list_backends', action='store_true', help='List built-in store backends') argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose') argparser.add_argument('-vvv', action='store_true', help='Be incredibly verbose') args = argparser.parse_args(sys.argv[1:]) +if args.list_backends: + for v in [ + 'fs', + 'rocksdb', + ]: + print(v) + sys.exit(0) + if args.vvv: logg.setLevel(logging.STATETRACE) else: @@ -110,6 +120,7 @@ config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), o config.process() args_override = { 'CHAIN_SPEC': getattr(args, 'i'), + 'SYNCER_BACKEND': getattr(args, 'backend'), } config.dict_override(args_override, 'cli') config.add(args.offset, '_SYNC_OFFSET', True) @@ -383,7 +394,22 @@ def main(): out_filter = OutFilter(chain_spec, rules_filter=address_rules, renderers=renderers_mods) filters.append(out_filter) - sync_store = SyncFsStore(config.get('_STATE_DIR'), session_id=config.get('_SESSION_ID'), state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback) + syncer_store_module = None + syncer_store_class = None + if config.get('SYNCER_BACKEND') == 'fs': + syncer_store_module = importlib.import_module('chainsyncer.store.fs') + syncer_store_class = getattr(syncer_store_module, 'SyncFsStore') + elif config.get('SYNCER_BACKEND') == 'rocksdb': + syncer_store_module = importlib.import_module('chainsyncer.store.rocksdb') + syncer_store_class = getattr(syncer_store_module, 'SyncRocksDbStore') + else: + syncer_store_module = importlib.import_module(config.get('SYNCER_BACKEND')) + syncer_store_class = getattr(syncer_store_module, 'SyncStore') + + logg.info('using engine {} module {}.{}'.format(config.get('SYNCER_BACKEND'), syncer_store_module.__file__, syncer_store_class.__name__)) + + state_dir = os.path.join(config.get('_STATE_DIR'), config.get('SYNCER_BACKEND')) + sync_store = syncer_store_class(state_dir, session_id=config.get('_SESSION_ID'), state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback) logg.info('session is {}'.format(sync_store.session_id)) for fltr in filters: diff --git a/requirements.txt b/requirements.txt index 2cb58fa..fd6b644 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -chainlib-eth>=0.1.0b4,<=0.1.0 -chainlib>=0.1.0b1,<=0.1.0 -chainsyncer~=0.3.1 +chainlib-eth>=0.1.0b4,<0.2.0 +chainlib>=0.1.0b1,<0.2.0 +chainsyncer~=0.3.2 eth-erc20~=0.2.0 leveldir~=0.3.0 eth-cache~=0.1.0 diff --git a/setup.cfg b/setup.cfg index 60bde6b..d41806d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eth-monitor -version = 0.3.3 +version = 0.4.0 description = Monitor and cache transactions using match filters author = Louis Holbrook author_email = dev@holbrook.no @@ -26,7 +26,7 @@ licence_files = [options] include_package_data = True -python_requires = >= 3.7 +python_requires = >=3.7,<3.10 packages = eth_monitor eth_monitor.importers