From e1f411443cff46a38db2e759fe31867d73a82b94 Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 6 Mar 2022 19:42:01 +0000 Subject: [PATCH] Allow filter and renderers set with config, envs --- CHANGELOG | 2 + eth_monitor/config.py | 56 ++++++++++++++++++++++++++++ eth_monitor/data/config/filter.ini | 2 + eth_monitor/data/config/renderer.ini | 2 + eth_monitor/runnable/sync.py | 32 ++++++++-------- requirements.txt | 8 ++-- setup.cfg | 3 +- 7 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 eth_monitor/config.py create mode 100644 eth_monitor/data/config/filter.ini create mode 100644 eth_monitor/data/config/renderer.ini diff --git a/CHANGELOG b/CHANGELOG index fc8eb62..7b693ba 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +- 0.2.0 + * Dependency upgrades - 0.1.0 * Read blocks, tx, rcpt from cache * Read include and exclude address lists from cli options diff --git a/eth_monitor/config.py b/eth_monitor/config.py new file mode 100644 index 0000000..0ce5ab1 --- /dev/null +++ b/eth_monitor/config.py @@ -0,0 +1,56 @@ +# standard imports +import re +import logging + + +logg = logging.getLogger(__name__) + + +re_env='^ETHMONITOR_({}_(.+))$' + + +def __override_env(config, config_stem, env): + re_env_instance = re_env.format(config_stem) + for k in env: + r = re.search(re_env_instance, k) + if r != None: + logg.debug('match renderer environment variable: {} '.format(r.group(1))) + config.add(env[k], r.group(1), True) + + +def __override_arg(config, config_stem, args): + if args == None: + return + + args_array = getattr(args, config_stem.lower()) + if args_array == None: + return + + i = 0 + for a in args_array: + s = config_stem + '_ARG_' + str(i) + config.add(a, s, True) + i += 1 + + +def override(config, keyword, env={}, args=None): + config_stem = keyword.upper() + __override_arg(config, config_stem, args) + __override_env(config, config_stem, env) + + + +def list_from_prefix(config, keyword): + re_config = keyword.upper() + '_' + k_default = re_config + 'DEFAULT' + r = [] + for k in config.all(): + if re.match(re_config, k): + v = config.get(k) + if k == k_default: + try: + v = v.split(',') + except AttributeError: + continue + r.append(v) + return r diff --git a/eth_monitor/data/config/filter.ini b/eth_monitor/data/config/filter.ini new file mode 100644 index 0000000..fda399b --- /dev/null +++ b/eth_monitor/data/config/filter.ini @@ -0,0 +1,2 @@ +[filter] +default = diff --git a/eth_monitor/data/config/renderer.ini b/eth_monitor/data/config/renderer.ini new file mode 100644 index 0000000..75abc28 --- /dev/null +++ b/eth_monitor/data/config/renderer.ini @@ -0,0 +1,2 @@ +[renderer] +default = diff --git a/eth_monitor/runnable/sync.py b/eth_monitor/runnable/sync.py index cbeb871..ed7c230 100644 --- a/eth_monitor/runnable/sync.py +++ b/eth_monitor/runnable/sync.py @@ -31,6 +31,7 @@ from eth_monitor.rules import ( ) from eth_monitor.filters import RuledFilter from eth_monitor.filters.out import OutFilter +from eth_monitor.config import override, list_from_prefix logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -104,9 +105,9 @@ config.add(args.offset, '_SYNC_OFFSET', True) config.add(args.skip_history, '_NO_HISTORY', True) config.add(args.single, '_SINGLE', True) config.add(args.head, '_HEAD', True) -logg.debug('loaded config:\{}'.format(config)) - -logg.debug('config loaded:\n{}'.format(config)) +override(config, 'renderer', env=os.environ, args=args) +override(config, 'filter', env=os.environ, args=args) +logg.debug('loaded config:\n{}'.format(config)) chain_spec = ChainSpec.from_chain_str(args.i) @@ -326,12 +327,18 @@ def main(): cache_filter, ] - if args.filter != None: - for fltr in args.filter: - m = importlib.import_module(fltr) - fltr_object = m.Filter(rules_filter=address_rules) - filters.append(fltr_object) - + for fltr in list_from_prefix(config, 'filter'): + m = importlib.import_module(fltr) + fltr_object = m.Filter(rules_filter=address_rules) + filters.append(fltr_object) + logg.info('using filter module {}'.format(fltr)) + + renderers_mods = [] + for renderer in list_from_prefix(config, 'renderer'): + m = importlib.import_module(renderer) + renderers_mods.append(m) + logg.info('using renderer module {}'.format(renderer)) + syncer_setup_func = None if config.true('_HEAD'): syncer_setup_func = setup_backend_head @@ -352,18 +359,13 @@ def main(): skip_history=config.true('_NO_HISTORY'), ) - renderers_mods = [] - for renderer in args.renderer: - m = importlib.import_module(renderer) - renderers_mods.append(m) - out_filter = OutFilter(chain_spec, rules_filter=address_rules, renderers=renderers_mods) filters.append(out_filter) use_rpc = rpc if not args.fresh: use_rpc = CacheRPC(rpc, store) - + i = 0 for syncer in syncers: logg.info('running syncer index {} {}'.format(i, str(syncer))) diff --git a/requirements.txt b/requirements.txt index ceddf05..be6b340 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -chainlib-eth>=0.1.0b3,<=0.1.0 -chainlib~=0.0.23 -chainsyncer~=0.1.0 -eth-erc20~=0.1.11 +chainlib-eth>=0.1.0b4,<=0.1.0 +chainlib>=0.1.0b1,<=0.1.0 +chainsyncer~=0.2.0 +eth-erc20~=0.2.0 leveldir~=0.3.0 eth-cache~=0.1.0 diff --git a/setup.cfg b/setup.cfg index 126cc1d..70f0368 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eth-monitor -version = 0.1.0rc1 +version = 0.2.0 description = Monitor and cache transactions using match filters author = Louis Holbrook author_email = dev@holbrook.no @@ -31,7 +31,6 @@ packages = eth_monitor eth_monitor.importers eth_monitor.filters - eth_monitor.store eth_monitor.runnable eth_monitor.mock