From d6157652c10cfbabfcd6f6542646450f23dc0abd Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 23 Jul 2021 05:35:24 +0200 Subject: [PATCH] Add cic-eth chainlib cli util override, implement for account create cli --- apps/cic-eth/cic_eth/cli/__init__.py | 102 ++++++++++++++++++++ apps/cic-eth/cic_eth/data/config/celery.ini | 5 + apps/cic-eth/cic_eth/data/config/cic.ini | 2 + apps/cic-eth/cic_eth/data/config/redis.ini | 5 + apps/cic-eth/cic_eth/runnable/create.py | 53 +++------- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/requirements.txt | 2 +- 7 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 apps/cic-eth/cic_eth/cli/__init__.py create mode 100644 apps/cic-eth/cic_eth/data/config/celery.ini create mode 100644 apps/cic-eth/cic_eth/data/config/cic.ini create mode 100644 apps/cic-eth/cic_eth/data/config/redis.ini diff --git a/apps/cic-eth/cic_eth/cli/__init__.py b/apps/cic-eth/cic_eth/cli/__init__.py new file mode 100644 index 00000000..43ccb6e8 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/__init__.py @@ -0,0 +1,102 @@ +# standard imports +import celery +import os +import enum +import logging + +# external imports +from chainlib.eth.cli import ( + ArgumentParser, + argflag_std_read, + argflag_std_write, + argflag_std_base, + Config as BaseConfig, + ArgumentParser as BaseArgumentParser, + Flag, + ) + +logg = logging.getLogger(__name__) + +script_dir = os.path.dirname(os.path.realpath(__file__)) + + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + CELERY_QUEUE = 2 + + # redis - nibble 2 + REDIS = 16 + REDIS_CALLBACK = 32 + +argflag_local_task = CICFlag.CELERY +argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None, logger=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} + if local_arg_flags & CICFlag.REDIS: + local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') + local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') + local_args_override['REDIS_DB'] = getattr(args, 'redis_db') + local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + config.dict_override(local_args_override, 'local cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') + config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.REDIS: + self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') + self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') + self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') + if local_arg_flags & CICFlag.REDIS_CALLBACK: + self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') + self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-eth/cic_eth/data/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini new file mode 100644 index 00000000..f52b27bd --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -0,0 +1,2 @@ +[cic] +registry_address = diff --git a/apps/cic-eth/cic_eth/data/config/redis.ini b/apps/cic-eth/cic_eth/data/config/redis.ini new file mode 100644 index 00000000..f836ebfd --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/redis.ini @@ -0,0 +1,5 @@ +[redis] +host = localhost +port = 6379 +db = 0 +timeout = 20.0 diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 973ac2f5..470ad08b 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -7,12 +7,13 @@ import json import argparse # external imports -import celery import confini import redis from xdg.BaseDirectory import xdg_config_home +from chainlib.chain import ChainSpec # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) @@ -20,41 +21,19 @@ logg = logging.getLogger('create_account_script') logging.getLogger('confini').setLevel(logging.WARNING) logging.getLogger('gnupg').setLevel(logging.WARNING) -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'no_register': None, } -config.dict_override(args_override, 'cli') - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): redis_host = config.get('REDIS_HOST') @@ -68,20 +47,20 @@ def main(): ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE'), ) - register = not args.no_register + register = not config.get('_NO_REGISTER') logg.debug('register {}'.format(register)) t = api.create_account(register=register) ps.get_message() try: - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) except TimeoutError as e: sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e)) sys.exit(1) diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 039b9baf..f9ac618b 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 2, - 'alpha.3', + 'alpha.4', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 09811a0f..0acfe26a 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.6a1,<0.1.0 +chainlib-eth>=0.0.6a2,<0.1.0 semver==2.13.0