From d88ae00b723e3d3727420df33f54e905fbe79621 Mon Sep 17 00:00:00 2001 From: nolash Date: Sun, 31 Oct 2021 07:58:35 +0100 Subject: [PATCH] Add celery cli args with defaults from redis --- apps/cic-eth/cic_eth/cli/arg.py | 16 +++-- apps/cic-eth/cic_eth/cli/base.py | 4 +- apps/cic-eth/cic_eth/cli/config.py | 65 +++++++++++++++++++-- apps/cic-eth/cic_eth/data/config/celery.ini | 2 +- apps/cic-eth/cic_eth/eth/util.py | 7 ++- apps/cic-eth/cic_eth/runnable/transfer.py | 2 +- apps/cic-eth/cic_eth/runnable/view.py | 54 ----------------- apps/cic-eth/requirements.txt | 4 +- apps/cic-eth/setup.cfg | 2 +- 9 files changed, 83 insertions(+), 73 deletions(-) diff --git a/apps/cic-eth/cic_eth/cli/arg.py b/apps/cic-eth/cic_eth/cli/arg.py index 7604b72b..acdbc632 100644 --- a/apps/cic-eth/cic_eth/cli/arg.py +++ b/apps/cic-eth/cic_eth/cli/arg.py @@ -16,16 +16,22 @@ class ArgumentParser(BaseArgumentParser): self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') if local_arg_flags & CICFlag.REDIS_CALLBACK: - self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') - self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)') + self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)') self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') if local_arg_flags & CICFlag.CELERY: + self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")') + self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)') + self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)') + self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)') + self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)') + self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)') + self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)') + self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)') + self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend') self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') if local_arg_flags & CICFlag.SYNCER: self.add_argument('--offset', type=int, help='Start block height for initial history sync') self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') if local_arg_flags & CICFlag.CHAIN: self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') - - - diff --git a/apps/cic-eth/cic_eth/cli/base.py b/apps/cic-eth/cic_eth/cli/base.py index b8b3ef2f..71108b29 100644 --- a/apps/cic-eth/cic_eth/cli/base.py +++ b/apps/cic-eth/cic_eth/cli/base.py @@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum): # sync - nibble 4 SYNCER = 4096 - -argflag_local_task = CICFlag.CELERY +argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC +argflag_local_task = CICFlag.CELERY argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK argflag_local_chain = CICFlag.CHAIN argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-eth/cic_eth/cli/config.py b/apps/cic-eth/cic_eth/cli/config.py index 8bfadc67..24ffcf11 100644 --- a/apps/cic-eth/cic_eth/cli/config.py +++ b/apps/cic-eth/cic_eth/cli/config.py @@ -1,12 +1,18 @@ # standard imports import os import logging +import urllib.parse +import copy # external imports from chainlib.eth.cli import ( Config as BaseConfig, Flag, ) +from urlybird.merge import ( + urlhostmerge, + urlmerge, + ) # local imports from .base import CICFlag @@ -40,6 +46,7 @@ class Config(BaseConfig): if local_arg_flags & CICFlag.CHAIN: local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + if local_arg_flags & CICFlag.CELERY: local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') @@ -49,15 +56,61 @@ class Config(BaseConfig): config.dict_override(local_args_override, 'local cli args') - if local_arg_flags & CICFlag.REDIS_CALLBACK: - config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') - config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') - + local_celery_args_override = {} if local_arg_flags & CICFlag.CELERY: + hostport = urlhostmerge( + None, + config.get('REDIS_HOST'), + config.get('REDIS_PORT'), + ) + redis_url = ( + 'redis', + hostport, + getattr(args, 'redis_db', None), + ) + celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL')) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_host', None), + getattr(args, 'celery_port', None), + ) + celery_arg_url = ( + getattr(args, 'celery_scheme', None), + hostport, + getattr(args, 'celery_db', None), + ) + celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string + if not getattr(args, 'celery_no_result'): + local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL') + if local_celery_args_override['CELERY_RESULT_URL'] == None: + local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL'] + celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL']) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_result_host', None), + getattr(args, 'celery_result_port', None), + ) + celery_arg_url = ( + getattr(args, 'celery_result_scheme', None), + hostport, + getattr(args, 'celery_result_db', None), + ) + celery_url = urlmerge(celery_config_url, celery_arg_url) + logg.debug('celery url {} {}'.format(celery_config_url, celery_url)) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + config.dict_override(local_celery_args_override, 'local celery cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST')) + redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT')) + config.add(redis_host_callback, '_REDIS_HOST_CALLBACK') + config.add(redis_port_callback, '_REDIS_PORT_CALLBACK') + logg.debug('config loaded:\n{}'.format(config)) return config - - diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini index f2ad10ab..5b68ee98 100644 --- a/apps/cic-eth/cic_eth/data/config/celery.ini +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -1,5 +1,5 @@ [celery] -broker_url = redis://localhost:6379 +broker_url = result_url = queue = cic-eth debug = 0 diff --git a/apps/cic-eth/cic_eth/eth/util.py b/apps/cic-eth/cic_eth/eth/util.py index 5f59b75e..6720c669 100644 --- a/apps/cic-eth/cic_eth/eth/util.py +++ b/apps/cic-eth/cic_eth/eth/util.py @@ -1,3 +1,6 @@ +# standard imports +import logging + # external imports from chainlib.eth.gas import RPCGasOracle from hexathon import strip_0x @@ -7,9 +10,11 @@ from cic_eth.db.models.gas_cache import GasCache from cic_eth.encode import tx_normalize from cic_eth.db.models.base import SessionBase - MAXIMUM_FEE_UNITS = 8000000 +logg = logging.getLogger(__name__) + + class MaxGasOracle(RPCGasOracle): def get_fee_units(self, code=None): diff --git a/apps/cic-eth/cic_eth/runnable/transfer.py b/apps/cic-eth/cic_eth/runnable/transfer.py index 4bbaa306..67c8314c 100644 --- a/apps/cic-eth/cic_eth/runnable/transfer.py +++ b/apps/cic-eth/cic_eth/runnable/transfer.py @@ -18,7 +18,7 @@ from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger('create_account_script') -arg_flags = cic_eth.cli.argflag_std_base +arg_flags = cic_eth.cli.argflag_local_base local_arg_flags = cic_eth.cli.argflag_local_taskcallback argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol') diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index 42281980..bdc40c33 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -46,27 +46,6 @@ argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -#default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -#argparser = argparse.ArgumentParser() -#argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -#argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address') -#argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format') -#argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only') -#argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -#argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -#argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -#argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -#argparser.add_argument('-v', action='store_true', help='Be verbose') -#argparser.add_argument('-vv', help='be more verbose', action='store_true') -#argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"') -#args = argparser.parse_args() - -#if args.v == True: -# logging.getLogger().setLevel(logging.INFO) -#elif args.vv == True: -# logging.getLogger().setLevel(logging.DEBUG) -# extra_args = { 'f': '_FORMAT', 'query': '_QUERY', @@ -74,38 +53,10 @@ extra_args = { config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) celery_app = cic_eth.cli.CeleryApp.from_config(config) - - -#config_dir = os.path.join(args.c) -#os.makedirs(config_dir, 0o777, True) -#config = confini.Config(config_dir, args.env_prefix) -#config.process() -#args_override = { -# 'ETH_PROVIDER': getattr(args, 'p'), -# 'CIC_CHAIN_SPEC': getattr(args, 'i'), -# 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), -# } -## override args -#config.dict_override(args_override, 'cli args') -#config.censor('PASSWORD', 'DATABASE') -#config.censor('PASSWORD', 'SSL') -#logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -# -#try: -# config.add(add_0x(args.query), '_QUERY', True) -#except: -# config.add(args.query, '_QUERY', True) - -#celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - queue = config.get('CELERY_QUEUE') chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -#rpc = EthHTTPConnection(args.p) - -#registry_address = config.get('CIC_REGISTRY_ADDRESS') - # connect to celery celery_app = cic_eth.cli.CeleryApp.from_config(config) @@ -194,16 +145,11 @@ def main(): pass if len(query) > 64: - #registry = connect_registry(rpc, chain_spec, registry_address) - #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer) admin_api.tx(chain_spec, tx_raw=query, renderer=renderer) elif len(query) > 40: - #registry = connect_registry(rpc, chain_spec, registry_address) - #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer) admin_api.tx(chain_spec, tx_hash=query, renderer=renderer) elif len(query) == 40: - #registry = connect_registry(rpc, chain_spec, registry_address) txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account) renderer = render_account elif len(query) >= 4 and query[:4] == 'lock': diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 7a996c44..b1097b28 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ celery==4.4.7 -chainlib-eth>=0.0.10a16,<0.1.0 +chainlib-eth>=0.0.10a20,<0.1.0 semver==2.13.0 -crypto-dev-signer>=0.4.15rc2,<0.5.0 +urlybird~=0.0.1a2 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index f9494df9..dc94ccac 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = cic-eth #version = attr: cic_eth.version.__version_string__ -version = 0.12.4a14 +version = 0.12.4a15 description = CIC Network Ethereum interaction author = Louis Holbrook author_email = dev@holbrook.no