From c55716cb9b096f75b18ed6ab8a00d7cf4b1380fc Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 24 Apr 2022 15:25:58 +0000 Subject: [PATCH] Import cic-eth cli sources --- cic_base/cli/__init__.py | 10 +++ cic_base/cli/arg.py | 39 +++++++++ cic_base/cli/base.py | 35 ++++++++ cic_base/cli/celery.py | 24 ++++++ cic_base/cli/chain.py | 21 +++++ cic_base/cli/config.py | 130 ++++++++++++++++++++++++++++++ cic_base/cli/rpc.py | 90 +++++++++++++++++++++ cic_base/data/config/celery.ini | 8 ++ cic_base/data/config/cic.ini | 6 ++ cic_base/data/config/database.ini | 10 +++ cic_base/data/config/eth.ini | 8 ++ cic_base/data/config/redis.ini | 5 ++ cic_base/data/config/server.ini | 5 ++ cic_base/data/config/signer.ini | 2 + cic_base/settings.py | 6 +- requirements.txt | 2 + 16 files changed, 398 insertions(+), 3 deletions(-) create mode 100644 cic_base/cli/__init__.py create mode 100644 cic_base/cli/arg.py create mode 100644 cic_base/cli/base.py create mode 100644 cic_base/cli/celery.py create mode 100644 cic_base/cli/chain.py create mode 100644 cic_base/cli/config.py create mode 100644 cic_base/cli/rpc.py create mode 100644 cic_base/data/config/celery.ini create mode 100644 cic_base/data/config/cic.ini create mode 100644 cic_base/data/config/database.ini create mode 100644 cic_base/data/config/eth.ini create mode 100644 cic_base/data/config/redis.ini create mode 100644 cic_base/data/config/server.ini create mode 100644 cic_base/data/config/signer.ini diff --git a/cic_base/cli/__init__.py b/cic_base/cli/__init__.py new file mode 100644 index 0000000..60eb0fb --- /dev/null +++ b/cic_base/cli/__init__.py @@ -0,0 +1,10 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp diff --git a/cic_base/cli/arg.py b/cic_base/cli/arg.py new file mode 100644 index 0000000..70d5e23 --- /dev/null +++ b/cic_base/cli/arg.py @@ -0,0 +1,39 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import CICFlag, Flag + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.REDIS: + self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') + self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') + self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') + if local_arg_flags & CICFlag.REDIS_CALLBACK: + self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)') + self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)') + self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') + if local_arg_flags & CICFlag.CELERY: + self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")') + self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)') + self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)') + self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)') + self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)') + self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)') + self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)') + self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)') + self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend') + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SERVER: + self.add_argument('--server-port', type=int, default=5000, help='Server port') + self.add_argument('--server-host', type=str, default="0.0.0.0", help='Server host') + self.add_argument('--server-workers', type=int, default=1, help='The number of worker processes for handling requests') + self.add_argument('--server-config', type=str, default=None, help='Gunicorn config file, or python module. It will override all other server args. (see https://docs.gunicorn.org/en/19.2.1/settings.html#config-file)') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') diff --git a/cic_base/cli/base.py b/cic_base/cli/base.py new file mode 100644 index 0000000..113ead3 --- /dev/null +++ b/cic_base/cli/base.py @@ -0,0 +1,35 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 + REDIS = 16 + REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + # server - nibble 5 + SERVER=65536 + +argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC +argflag_local_task = CICFlag.CELERY +argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN +argflag_local_server = CICFlag.SERVER | CICFlag.REDIS | CICFlag.REDIS_CALLBACK | CICFlag.CELERY | Flag.CHAIN_SPEC diff --git a/cic_base/cli/celery.py b/cic_base/cli/celery.py new file mode 100644 index 0000000..16180d4 --- /dev/null +++ b/cic_base/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/cic_base/cli/chain.py b/cic_base/cli/chain.py new file mode 100644 index 0000000..100139e --- /dev/null +++ b/cic_base/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/cic_base/cli/config.py b/cic_base/cli/config.py new file mode 100644 index 0000000..6ce2bb6 --- /dev/null +++ b/cic_base/cli/config.py @@ -0,0 +1,130 @@ +# standard imports +import os +import logging +import urllib.parse +import copy + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) +from urlybird.merge import ( + urlhostmerge, + urlmerge, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} + if local_arg_flags & CICFlag.REDIS: + local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') + local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') + local_args_override['REDIS_DB'] = getattr(args, 'redis_db') + local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + if local_arg_flags & CICFlag.SERVER: + local_args_override['SERVER_PORT'] = getattr(args, 'server_port') + local_args_override['SERVER_HOST'] = getattr(args, 'server_host') + local_args_override['SERVER_WORKERS'] = getattr(args, 'server_workers') + local_args_override['SERVER_CONFIG'] = getattr(args, 'server_config') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + + local_celery_args_override = {} + if local_arg_flags & CICFlag.CELERY: + hostport = urlhostmerge( + None, + config.get('REDIS_HOST'), + config.get('REDIS_PORT'), + ) + db = getattr(args, 'redis_db', None) + if db != None: + db = str(db) + + redis_url = ( + 'redis', + hostport, + db, + ) + + + celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL')) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_host', None), + getattr(args, 'celery_port', None), + ) + db = getattr(args, 'redis_db', None) + if db != None: + db = str(db) + celery_arg_url = ( + getattr(args, 'celery_scheme', None), + hostport, + db, + ) + + celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string + if not getattr(args, 'celery_no_result'): + local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL') + if local_celery_args_override['CELERY_RESULT_URL'] == None: + local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL'] + celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL']) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_result_host', None), + getattr(args, 'celery_result_port', None), + ) + celery_arg_url = ( + getattr(args, 'celery_result_scheme', None), + hostport, + getattr(args, 'celery_result_db', None), + ) + celery_url = urlmerge(celery_config_url, celery_arg_url) + logg.debug('celery url {} {}'.format(celery_config_url, celery_url)) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + config.dict_override(local_celery_args_override, 'local celery cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST')) + redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT')) + config.add(redis_host_callback, '_REDIS_HOST_CALLBACK') + config.add(redis_port_callback, '_REDIS_PORT_CALLBACK') + + logg.debug('config loaded:\n{}'.format(config)) + + return config diff --git a/cic_base/cli/rpc.py b/cic_base/cli/rpc.py new file mode 100644 index 0000000..c695402 --- /dev/null +++ b/cic_base/cli/rpc.py @@ -0,0 +1,90 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import ( + EthUnixSignerConnection, + EthHTTPSignerConnection, + ) +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return self.get_by_label('default') + + + def get_by_label(self, label): + return RPCConnection.connect(self.chain_spec, label) + + + @staticmethod + def from_config(config, use_signer=False, default_label='default', signer_label='signer'): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label) + if use_signer: + + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label) + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) + rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + +# TOOD: re-implement file backend option from omittec code: +#broker = config.get('CELERY_BROKER_URL') +#if broker[:4] == 'file': +# bq = tempfile.mkdtemp() +# bp = tempfile.mkdtemp() +# conf_update = { +# 'broker_url': broker, +# 'broker_transport_options': { +# 'data_folder_in': bq, +# 'data_folder_out': bq, +# 'data_folder_processed': bp, +# }, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) +#else: +# conf_update = { +# 'broker_url': broker, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# +#result = config.get('CELERY_RESULT_URL') +#if result[:4] == 'file': +# rq = tempfile.mkdtemp() +# current_app.conf.update({ +# 'result_backend': 'file://{}'.format(rq), +# }) +# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) +#else: +# current_app.conf.update({ +# 'result_backend': result, +# }) +# diff --git a/cic_base/data/config/celery.ini b/cic_base/data/config/celery.ini new file mode 100644 index 0000000..b6f1ea8 --- /dev/null +++ b/cic_base/data/config/celery.ini @@ -0,0 +1,8 @@ +[celery] +broker_url = +result_url = +queue = cic-eth +debug = 0 +worker_pool = prefork +worker_count = 0 +debug_log = 0 diff --git a/cic_base/data/config/cic.ini b/cic_base/data/config/cic.ini new file mode 100644 index 0000000..7d36d45 --- /dev/null +++ b/cic_base/data/config/cic.ini @@ -0,0 +1,6 @@ +[cic] +registry_address = +trust_address = +default_token_symbol = +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start +run_dir = /run diff --git a/cic_base/data/config/database.ini b/cic_base/data/config/database.ini new file mode 100644 index 0000000..7a295a5 --- /dev/null +++ b/cic_base/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = +user = +password = +debug = 0 +pool_size = 0 diff --git a/cic_base/data/config/eth.ini b/cic_base/data/config/eth.ini new file mode 100644 index 0000000..db0532e --- /dev/null +++ b/cic_base/data/config/eth.ini @@ -0,0 +1,8 @@ +[eth] +gas_holder_minimum_units = 180000 +gas_holder_refill_units = 15 +gas_holder_refill_threshold = 3 +gas_gifter_refill_buffer = 3 +gas_gift_min_price_buffer = 20 +min_fee_price = 1 +max_fee_units = 8000000 diff --git a/cic_base/data/config/redis.ini b/cic_base/data/config/redis.ini new file mode 100644 index 0000000..f836ebf --- /dev/null +++ b/cic_base/data/config/redis.ini @@ -0,0 +1,5 @@ +[redis] +host = localhost +port = 6379 +db = 0 +timeout = 20.0 diff --git a/cic_base/data/config/server.ini b/cic_base/data/config/server.ini new file mode 100644 index 0000000..8c6f3c0 --- /dev/null +++ b/cic_base/data/config/server.ini @@ -0,0 +1,5 @@ +[server] +port=5000 +host="0.0.0.0" +workers=1 +config= diff --git a/cic_base/data/config/signer.ini b/cic_base/data/config/signer.ini new file mode 100644 index 0000000..21327ba --- /dev/null +++ b/cic_base/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/cic_base/settings.py b/cic_base/settings.py index 925f6c6..724e0aa 100644 --- a/cic_base/settings.py +++ b/cic_base/settings.py @@ -2,7 +2,6 @@ import logging # external imports -import cic_eth.cli from chainlib.chain import ChainSpec from chainlib.eth.address import is_checksum_address from cic_eth.registry import ( @@ -14,6 +13,7 @@ from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError # legacy imports +import cic_base.cli from cic_base.legacy.db import SessionBase logg = logging.getLogger(__name__) @@ -30,12 +30,12 @@ class CICSettings: def process_common(self, config): self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) - rpc = cic_eth.cli.RPC.from_config(config) + rpc = cic_base.cli.RPC.from_config(config) self.o['RPC'] = rpc.get_default() def process_celery(self, config): - cic_eth.cli.CeleryApp.from_config(config) + cic_base.cli.CeleryApp.from_config(config) self.o['CELERY_QUEUE'] = config.get('CELERY_QUEUE') diff --git a/requirements.txt b/requirements.txt index 8e86f2e..41d741a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ cic-eth-registry~=0.6.9 celery~=4.4.7 +chainlib-eth~=0.1.0 +urlybird~=0.0.2