From bb3d38a1f9ee0895f364d7d9689de48bea83b67f Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Tue, 17 Aug 2021 06:46:51 +0000 Subject: [PATCH] Replace cic-base with chainlib cli utils --- apps/cic-cache/.coveragerc | 1 + apps/cic-cache/MANIFEST.in | 1 + apps/cic-cache/cic_cache/cli/__init__.py | 15 ++ apps/cic-cache/cic_cache/cli/arg.py | 20 +++ apps/cic-cache/cic_cache/cli/base.py | 31 ++++ apps/cic-cache/cic_cache/cli/celery.py | 24 +++ apps/cic-cache/cic_cache/cli/chain.py | 21 +++ apps/cic-cache/cic_cache/cli/config.py | 63 +++++++ apps/cic-cache/cic_cache/cli/registry.py | 33 ++++ apps/cic-cache/cic_cache/cli/rpc.py | 43 +++++ .../cic_cache/data/config/celery.ini | 5 + apps/cic-cache/cic_cache/data/config/cic.ini | 4 + .../cic_cache/data/config/database.ini | 10 ++ .../cic_cache/data/config/signer.ini | 2 + .../cic_cache/data/config/syncer.ini | 4 + .../versions/6604de4203e2_add_chain_syncer.py | 2 +- .../cic_cache/runnable/daemons/tracker.py | 61 +++---- apps/cic-cache/config/bancor.ini | 2 - apps/cic-cache/config/cic.ini | 1 - apps/cic-cache/config/docker/bancor.ini | 3 - apps/cic-cache/config/docker/cic.ini | 1 - apps/cic-cache/config/docker/eth.ini | 2 - apps/cic-cache/config/docker/syncer.ini | 3 +- apps/cic-cache/config/eth.ini | 2 - apps/cic-cache/config/syncer.ini | 3 - apps/cic-cache/requirements.txt | 13 +- apps/cic-cache/setup.cfg | 2 + apps/cic-cache/test_requirements.txt | 4 +- .../erc20_demurrage_token/__init__.py | 2 +- .../erc20-demurrage-token/requirements.txt | 8 +- .../erc20-demurrage-token/setup.cfg | 2 +- .../test_requirements.txt | 7 +- apps/cic-eth/.coveragerc | 1 + apps/cic-eth/MANIFEST.in | 2 +- apps/cic-eth/cic_eth/check/gas.py | 2 +- apps/cic-eth/cic_eth/check/signer.py | 2 +- apps/cic-eth/cic_eth/cli/__init__.py | 10 ++ apps/cic-eth/cic_eth/cli/arg.py | 31 ++++ apps/cic-eth/cic_eth/cli/base.py | 31 ++++ apps/cic-eth/cic_eth/cli/celery.py | 24 +++ apps/cic-eth/cic_eth/cli/chain.py | 21 +++ apps/cic-eth/cic_eth/cli/config.py | 63 +++++++ apps/cic-eth/cic_eth/cli/rpc.py | 86 +++++++++ apps/cic-eth/cic_eth/data/config/celery.ini | 5 + .../cic-eth/{ => cic_eth/data}/config/cic.ini | 4 +- apps/cic-eth/cic_eth/data/config/database.ini | 10 ++ apps/cic-eth/cic_eth/data/config/eth.ini | 2 + .../docker => cic_eth/data/config}/redis.ini | 3 +- apps/cic-eth/cic_eth/data/config/signer.ini | 2 + apps/cic-eth/cic_eth/data/config/syncer.ini | 4 + .../docker => cic_eth/data/config}/tasks.ini | 0 apps/cic-eth/cic_eth/runnable/create.py | 58 ++----- apps/cic-eth/cic_eth/runnable/ctrl.py | 50 ++---- .../cic_eth/runnable/daemons/dispatcher.py | 54 ++---- .../cic-eth/cic_eth/runnable/daemons/retry.py | 67 +++---- .../cic_eth/runnable/daemons/tasker.py | 164 +++++------------- .../cic_eth/runnable/daemons/tracker.py | 95 +++++----- apps/cic-eth/cic_eth/runnable/info.py | 41 +---- apps/cic-eth/cic_eth/runnable/resend.py | 59 ++----- apps/cic-eth/cic_eth/runnable/tag.py | 46 ++--- apps/cic-eth/cic_eth/runnable/transfer.py | 78 +++------ apps/cic-eth/cic_eth/sync/retry.py | 6 +- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/config/bancor.ini | 2 - apps/cic-eth/config/celery.ini | 4 - apps/cic-eth/config/custody.ini | 2 - apps/cic-eth/config/database.ini | 6 +- apps/cic-eth/config/dispatcer.ini | 2 - apps/cic-eth/config/docker/bancor.ini | 2 - apps/cic-eth/config/docker/celery.ini | 4 - apps/cic-eth/config/docker/cic.ini | 8 - apps/cic-eth/config/docker/custody.ini | 2 - apps/cic-eth/config/docker/dispatcer.ini | 2 - apps/cic-eth/config/docker/eth.ini | 3 - apps/cic-eth/config/docker/signer.ini | 5 - apps/cic-eth/config/docker/ssl.ini | 6 - apps/cic-eth/config/docker/syncer.ini | 3 - apps/cic-eth/config/eth.ini | 3 - apps/cic-eth/config/lash/bancor.ini | 3 - apps/cic-eth/config/lash/database.ini | 8 - apps/cic-eth/config/lash/eth.ini | 3 - apps/cic-eth/config/lash/redis.ini | 3 - apps/cic-eth/config/lash/signer.ini | 2 - apps/cic-eth/config/lash/ssl.ini | 6 - apps/cic-eth/config/lash/syncer.ini | 2 - apps/cic-eth/config/redis.ini | 4 - apps/cic-eth/config/signer.ini | 5 - apps/cic-eth/config/ssl.ini | 6 - apps/cic-eth/config/syncer.ini | 3 - apps/cic-eth/config/tasks.ini | 3 - apps/cic-eth/config/test/chain.ini | 2 + apps/cic-eth/docker/Dockerfile | 6 + apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/services_requirements.txt | 14 +- apps/cic-eth/setup.cfg | 2 + apps/cic-eth/test_requirements.txt | 2 +- apps/cic-eth/tests/check/test_check_gas.py | 2 +- apps/cic-eth/tests/check/test_check_signer.py | 2 +- apps/cic-eth/tools_requirements.txt | 4 +- apps/cic-ussd/requirements.txt | 8 +- .../config_template/celery.ini | 3 - .../config_template/chain.ini | 2 - .../config_template/cic.ini | 7 - .../config_template/database.ini | 8 - .../config_template/dev.ini | 33 ---- .../config_template/empty/config.ini | 0 .../config_template/eth.ini | 3 - .../config_template/legacy.ini | 7 + .../config_template/meta.ini | 2 - .../config_template/pgp.ini | 4 - .../config_template/redis.ini | 4 - .../config_template/rpc.ini | 5 - .../config_template/signer.ini | 3 - .../config_template/syncer.ini | 2 - .../config_template/tasks.ini | 5 - .../config_template/token.ini | 8 - .../config_template/wallet.ini | 3 - apps/contract-migration/requirements.txt | 20 +-- apps/contract-migration/reset.sh | 20 ++- apps/contract-migration/seed_cic_eth.sh | 41 +++-- apps/data-seeding/requirements.txt | 22 +-- docker-compose.yml | 50 ++++-- 122 files changed, 984 insertions(+), 832 deletions(-) create mode 100644 apps/cic-cache/MANIFEST.in create mode 100644 apps/cic-cache/cic_cache/cli/__init__.py create mode 100644 apps/cic-cache/cic_cache/cli/arg.py create mode 100644 apps/cic-cache/cic_cache/cli/base.py create mode 100644 apps/cic-cache/cic_cache/cli/celery.py create mode 100644 apps/cic-cache/cic_cache/cli/chain.py create mode 100644 apps/cic-cache/cic_cache/cli/config.py create mode 100644 apps/cic-cache/cic_cache/cli/registry.py create mode 100644 apps/cic-cache/cic_cache/cli/rpc.py create mode 100644 apps/cic-cache/cic_cache/data/config/celery.ini create mode 100644 apps/cic-cache/cic_cache/data/config/cic.ini create mode 100644 apps/cic-cache/cic_cache/data/config/database.ini create mode 100644 apps/cic-cache/cic_cache/data/config/signer.ini create mode 100644 apps/cic-cache/cic_cache/data/config/syncer.ini delete mode 100644 apps/cic-cache/config/bancor.ini delete mode 100644 apps/cic-cache/config/docker/bancor.ini delete mode 100644 apps/cic-cache/config/docker/eth.ini delete mode 100644 apps/cic-cache/config/eth.ini delete mode 100644 apps/cic-cache/config/syncer.ini create mode 100644 apps/cic-eth/cic_eth/cli/__init__.py create mode 100644 apps/cic-eth/cic_eth/cli/arg.py create mode 100644 apps/cic-eth/cic_eth/cli/base.py create mode 100644 apps/cic-eth/cic_eth/cli/celery.py create mode 100644 apps/cic-eth/cic_eth/cli/chain.py create mode 100644 apps/cic-eth/cic_eth/cli/config.py create mode 100644 apps/cic-eth/cic_eth/cli/rpc.py create mode 100644 apps/cic-eth/cic_eth/data/config/celery.ini rename apps/cic-eth/{ => cic_eth/data}/config/cic.ini (66%) create mode 100644 apps/cic-eth/cic_eth/data/config/database.ini create mode 100644 apps/cic-eth/cic_eth/data/config/eth.ini rename apps/cic-eth/{config/docker => cic_eth/data/config}/redis.ini (54%) create mode 100644 apps/cic-eth/cic_eth/data/config/signer.ini create mode 100644 apps/cic-eth/cic_eth/data/config/syncer.ini rename apps/cic-eth/{config/docker => cic_eth/data/config}/tasks.ini (100%) delete mode 100644 apps/cic-eth/config/bancor.ini delete mode 100644 apps/cic-eth/config/celery.ini delete mode 100644 apps/cic-eth/config/custody.ini delete mode 100644 apps/cic-eth/config/dispatcer.ini delete mode 100644 apps/cic-eth/config/docker/bancor.ini delete mode 100644 apps/cic-eth/config/docker/celery.ini delete mode 100644 apps/cic-eth/config/docker/cic.ini delete mode 100644 apps/cic-eth/config/docker/custody.ini delete mode 100644 apps/cic-eth/config/docker/dispatcer.ini delete mode 100644 apps/cic-eth/config/docker/eth.ini delete mode 100644 apps/cic-eth/config/docker/signer.ini delete mode 100644 apps/cic-eth/config/docker/ssl.ini delete mode 100644 apps/cic-eth/config/docker/syncer.ini delete mode 100644 apps/cic-eth/config/eth.ini delete mode 100644 apps/cic-eth/config/lash/bancor.ini delete mode 100644 apps/cic-eth/config/lash/database.ini delete mode 100644 apps/cic-eth/config/lash/eth.ini delete mode 100644 apps/cic-eth/config/lash/redis.ini delete mode 100644 apps/cic-eth/config/lash/signer.ini delete mode 100644 apps/cic-eth/config/lash/ssl.ini delete mode 100644 apps/cic-eth/config/lash/syncer.ini delete mode 100644 apps/cic-eth/config/redis.ini delete mode 100644 apps/cic-eth/config/signer.ini delete mode 100644 apps/cic-eth/config/ssl.ini delete mode 100644 apps/cic-eth/config/syncer.ini delete mode 100644 apps/cic-eth/config/tasks.ini create mode 100644 apps/cic-eth/config/test/chain.ini delete mode 100644 apps/contract-migration/config_template/chain.ini delete mode 100644 apps/contract-migration/config_template/cic.ini delete mode 100644 apps/contract-migration/config_template/database.ini delete mode 100644 apps/contract-migration/config_template/dev.ini create mode 100644 apps/contract-migration/config_template/empty/config.ini delete mode 100644 apps/contract-migration/config_template/eth.ini create mode 100644 apps/contract-migration/config_template/legacy.ini delete mode 100644 apps/contract-migration/config_template/meta.ini delete mode 100644 apps/contract-migration/config_template/pgp.ini delete mode 100644 apps/contract-migration/config_template/redis.ini delete mode 100644 apps/contract-migration/config_template/rpc.ini delete mode 100644 apps/contract-migration/config_template/signer.ini delete mode 100644 apps/contract-migration/config_template/syncer.ini delete mode 100644 apps/contract-migration/config_template/tasks.ini delete mode 100644 apps/contract-migration/config_template/token.ini delete mode 100644 apps/contract-migration/config_template/wallet.ini diff --git a/apps/cic-cache/.coveragerc b/apps/cic-cache/.coveragerc index a8162630..1fd1312c 100644 --- a/apps/cic-cache/.coveragerc +++ b/apps/cic-cache/.coveragerc @@ -4,3 +4,4 @@ omit = scripts/* cic_cache/db/migrations/* cic_cache/version.py + cic_cache/cli diff --git a/apps/cic-cache/MANIFEST.in b/apps/cic-cache/MANIFEST.in new file mode 100644 index 00000000..2630c39c --- /dev/null +++ b/apps/cic-cache/MANIFEST.in @@ -0,0 +1 @@ +include *requirements.txt cic_cache/data/config/* diff --git a/apps/cic-cache/cic_cache/cli/__init__.py b/apps/cic-cache/cic_cache/cli/__init__.py new file mode 100644 index 00000000..4952837d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/__init__.py @@ -0,0 +1,15 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp +from .registry import ( + connect_registry, + connect_token_registry, + connect_declarator, + ) diff --git a/apps/cic-cache/cic_cache/cli/arg.py b/apps/cic-cache/cic_cache/cli/arg.py new file mode 100644 index 00000000..43428cb1 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/arg.py @@ -0,0 +1,20 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') diff --git a/apps/cic-cache/cic_cache/cli/base.py b/apps/cic-cache/cic_cache/cli/base.py new file mode 100644 index 00000000..84e4ab8d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 +# REDIS = 16 +# REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-cache/cic_cache/cli/celery.py b/apps/cic-cache/cic_cache/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-cache/cic_cache/cli/chain.py b/apps/cic-cache/cic_cache/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-cache/cic_cache/cli/config.py b/apps/cic-cache/cic_cache/cli/config.py new file mode 100644 index 00000000..c21d46dc --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} +# if local_arg_flags & CICFlag.REDIS: +# local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') +# local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') +# local_args_override['REDIS_DB'] = getattr(args, 'redis_db') +# local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + +# if local_arg_flags & CICFlag.REDIS_CALLBACK: +# config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') +# config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-cache/cic_cache/cli/registry.py b/apps/cic-cache/cic_cache/cli/registry.py new file mode 100644 index 00000000..1d3e0d31 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/registry.py @@ -0,0 +1,33 @@ +# standard imports +import logging + +# external imports +from cic_eth_registry import CICRegistry +from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup +from cic_eth_registry.lookup.tokenindex import TokenIndexLookup +from chainlib.eth.constant import ZERO_ADDRESS + +logg = logging.getLogger() + + +def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address) + logg.debug('using token registry address {}'.format(token_registry_address)) + lookup = TokenIndexLookup(chain_spec, token_registry_address) + CICRegistry.add_lookup(lookup) + + +def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address) + logg.debug('using declarator address {}'.format(declarator_address)) + lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) + CICRegistry.add_lookup(lookup) + + +def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS): + CICRegistry.address = registry_address + registry = CICRegistry(chain_spec, conn) + registry_address = registry.by_name('ContractRegistry', sender_address=sender_address) + return registry diff --git a/apps/cic-cache/cic_cache/cli/rpc.py b/apps/cic-cache/cic_cache/cli/rpc.py new file mode 100644 index 00000000..7f6500ae --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/rpc.py @@ -0,0 +1,43 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import EthUnixSignerConnection +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return RPCConnection.connect(self.chain_spec, 'default') + + + @staticmethod + def from_config(config): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + if config.get('SIGNER_PROVIDER'): + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer') + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + + diff --git a/apps/cic-cache/cic_cache/data/config/celery.ini b/apps/cic-cache/cic_cache/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini new file mode 100644 index 00000000..b4275352 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/cic.ini @@ -0,0 +1,4 @@ +[cic] +registry_address = +trust_address = +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas diff --git a/apps/cic-cache/cic_cache/data/config/database.ini b/apps/cic-cache/cic_cache/data/config/database.ini new file mode 100644 index 00000000..e32e62f3 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = cic-cache +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-cache/cic_cache/data/config/signer.ini b/apps/cic-cache/cic_cache/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-cache/cic_cache/data/config/syncer.ini b/apps/cic-cache/cic_cache/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py index 55a5ecca..1b845ff6 100644 --- a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py +++ b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py @@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243 """ from alembic import op import sqlalchemy as sa -from chainsyncer.db.migrations.sqlalchemy import ( +from chainsyncer.db.migrations.default.export import ( chainsyncer_upgrade, chainsyncer_downgrade, ) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index ec03f6b7..50876e5f 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -8,15 +8,7 @@ import sys import re # external imports -import confini -import celery import sqlalchemy -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec @@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports +import cic_cache.cli from cic_cache.db import ( dsn_from_config, add_tag, @@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import ( FaucetFilter, ) -script_dir = os.path.realpath(os.path.dirname(__file__)) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process args +arg_flags = cic_cache.cli.argflag_std_read +local_arg_flags = cic_cache.cli.argflag_local_sync +argparser = cic_cache.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +# process config +config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# connect to database dsn = dsn_from_config(config) - SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_cache.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None +try: + registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) +except UnknownContractError as e: + logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) + sys.exit(1) +logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) def register_filter_tags(filters, session): @@ -95,14 +92,12 @@ def main(): syncers = [] - #if SQLBackend.first(chain_spec): - # backend = SQLBackend.initial(chain_spec, block_offset) syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.get('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -112,10 +107,10 @@ def main(): logg.info('resuming sync session {}'.format(syncer_backend)) for syncer_backend in syncer_backends: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface)) syncer_backend = SQLBackend.live(chain_spec, block_offset+1) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-cache/config/bancor.ini b/apps/cic-cache/config/bancor.ini deleted file mode 100644 index c1591774..00000000 --- a/apps/cic-cache/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini index cbb3fb2e..48c32fad 100644 --- a/apps/cic-cache/config/cic.ini +++ b/apps/cic-cache/config/cic.ini @@ -1,4 +1,3 @@ [cic] registry_address = -chain_spec = trust_address = diff --git a/apps/cic-cache/config/docker/bancor.ini b/apps/cic-cache/config/docker/bancor.ini deleted file mode 100644 index 7e53c791..00000000 --- a/apps/cic-cache/config/docker/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = -dir = /usr/local/share/bancor diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini index 1fd98612..af112724 100644 --- a/apps/cic-cache/config/docker/cic.ini +++ b/apps/cic-cache/config/docker/cic.ini @@ -1,4 +1,3 @@ [cic] -chain_spec = registry_address = trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C diff --git a/apps/cic-cache/config/docker/eth.ini b/apps/cic-cache/config/docker/eth.ini deleted file mode 100644 index 321384f8..00000000 --- a/apps/cic-cache/config/docker/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = http://localhost:63545 diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini index c811bccc..255dc00f 100644 --- a/apps/cic-cache/config/docker/syncer.ini +++ b/apps/cic-cache/config/docker/syncer.ini @@ -1,3 +1,4 @@ [syncer] loop_interval = 1 -history_start = 0 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/config/eth.ini b/apps/cic-cache/config/eth.ini deleted file mode 100644 index 98b831e5..00000000 --- a/apps/cic-cache/config/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = ws://localhost:8545 diff --git a/apps/cic-cache/config/syncer.ini b/apps/cic-cache/config/syncer.ini deleted file mode 100644 index f26739c7..00000000 --- a/apps/cic-cache/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[syncer] -loop_interval = 5 -history_start = 0 diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 54b7a895..e01c9bbe 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,14 @@ -cic-base~=0.2.0a4 alembic==1.4.2 -confini>=0.3.6rc3,<0.5.0 +confini>=0.3.6rc4,<0.5.0 uwsgi==2.0.19.1 -moolb~=0.1.0 -cic-eth-registry~=0.5.6a2 +moolb~=0.1.1b2 +cic-eth-registry~=0.5.8a1 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.3a5 -erc20-faucet~=0.2.2a2 +chainsyncer[sql]>=0.0.6a1,<0.1.0 +erc20-faucet~=0.2.4a1 +chainlib>=0.0.7a1,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 diff --git a/apps/cic-cache/setup.cfg b/apps/cic-cache/setup.cfg index 4616a475..b6de6a9c 100644 --- a/apps/cic-cache/setup.cfg +++ b/apps/cic-cache/setup.cfg @@ -23,11 +23,13 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_cache cic_cache.tasks cic_cache.db cic_cache.db.models + cic_cache.cli cic_cache.runnable cic_cache.runnable.daemons cic_cache.runnable.daemons.filters diff --git a/apps/cic-cache/test_requirements.txt b/apps/cic-cache/test_requirements.txt index f602fe01..9d46b791 100644 --- a/apps/cic-cache/test_requirements.txt +++ b/apps/cic-cache/test_requirements.txt @@ -6,5 +6,5 @@ sqlparse==0.4.1 pytest-celery==0.0.0a1 eth_tester==0.5.0b3 py-evm==0.3.0a20 -cic_base[full]==0.1.3a3+build.984b5cff -sarafu-faucet~=0.0.4a1 +sarafu-faucet~=0.0.5a2 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 diff --git a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py index ff63b895..eef7ecda 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py +++ b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py @@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp): def aux_setup(rpc, config, sender_address=ZERO_ADDRESS): - chain_spec_str = config.get('CIC_CHAIN_SPEC') + chain_spec_str = config.get('CHAIN_SPEC') chain_spec = ChainSpec.from_chain_str(chain_spec_str) token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') diff --git a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt index 9d7f6f9a..764056bb 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt @@ -1,5 +1,5 @@ celery==4.4.7 -erc20-demurrage-token~=0.0.2a3 -cic-eth-registry~=0.5.6a1 -chainlib~=0.0.5a1 -cic_eth~=0.12.0a2 +erc20-demurrage-token~=0.0.3a1 +cic-eth-registry~=0.5.8a1 +chainlib~=0.0.7a1 +cic_eth~=0.12.2a4 diff --git a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg index 69a41b7c..0e32d37f 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg +++ b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = cic-eth-aux-erc20-demurrage-token -version = 0.0.2a4 +version = 0.0.2a6 description = cic-eth tasks supporting erc20 demurrage token author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt index 0faedff4..a273b82a 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt @@ -5,8 +5,7 @@ pytest-cov==2.10.1 eth-tester==0.5.0b3 py-evm==0.3.0a20 SQLAlchemy==1.3.20 -cic-eth~=0.12.0a1 liveness~=0.0.1a7 -eth-accounts-index==0.0.12a1 -eth-contract-registry==0.5.6a1 -eth-address-index==0.1.2a1 +eth-accounts-index==0.1.1a1 +eth-contract-registry==0.5.8a1 +eth-address-index==0.2.1a1 diff --git a/apps/cic-eth/.coveragerc b/apps/cic-eth/.coveragerc index 47acdac6..8c586e19 100644 --- a/apps/cic-eth/.coveragerc +++ b/apps/cic-eth/.coveragerc @@ -6,4 +6,5 @@ omit = cic_eth/sync/head.py cic_eth/sync/mempool.py cic_eth/queue/state.py + cic_eth/cli *redis*.py diff --git a/apps/cic-eth/MANIFEST.in b/apps/cic-eth/MANIFEST.in index 5a9554ac..52d3a6f4 100644 --- a/apps/cic-eth/MANIFEST.in +++ b/apps/cic-eth/MANIFEST.in @@ -1,2 +1,2 @@ -include *requirements.txt config/test/* +include *requirements.txt config/test/* cic_eth/data/config/* diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py index b1646fd6..4add981b 100644 --- a/apps/cic-eth/cic_eth/check/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -21,7 +21,7 @@ def health(*args, **kwargs): session = SessionBase.create_session() config = kwargs['config'] - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec)) try: diff --git a/apps/cic-eth/cic_eth/check/signer.py b/apps/cic-eth/cic_eth/check/signer.py index 87b7785f..0395f0a1 100644 --- a/apps/cic-eth/cic_eth/check/signer.py +++ b/apps/cic-eth/cic_eth/check/signer.py @@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__) def health(*args, **kwargs): blocked = True max_attempts = 5 - conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer') + conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer') for i in range(max_attempts): idx = i + 1 logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts)) diff --git a/apps/cic-eth/cic_eth/cli/__init__.py b/apps/cic-eth/cic_eth/cli/__init__.py new file mode 100644 index 00000000..60eb0fb1 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/__init__.py @@ -0,0 +1,10 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp diff --git a/apps/cic-eth/cic_eth/cli/arg.py b/apps/cic-eth/cic_eth/cli/arg.py new file mode 100644 index 00000000..91fcc071 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/arg.py @@ -0,0 +1,31 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.REDIS: + self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') + self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') + self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') + if local_arg_flags & CICFlag.REDIS_CALLBACK: + self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') + self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') + + + diff --git a/apps/cic-eth/cic_eth/cli/base.py b/apps/cic-eth/cic_eth/cli/base.py new file mode 100644 index 00000000..b8b3ef2f --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 + REDIS = 16 + REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-eth/cic_eth/cli/celery.py b/apps/cic-eth/cic_eth/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-eth/cic_eth/cli/chain.py b/apps/cic-eth/cic_eth/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-eth/cic_eth/cli/config.py b/apps/cic-eth/cic_eth/cli/config.py new file mode 100644 index 00000000..8bfadc67 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} + if local_arg_flags & CICFlag.REDIS: + local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') + local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') + local_args_override['REDIS_DB'] = getattr(args, 'redis_db') + local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') + config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-eth/cic_eth/cli/rpc.py b/apps/cic-eth/cic_eth/cli/rpc.py new file mode 100644 index 00000000..cc073767 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/rpc.py @@ -0,0 +1,86 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import ( + EthUnixSignerConnection, + EthHTTPSignerConnection, + ) +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return RPCConnection.connect(self.chain_spec, 'default') + + + @staticmethod + def from_config(config, use_signer=False): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + if use_signer: + + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') + RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') + RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + +# TOOD: re-implement file backend option from omittec code: +#broker = config.get('CELERY_BROKER_URL') +#if broker[:4] == 'file': +# bq = tempfile.mkdtemp() +# bp = tempfile.mkdtemp() +# conf_update = { +# 'broker_url': broker, +# 'broker_transport_options': { +# 'data_folder_in': bq, +# 'data_folder_out': bq, +# 'data_folder_processed': bp, +# }, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) +#else: +# conf_update = { +# 'broker_url': broker, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# +#result = config.get('CELERY_RESULT_URL') +#if result[:4] == 'file': +# rq = tempfile.mkdtemp() +# current_app.conf.update({ +# 'result_backend': 'file://{}'.format(rq), +# }) +# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) +#else: +# current_app.conf.update({ +# 'result_backend': result, +# }) +# diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-eth/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini similarity index 66% rename from apps/cic-eth/config/cic.ini rename to apps/cic-eth/cic_eth/data/config/cic.ini index d3840fac..103566ff 100644 --- a/apps/cic-eth/config/cic.ini +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -1,8 +1,6 @@ [cic] registry_address = -chain_spec = evm:bloxberg:8996 -tx_retry_delay = trust_address = -default_token_symbol = GFT +default_token_symbol = health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas run_dir = /run diff --git a/apps/cic-eth/cic_eth/data/config/database.ini b/apps/cic-eth/cic_eth/data/config/database.ini new file mode 100644 index 00000000..7a295a56 --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-eth/cic_eth/data/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini new file mode 100644 index 00000000..3bba24bf --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -0,0 +1,2 @@ +[eth] +gas_gifter_minimum_balance = 10000000000000000000 diff --git a/apps/cic-eth/config/docker/redis.ini b/apps/cic-eth/cic_eth/data/config/redis.ini similarity index 54% rename from apps/cic-eth/config/docker/redis.ini rename to apps/cic-eth/cic_eth/data/config/redis.ini index ea321378..f836ebfd 100644 --- a/apps/cic-eth/config/docker/redis.ini +++ b/apps/cic-eth/cic_eth/data/config/redis.ini @@ -1,4 +1,5 @@ [redis] host = localhost -port = 63379 +port = 6379 db = 0 +timeout = 20.0 diff --git a/apps/cic-eth/cic_eth/data/config/signer.ini b/apps/cic-eth/cic_eth/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-eth/cic_eth/data/config/syncer.ini b/apps/cic-eth/cic_eth/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-eth/config/docker/tasks.ini b/apps/cic-eth/cic_eth/data/config/tasks.ini similarity index 100% rename from apps/cic-eth/config/docker/tasks.ini rename to apps/cic-eth/cic_eth/data/config/tasks.ini diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 973ac2f5..31dd1991 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -7,54 +7,30 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home +from chainlib.chain import ChainSpec # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) -logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) +logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'no_register': None, } -config.dict_override(args_override, 'cli') - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): redis_host = config.get('REDIS_HOST') @@ -68,20 +44,20 @@ def main(): ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE'), ) - register = not args.no_register + register = not config.get('_NO_REGISTER') logg.debug('register {}'.format(register)) t = api.create_account(register=register) ps.get_message() try: - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) except TimeoutError as e: sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e)) sys.exit(1) diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index fc35548e..5f260d16 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -12,64 +12,38 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.address import is_checksum_address # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi from cic_eth.db.enum import LockEnum logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task | cic_eth.cli.argflag_local_chain +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') +argparser.process_local_flags(local_arg_flags) def process_lock_args(argparser): argparser.add_argument('flags', type=str, help='Flags to manipulate') argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,') -sub = argparser.add_subparsers() +sub = argparser.add_subparsers(help='') sub.dest = "command" sub_lock = sub.add_parser('lock', help='Set or reset locks') sub_unlock = sub.add_parser('unlock', help='Set or reset locks') process_lock_args(sub_lock) process_lock_args(sub_unlock) - args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -# override args -config.dict_override(args_override, 'cli') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -queue = args.q - -chain_spec = None -if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::': - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) admin_api = AdminApi(None) @@ -100,7 +74,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('unlock {} on {} task {}'.format(flags, args.address, t)) @@ -119,7 +93,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('lock {} on {} task {}'.format(flags, args.address, t)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index 7e22e911..64b51937 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -8,8 +8,7 @@ import sys import re import datetime -# third-party imports -import confini +# external imports import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -24,7 +23,7 @@ from chainqueue.error import NotLocalTxError from chainqueue.sql.state import set_reserved # local imports -import cic_eth +import cic_eth.cli from cic_eth.db import SessionBase from cic_eth.db.enum import LockEnum from cic_eth.db import dsn_from_config @@ -39,51 +38,30 @@ from cic_eth.error import ( logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -config_dir = os.path.join('/usr/local/etc/cic-eth') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() run = True + class DispatchSyncer: yield_delay = 0.0005 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/retry.py b/apps/cic-eth/cic_eth/runnable/daemons/retry.py index 8974c33d..354fd5bc 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/retry.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/retry.py @@ -6,7 +6,6 @@ import argparse import re # external imports -import confini import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -14,6 +13,7 @@ from chainlib.connection import RPCConnection from chainsyncer.filter import SyncFilter # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.db import SessionBase from cic_eth.admin.ctrl import lock_send @@ -25,66 +25,41 @@ from cic_eth.stat import init_chain_stat logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--batch-size', dest='batch_size', type=int, default=50, help='max amount of txs to resend per iteration') -argparser.add_argument('--retry-delay', dest='retry_delay', type=int, help='seconds to wait for retrying a transaction that is marked as sent') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) +argparser.add_argument('--retry-delay', dest='retry_delay', type=int, default=20, help='seconds to wait for retrying a transaction that is marked as sent') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'), +extra_args = { + 'retry_delay': 'RETRY_DELAY', + 'batch_size': 'RETRY_BATCH_SIZE', } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.batch_size, '_BATCH_SIZE', True) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) - -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() + def main(): - conn = RPCConnection.connect(chain_spec, 'default') - - straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) + straggler_delay = int(config.get('RETRY_DELAY')) loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: stat = init_chain_stat(conn) loop_interval = stat.block_average() - syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE')) + syncer = RetrySyncer(conn, chain_spec, cic_eth.cli.chain_interface, straggler_delay, batch_size=config.get('RETRY_BATCH_SIZE')) syncer.backend.set(0, 0) - fltr = StragglerFilter(chain_spec, queue=queue) + fltr = StragglerFilter(chain_spec, queue=config.get('CELERY_QUEUE')) syncer.add_filter(fltr) syncer.loop(int(loop_interval), conn) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 837da5cb..fab470f3 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -21,14 +21,17 @@ from chainlib.eth.connection import ( EthUnixSignerConnection, EthHTTPSignerConnection, ) +from chainlib.eth.address import to_checksum_address from chainlib.chain import ChainSpec from chainqueue.db.models.otx import Otx from cic_eth_registry.error import UnknownContractError from cic_eth_registry.erc20 import ERC20Token +from hexathon import add_0x import liveness.linux # local imports +import cic_eth.cli from cic_eth.eth import ( erc20, tx, @@ -70,114 +73,53 @@ from cic_eth.task import BaseTask logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -script_dir = os.path.dirname(os.path.realpath(__file__)) - -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks') -argparser.add_argument('-r', type=str, help='CIC registry address') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use') argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path') argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'), - 'ETH_PROVIDER': getattr(args, 'p'), - 'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'), +# process config +extra_args = { + 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL', + 'aux_all': None, + 'aux': None, + 'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS', } -config.add(args.q, '_CELERY_QUEUE', True) -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(args.c, config)) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -health_modules = config.get('CIC_HEALTH_MODULES', []) -if len(health_modules) != 0: - health_modules = health_modules.split(',') -logg.debug('health mods {}'.format(health_modules)) +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config, use_signer=True) +conn = rpc.get_default() # connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) - - -# set up celery -current_app = celery.Celery(__name__) - -broker = config.get('CELERY_BROKER_URL') -if broker[:4] == 'file': - bq = tempfile.mkdtemp() - bp = tempfile.mkdtemp() - conf_update = { - 'broker_url': broker, - 'broker_transport_options': { - 'data_folder_in': bq, - 'data_folder_out': bq, - 'data_folder_processed': bp, - }, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) -else: - conf_update = { - 'broker_url': broker, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - -result = config.get('CELERY_RESULT_URL') -if result[:4] == 'file': - rq = tempfile.mkdtemp() - current_app.conf.update({ - 'result_backend': 'file://{}'.format(rq), - }) - logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) -else: - current_app.conf.update({ - 'result_backend': result, - }) - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) -RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default') -RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer') - Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') -#import cic_eth.checks.gas -#if not cic_eth.checks.gas.health(config=config): -# raise RuntimeError() + +# execute health checks +# TODO: health should be separate service with endpoint that can be queried +health_modules = config.get('CIC_HEALTH_MODULES', []) +if len(health_modules) != 0: + health_modules = health_modules.split(',') +logg.debug('health mods {}'.format(health_modules)) liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker') -rpc = RPCConnection.connect(chain_spec, 'default') + +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) @@ -188,15 +130,15 @@ if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') -for address in trusted_addresses: +for i, address in enumerate(trusted_addresses): + if config.get('_UNSAFE'): + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) +connect_declarator(conn, chain_spec, trusted_addresses) +connect_token_registry(conn, chain_spec) -connect_declarator(rpc, chain_spec, trusted_addresses) -connect_token_registry(rpc, chain_spec) - -# detect aux +# detect auxiliary task modules (plugins) # TODO: move to separate file -#aux_dir = os.path.join(script_dir, '..', '..', 'aux') aux = [] if args.aux_all: if len(args.aux) > 0: @@ -249,36 +191,24 @@ elif len(args.aux) > 0: for v in aux: mname = 'cic_eth_aux.' + v mod = importlib.import_module(mname) - mod.aux_setup(rpc, config) + mod.aux_setup(conn, config) logg.info('loaded aux module {}'.format(mname)) def main(): argv = ['worker'] - if args.vv: - argv.append('--loglevel=DEBUG') - elif args.v: - argv.append('--loglevel=INFO') + log_level = logg.getEffectiveLevel() + log_level_name = logging.getLevelName(log_level) + argv.append('--loglevel=' + log_level_name) argv.append('-Q') - argv.append(args.q) + argv.append(config.get('CELERY_QUEUE')) argv.append('-n') - argv.append(args.q) - -# if config.true('SSL_ENABLE_CLIENT'): -# Callback.ssl = True -# Callback.ssl_cert_file = config.get('SSL_CERT_FILE') -# Callback.ssl_key_file = config.get('SSL_KEY_FILE') -# Callback.ssl_password = config.get('SSL_PASSWORD') -# -# if config.get('SSL_CA_FILE') != '': -# Callback.ssl_ca_file = config.get('SSL_CA_FILE') - - rpc = RPCConnection.connect(chain_spec, 'default') + argv.append(config.get('CELERY_QUEUE')) BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) - default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address) - default_token.load(rpc) + default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address) + default_token.load(conn) BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name @@ -286,13 +216,13 @@ def main(): logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) liveness.linux.set(rundir=config.get('CIC_RUN_DIR')) - current_app.worker_main(argv) + celery_app.worker_main(argv) liveness.linux.reset(rundir=config.get('CIC_RUN_DIR')) @celery.signals.eventlet_pool_postshutdown.connect def shutdown(sender=None, headers=None, body=None, **kwargs): - logg.warning('in shudown event hook') + logg.warning('in shutdown event hook') if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 94b29883..5117537c 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -8,14 +8,6 @@ import sys import re # external imports -import confini -import celery -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS @@ -30,8 +22,13 @@ from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver.head import HeadSyncer from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase +from chainlib.eth.address import ( + is_checksum_address, + to_checksum_address, + ) # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.runnable.daemons.filters import ( CallbackFilter, @@ -47,61 +44,50 @@ from cic_eth.registry import ( connect_token_registry, ) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -script_dir = os.path.realpath(os.path.dirname(__file__)) +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process config +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) +# connect to celery +cic_eth.cli.CeleryApp.from_config(config) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) - -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.y, '_KEYSTORE_FILE', True) -config.add(args.q, '_CELERY_QUEUE', True) -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# set up database dsn = dsn_from_config(config) - SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) - -rpc = RPCConnection.connect(chain_spec, 'default') +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) - def main(): - # connect to celery - celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - # Connect to blockchain with chainlib o = block_latest() - r = rpc.do(o) + r = conn.do(o) block_current = int(r, 16) block_offset = block_current + 1 loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: - stat = init_chain_stat(rpc, block_start=block_current) + stat = init_chain_stat(conn, block_start=block_current) loop_interval = stat.block_average() logg.debug('current block height {}'.format(block_offset)) @@ -113,9 +99,9 @@ def main(): syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.true('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -128,40 +114,45 @@ def main(): for syncer_backend in syncer_backends: try: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface)) logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend)) except AttributeError: logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface)) - connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') - for address in trusted_addresses: + for i, address in enumerate(trusted_addresses): + if not config.get('_UNSAFE'): + if not is_checksum_address(address): + raise ValueError('address {} is not a valid checksum address'.format(address)) + else: + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) - connect_declarator(rpc, chain_spec, trusted_addresses) - connect_token_registry(rpc, chain_spec) + connect_declarator(conn, chain_spec, trusted_addresses) + connect_token_registry(conn, chain_spec) CallbackFilter.trusted_addresses = trusted_addresses callback_filters = [] for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','): task_split = cb.split(':') - task_queue = config.get('_CELERY_QUEUE') + task_queue = config.get('CELERY_QUEUE') if len(task_split) > 1: task_queue = task_split[0] callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue) callback_filters.append(callback_filter) - tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE')) + tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE')) account_registry_address = registry.by_name('AccountRegistry') - registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE')) + registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE')) - gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE')) + gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE')) #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) @@ -176,7 +167,7 @@ def main(): for cf in callback_filters: syncer.add_filter(cf) - r = syncer.loop(int(loop_interval), rpc) + r = syncer.loop(int(loop_interval), conn) sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) i += 1 diff --git a/apps/cic-eth/cic_eth/runnable/info.py b/apps/cic-eth/cic_eth/runnable/info.py index f8e7b50f..40e85103 100644 --- a/apps/cic-eth/cic_eth/runnable/info.py +++ b/apps/cic-eth/cic_eth/runnable/info.py @@ -12,50 +12,27 @@ import confini import celery # local imports +import cic_eth.cli from cic_eth.api import Api from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli args') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - -queue = args.q - -api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue) +api = Api(config.get('CHAIN_SPEC'), queue=config.get('CELERY_QUEUE')) admin_api = AdminApi(None) + def main(): t = admin_api.registry() registry_address = t.get() diff --git a/apps/cic-eth/cic_eth/runnable/resend.py b/apps/cic-eth/cic_eth/runnable/resend.py index 6901ef34..9861e293 100644 --- a/apps/cic-eth/cic_eth/runnable/resend.py +++ b/apps/cic-eth/cic_eth/runnable/resend.py @@ -5,65 +5,38 @@ import re import os # third-party imports -import celery -import confini from chainlib.chain import ChainSpec from chainlib.eth.connection import EthHTTPConnection # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -logging.getLogger('web3').setLevel(logging.WARNING) -logging.getLogger('urllib3').setLevel(logging.WARNING) - -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string') -argparser.add_argument('--unlock', action='store_true', help='Append task to unlock account') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('tx_hash', type=str, help='Transaction hash') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--unlock', action='store_true', help='Unlock account after resend') +argparser.add_positional('tx_hash', type=str, help='Transaction hash') +argparser.process_local_flags(local_arg_flags) +extra_args = { + 'unlock': None, + 'tx_hash': None, + } args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) -if args.vv: - logg.setLevel(logging.DEBUG) -elif args.v: - logg.setLevel(logging.INFO) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } +celery_app = cic_eth.cli.CeleryApp.from_config(config) -# override args -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.tx_hash, '_TX_HASH', True) -config.add(args.unlock, '_UNLOCK', True) - -chain_spec = ChainSpec.from_chain_str(args.i) - -rpc = EthHTTPConnection(config.get('ETH_PROVIDER')) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) def main(): - api = AdminApi(rpc) - tx_details = api.tx(chain_spec, args.tx_hash) + api = AdminApi(None) + tx_details = api.tx(chain_spec, config.get('_TX_HASH')) t = api.resend(args.tx_hash, chain_spec, unlock=config.get('_UNLOCK')) print(t.get_leaf()) diff --git a/apps/cic-eth/cic_eth/runnable/tag.py b/apps/cic-eth/cic_eth/runnable/tag.py index f7d6af11..6aeffc90 100644 --- a/apps/cic-eth/cic_eth/runnable/tag.py +++ b/apps/cic-eth/cic_eth/runnable/tag.py @@ -6,8 +6,7 @@ import argparse import re # external imports -import celery -import confini +import cic_eth.cli from chainlib.chain import ChainSpec from xdg.BaseDirectory import xdg_config_home @@ -19,43 +18,28 @@ from cic_eth.db.models.base import SessionBase logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_positional('tag', type=str, help='address tag') +argparser.add_positional('address', type=str, help='address') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -argparser.add_argument('tag', type=str, help='address tag') -argparser.add_argument('address', type=str, help='address') -args = argparser.parse_args(sys.argv[1:]) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +admin_api = AdminApi(None) -config = confini.Config(args.c) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}\n{}'.format(args.c, config)) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -chain_spec = ChainSpec.from_chain_str(args.i) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +celery_app = cic_eth.cli.CeleryApp.from_config(config) +api = AdminApi(None) def main(): - api = AdminApi(None) - api.tag_account(args.tag, args.address, chain_spec) + admin_api.tag_account(args.tag, args.address, chain_spec) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/transfer.py b/apps/cic-eth/cic_eth/runnable/transfer.py index c63c72a7..4bbaa306 100644 --- a/apps/cic-eth/cic_eth/runnable/transfer.py +++ b/apps/cic-eth/cic_eth/runnable/transfer.py @@ -7,88 +7,58 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home from chainlib.eth.address import to_checksum_address # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() -argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Symbol of token to transfer') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('sender', type=str, help='Transaction sender') -argparser.add_argument('recipient', type=str, help='Transaction recipient') -argparser.add_argument('value', type=int, help='Transaction value with decimals') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol') +argparser.add_positional('sender', type=str, help='Token transaction sender') +argparser.add_positional('recipient', type=str, help='Token transaction recipient') +argparser.add_positional('value', type=int, help='Token transaction value') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'token_symbol': None, + 'sender': None, + 'recipient': None, + 'value': None, } -config.dict_override(args_override, 'cli') -config.add(to_checksum_address(args.sender), '_SENDER', True) -config.add(to_checksum_address(args.recipient), '_RECIPIENT', True) -config.add(args.value, '_VALUE', True) -config.add(args.token_symbol, '_SYMBOL', True) -if config.get('_SYMBOL') == None: - raise ValueError('gas transfers not yet supported; token symbol required') -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) + +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): - redis_host = config.get('REDIS_HOST') - redis_port = config.get('REDIS_PORT') - redis_db = config.get('REDIS_DB') redis_channel = str(uuid.uuid4()) - r = redis.Redis(redis_host, redis_port, redis_db) + r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB')) ps = r.pubsub() ps.subscribe(redis_channel) ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE') ) - t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL')) + t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_TOKEN_SYMBOL')) ps.get_message() - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) m = json.loads(o['data']) print(m['result']) diff --git a/apps/cic-eth/cic_eth/sync/retry.py b/apps/cic-eth/cic_eth/sync/retry.py index b311964d..9118ebf4 100644 --- a/apps/cic-eth/cic_eth/sync/retry.py +++ b/apps/cic-eth/cic_eth/sync/retry.py @@ -3,7 +3,7 @@ import logging import datetime # external imports -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainsyncer.backend.memory import MemBackend from chainsyncer.error import NoBlockForYou from chainlib.eth.block import ( @@ -39,9 +39,9 @@ class DbSessionMemBackend(MemBackend): class RetrySyncer(HeadSyncer): - def __init__(self, conn, chain_spec, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): + def __init__(self, conn, chain_spec, chain_interface, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): backend = DbSessionMemBackend(chain_spec, None) - super(RetrySyncer, self).__init__(backend) + super(RetrySyncer, self).__init__(backend, chain_interface) self.chain_spec = chain_spec if failed_grace_seconds == None: failed_grace_seconds = stalled_grace_seconds diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 039b9baf..f9ac618b 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 2, - 'alpha.3', + 'alpha.4', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/config/bancor.ini b/apps/cic-eth/config/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/celery.ini b/apps/cic-eth/config/celery.ini deleted file mode 100644 index 038f83ad..00000000 --- a/apps/cic-eth/config/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis:// -result_url = redis:// -debug = 0 diff --git a/apps/cic-eth/config/custody.ini b/apps/cic-eth/config/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/database.ini b/apps/cic-eth/config/database.ini index 4517445c..6448be0e 100644 --- a/apps/cic-eth/config/database.ini +++ b/apps/cic-eth/config/database.ini @@ -1,9 +1,9 @@ [database] -NAME=cic-eth +NAME=cic_eth USER=postgres -PASSWORD= +PASSWORD=tralala HOST=localhost -PORT=5432 +PORT=63432 ENGINE=postgresql DRIVER=psycopg2 POOL_SIZE=50 diff --git a/apps/cic-eth/config/dispatcer.ini b/apps/cic-eth/config/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/bancor.ini b/apps/cic-eth/config/docker/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/docker/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/docker/celery.ini b/apps/cic-eth/config/docker/celery.ini deleted file mode 100644 index 646d6603..00000000 --- a/apps/cic-eth/config/docker/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis://localhost:63379 -result_url = redis://localhost:63379 -debug = 0 diff --git a/apps/cic-eth/config/docker/cic.ini b/apps/cic-eth/config/docker/cic.ini deleted file mode 100644 index 4fdefca0..00000000 --- a/apps/cic-eth/config/docker/cic.ini +++ /dev/null @@ -1,8 +0,0 @@ -[cic] -registry_address = -chain_spec = evm:bloxberg:8996 -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -tx_retry_delay = 20 -default_token_symbol = GFT -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas -run_dir = /run diff --git a/apps/cic-eth/config/docker/custody.ini b/apps/cic-eth/config/docker/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/docker/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/docker/dispatcer.ini b/apps/cic-eth/config/docker/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/docker/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/eth.ini b/apps/cic-eth/config/docker/eth.ini deleted file mode 100644 index 534a2eea..00000000 --- a/apps/cic-eth/config/docker/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://localhost:63545 -gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/docker/signer.ini b/apps/cic-eth/config/docker/signer.ini deleted file mode 100644 index fe72206f..00000000 --- a/apps/cic-eth/config/docker/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/docker/ssl.ini b/apps/cic-eth/config/docker/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/docker/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/docker/syncer.ini b/apps/cic-eth/config/docker/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/docker/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/eth.ini b/apps/cic-eth/config/eth.ini deleted file mode 100644 index 3c589cb2..00000000 --- a/apps/cic-eth/config/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://localhost:8545 -gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/lash/bancor.ini b/apps/cic-eth/config/lash/bancor.ini deleted file mode 100644 index 0bff1ce0..00000000 --- a/apps/cic-eth/config/lash/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549 -dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6 diff --git a/apps/cic-eth/config/lash/database.ini b/apps/cic-eth/config/lash/database.ini deleted file mode 100644 index 77ac57d4..00000000 --- a/apps/cic-eth/config/lash/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -NAME=cic-eth -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=sqlite -DRIVER=pysqlite diff --git a/apps/cic-eth/config/lash/eth.ini b/apps/cic-eth/config/lash/eth.ini deleted file mode 100644 index 324dba08..00000000 --- a/apps/cic-eth/config/lash/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -provider = http://localhost:8545 diff --git a/apps/cic-eth/config/lash/redis.ini b/apps/cic-eth/config/lash/redis.ini deleted file mode 100644 index d21df9f8..00000000 --- a/apps/cic-eth/config/lash/redis.ini +++ /dev/null @@ -1,3 +0,0 @@ -[celery] -broker_url = redis:// -result_url = file:// diff --git a/apps/cic-eth/config/lash/signer.ini b/apps/cic-eth/config/lash/signer.ini deleted file mode 100644 index 9222aef8..00000000 --- a/apps/cic-eth/config/lash/signer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[signer] -socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc diff --git a/apps/cic-eth/config/lash/ssl.ini b/apps/cic-eth/config/lash/ssl.ini deleted file mode 100644 index 746ff856..00000000 --- a/apps/cic-eth/config/lash/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = true -cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt -key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key -password = test -ca_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt diff --git a/apps/cic-eth/config/lash/syncer.ini b/apps/cic-eth/config/lash/syncer.ini deleted file mode 100644 index 97236743..00000000 --- a/apps/cic-eth/config/lash/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[SYNCER] -loop_interval = 1 diff --git a/apps/cic-eth/config/redis.ini b/apps/cic-eth/config/redis.ini deleted file mode 100644 index 1a7acb52..00000000 --- a/apps/cic-eth/config/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = -port = -db = diff --git a/apps/cic-eth/config/signer.ini b/apps/cic-eth/config/signer.ini deleted file mode 100644 index 0fa1f015..00000000 --- a/apps/cic-eth/config/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = /run/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/ssl.ini b/apps/cic-eth/config/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/syncer.ini b/apps/cic-eth/config/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/tasks.ini b/apps/cic-eth/config/tasks.ini deleted file mode 100644 index 732d797e..00000000 --- a/apps/cic-eth/config/tasks.ini +++ /dev/null @@ -1,3 +0,0 @@ -[tasks] -transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop -trace_queue_status = 1 diff --git a/apps/cic-eth/config/test/chain.ini b/apps/cic-eth/config/test/chain.ini new file mode 100644 index 00000000..9fda0988 --- /dev/null +++ b/apps/cic-eth/config/test/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 64ce7d1b..e627361c 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -27,6 +27,12 @@ RUN python setup.py install ENV PYTHONPATH . +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY \ + --extra-index-url $EXTRA_INDEX_URL \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + COPY docker/entrypoints/* ./ RUN chmod 755 *.sh diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 09811a0f..196a1284 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.6a1,<0.1.0 +chainlib-eth>=0.0.7a1,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index c70193a6..5308d2f4 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -1,15 +1,15 @@ chainqueue>=0.0.3a2,<0.1.0 -chainsyncer[sql]>=0.0.5a1,<0.1.0 +chainsyncer[sql]>=0.0.6a1,<0.1.0 alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 liveness~=0.0.1a7 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a5,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a1,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a2,<0.1.0 moolb~=0.1.1b2 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index a0354019..2b798095 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -24,8 +24,10 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_eth + cic_eth.cli cic_eth.admin cic_eth.eth cic_eth.api diff --git a/apps/cic-eth/test_requirements.txt b/apps/cic-eth/test_requirements.txt index cf7c693d..bf5ecdcc 100644 --- a/apps/cic-eth/test_requirements.txt +++ b/apps/cic-eth/test_requirements.txt @@ -6,4 +6,4 @@ pytest-redis==2.0.0 redis==3.5.3 eth-tester==0.5.0b3 py-evm==0.3.0a20 -eth-erc20~=0.0.11a1 +eth-erc20~=0.0.12a1 diff --git a/apps/cic-eth/tests/check/test_check_gas.py b/apps/cic-eth/tests/check/test_check_gas.py index ea0ffb88..056120c8 100644 --- a/apps/cic-eth/tests/check/test_check_gas.py +++ b/apps/cic-eth/tests/check/test_check_gas.py @@ -11,7 +11,7 @@ def test_check_gas( whoever, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tests/check/test_check_signer.py b/apps/cic-eth/tests/check/test_check_signer.py index 5572e195..63959de9 100644 --- a/apps/cic-eth/tests/check/test_check_signer.py +++ b/apps/cic-eth/tests/check/test_check_signer.py @@ -9,5 +9,5 @@ def test_check_signer( eth_rpc, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 03a00145..9dcf8d51 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,7 +1,7 @@ crypto-dev-signer>=0.4.14b7,<=0.4.14 -chainqueue~=0.0.2b6 +chainqueue>=0.0.3a1,<0.1.0 confini>=0.3.6rc4,<0.5.0 -cic-eth-registry>=0.5.7a1,<0.6.0 +cic-eth-registry>=0.5.8a1,<0.6.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index b0b81f95..9d71a990 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -13,5 +13,11 @@ redis==3.5.3 semver==2.13.0 SQLAlchemy==1.3.20 tinydb==4.2.0 +phonenumbers==8.12.12 +redis==3.5.3 +celery==4.4.7 +python-i18n[YAML]==0.3.9 +pyxdg==0.27 +bcrypt==3.2.0 +uWSGI==2.0.19.1 transitions==0.8.4 -uWSGI==2.0.19.1 \ No newline at end of file diff --git a/apps/contract-migration/config_template/celery.ini b/apps/contract-migration/config_template/celery.ini index dac9a5f4..e69de29b 100644 --- a/apps/contract-migration/config_template/celery.ini +++ b/apps/contract-migration/config_template/celery.ini @@ -1,3 +0,0 @@ -[celery] -broker_url = redis://redis:6379 -result_url = redis://redis:6379 diff --git a/apps/contract-migration/config_template/chain.ini b/apps/contract-migration/config_template/chain.ini deleted file mode 100644 index 6a2a8cba..00000000 --- a/apps/contract-migration/config_template/chain.ini +++ /dev/null @@ -1,2 +0,0 @@ -[chain] -spec = evm:ethereum:1 diff --git a/apps/contract-migration/config_template/cic.ini b/apps/contract-migration/config_template/cic.ini deleted file mode 100644 index bf598b98..00000000 --- a/apps/contract-migration/config_template/cic.ini +++ /dev/null @@ -1,7 +0,0 @@ -[cic] -registry_address = -token_index_address = -accounts_index_address = -declarator_address = -approval_escrow_address = -chain_spec = Bloxberg:8996 diff --git a/apps/contract-migration/config_template/database.ini b/apps/contract-migration/config_template/database.ini deleted file mode 100644 index db8227bc..00000000 --- a/apps/contract-migration/config_template/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -user = postgres -password = tralala -name = -host = postgres -port = 5432 -engine = postgres -driver = psycopg2 diff --git a/apps/contract-migration/config_template/dev.ini b/apps/contract-migration/config_template/dev.ini deleted file mode 100644 index a2c663ea..00000000 --- a/apps/contract-migration/config_template/dev.ini +++ /dev/null @@ -1,33 +0,0 @@ -[dev] -mnemonic = "history stumble mystery avoid embark arrive mom foil pledge keep grain dice" -eth_reserve_address = -eth_accounts_index_address = -# 10m * 10^18 (10^7 * 10^18) -eth_reserve_amount = 1000000000000000000000000 - -eth_account_contract_deployer=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -eth_account_gas_provider=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_gas_gifter=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_reserve_owner=0xc14958CD9A605AB0d9A36850362AaD2b9D42DF97 -eth_account_reserve_minter=0xe3C4db5947409Aff0FF8D643047EA41515cA4B8e -eth_account_accounts_index_owner=0x1AB26f3AAaaa884B651E213508592F3Ec6BfCe74 -eth_account_accounts_index_writer=0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA -eth_account_sarafu_owner=0x5567139c7a1C2977A391f51D8cA45B1D6700f5F6 -eth_account_sarafu_gifter=0x1149D55B3495CdC8FcfaF2816384AafaeFfaEa24 -eth_account_approval_escrow_owner=0x8754122F5718dC02Db5062c62fD4a63A5448623E -eth_account_single_shot_faucet_owner=0x34C85E47f45b0bea09F37c83e2fb02ECBC3a395d - -eth_sarafu_token_address = -eth_token_index_address = -eth_sarafu_faucet_address = -eth_erc20_approval_escrow_address = - -eth_sarafu_token_name=Sarafu -eth_sarafu_token_symbol=SRF -eth_sarafu_token_decimals=18 - -pgp_publickeys_active_file = publickeys.asc -pgp_publickeys_trusted_file = -pgp_publickeys_encrypt_file = - -faucet_amount = 1000000 diff --git a/apps/contract-migration/config_template/empty/config.ini b/apps/contract-migration/config_template/empty/config.ini new file mode 100644 index 00000000..e69de29b diff --git a/apps/contract-migration/config_template/eth.ini b/apps/contract-migration/config_template/eth.ini deleted file mode 100644 index 5506851b..00000000 --- a/apps/contract-migration/config_template/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://eth:8545 -abi_dir = /usr/local/share/cic/solidity/abi diff --git a/apps/contract-migration/config_template/legacy.ini b/apps/contract-migration/config_template/legacy.ini new file mode 100644 index 00000000..02eb709c --- /dev/null +++ b/apps/contract-migration/config_template/legacy.ini @@ -0,0 +1,7 @@ +[celery] +broker_url = +result_url = +debug = + +[cic] +chain_spec = diff --git a/apps/contract-migration/config_template/meta.ini b/apps/contract-migration/config_template/meta.ini deleted file mode 100644 index cbcc16ac..00000000 --- a/apps/contract-migration/config_template/meta.ini +++ /dev/null @@ -1,2 +0,0 @@ -[meta] -provider = http://cic-meta-server:80 diff --git a/apps/contract-migration/config_template/pgp.ini b/apps/contract-migration/config_template/pgp.ini deleted file mode 100644 index fade0d5a..00000000 --- a/apps/contract-migration/config_template/pgp.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pgp] -exports_dir = /tmp/cic/pgp -privatekey_file = ge.priv.asc -passphrase = ge diff --git a/apps/contract-migration/config_template/redis.ini b/apps/contract-migration/config_template/redis.ini deleted file mode 100644 index ea321378..00000000 --- a/apps/contract-migration/config_template/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = localhost -port = 63379 -db = 0 diff --git a/apps/contract-migration/config_template/rpc.ini b/apps/contract-migration/config_template/rpc.ini deleted file mode 100644 index 31dc0614..00000000 --- a/apps/contract-migration/config_template/rpc.ini +++ /dev/null @@ -1,5 +0,0 @@ -[rpc] -http_provider = http://localhost:8545 -http_authentication = -http_username = -http_password = diff --git a/apps/contract-migration/config_template/signer.ini b/apps/contract-migration/config_template/signer.ini deleted file mode 100644 index 58ad0c0a..00000000 --- a/apps/contract-migration/config_template/signer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[signer] -socket_path = /tmp/cic/signer/jsonrpc.ipc -secret = deadbeef diff --git a/apps/contract-migration/config_template/syncer.ini b/apps/contract-migration/config_template/syncer.ini deleted file mode 100644 index 60c0c007..00000000 --- a/apps/contract-migration/config_template/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[syncer] -loop_interval = diff --git a/apps/contract-migration/config_template/tasks.ini b/apps/contract-migration/config_template/tasks.ini deleted file mode 100644 index b49ecad5..00000000 --- a/apps/contract-migration/config_template/tasks.ini +++ /dev/null @@ -1,5 +0,0 @@ -[tasks] -africastalking = cic_notify.tasks.sms.africastalking -sms_db = cic_notify.tasks.sms.db -log = cic_notify.tasks.sms.log -transfer_callbacks=cic-ussd:cic_ussd.tasks.callback_handler.process_incoming_transfer_callback diff --git a/apps/contract-migration/config_template/token.ini b/apps/contract-migration/config_template/token.ini deleted file mode 100644 index 865e7f84..00000000 --- a/apps/contract-migration/config_template/token.ini +++ /dev/null @@ -1,8 +0,0 @@ -[token] -name = -symbol = -decimals = -demurrage_level = -redistribution_period = -sink_address = -supply_limit = 0 diff --git a/apps/contract-migration/config_template/wallet.ini b/apps/contract-migration/config_template/wallet.ini deleted file mode 100644 index 696cee43..00000000 --- a/apps/contract-migration/config_template/wallet.ini +++ /dev/null @@ -1,3 +0,0 @@ -[wallet] -key_file = -passphrase = diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 9d5cb9b2..d374e8d5 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,10 +1,10 @@ -cic-eth[tools]==0.12.2a3 -eth-erc20>=0.0.11a1,<0.1.0 -erc20-demurrage-token>=0.0.2a5,<0.1.0 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<=0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a4,<0.1.0 -chainlib-eth>=0.0.6a1,<0.1.0 +cic-eth[tools]==0.12.2a4 +eth-erc20>=0.0.12a1,<0.1.0 +erc20-demurrage-token>=0.0.3a1,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a1,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a5,<0.1.0 +chainlib-eth>=0.0.7a1,<0.1.0 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index ed6912c6..3a9d1b7c 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -17,7 +17,9 @@ token_redistribution_period: $TOKEN_REDISTRIBUTION_PERIOD token_supply_limit: $TOKEN_SUPPLY_LIMIT EOF -CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995} +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000} @@ -30,6 +32,7 @@ DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $DEV_ETH_KEYSTORE_FILE | j if [ ! -z $DEV_ETH_GAS_PRICE ]; then gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE" + fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE" >&2 echo using static gas price $DEV_ETH_GAS_PRICE fi @@ -133,13 +136,22 @@ if [[ -n "${ETH_PROVIDER}" ]]; then # Sarafu faucet contract >&2 echo "deploy token faucet contract" - DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS` + export _CONFINI_DIR=$CONFINI_DIR + unset CONFINI_DIR + DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s` + + >&2 echo "set token faucet amount" + sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT + + export CONFINI_DIR=$_CONFINI_DIR + + >&2 echo "register faucet in registry" eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS + >&2 echo "set faucet as token minter" giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS - >&2 echo "set token faucet amount" - sarafu-faucet-set $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS -vv $DEV_FAUCET_AMOUNT + else diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index aba7ea6c..29a80220 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -16,15 +16,18 @@ ETH_PASSPHRASE='' CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + # Debug flag DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C keystore_file=./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c debug='-vv' -abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} gas_amount=100000000000000000000000 token_amount=${gas_amount} env_out_file=${CIC_DATA_DIR}/.env_seed init_level_file=${CIC_DATA_DIR}/.init +empty_config_dir=$CONFINI_DIR/empty truncate $env_out_file -s 0 set -e @@ -33,9 +36,9 @@ set -a #pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL eth-address-index==0.1.1a7 # get required addresses from registries -DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` -DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` -DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` +DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` +DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` +DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` cat <&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER -DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_GAS_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER +cic-eth-tag -i $CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER >&2 echo "create account for sarafu gifter" -DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_SARAFU_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER +cic-eth-tag -i $CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER >&2 echo "create account for approval escrow owner" -DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER +cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER #>&2 echo "create account for faucet owner" #DEV_ETH_ACCOUNT_FAUCET_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` @@ -65,32 +68,32 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER >&2 echo "create account for accounts index writer" -DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >&2 echo "add acccounts index writer account as writer on contract" -eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +eth-accounts-index-writer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER # Transfer gas to custodial gas provider adddress _CONFINI_DIR=$CONFINI_DIR unset CONFINI_DIR >&2 echo gift gas to gas gifter ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount >&2 echo gift gas to sarafu token owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount >&2 echo gift gas to account index owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount # Send token to token creator >&2 echo "gift tokens to sarafu owner" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount # Send token to token gifter >&2 echo "gift tokens to keystore address" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)" echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file @@ -98,7 +101,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS # Transfer tokens to gifter address >&2 echo "transfer tokens to token gifter address" ->&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} +>&2 erc20-transfer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} #echo -n 0 > $init_level_file diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 10a752fd..7317ddf8 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,13 +1,13 @@ -sarafu-faucet==0.0.4a3 -cic-eth[tools]==0.12.1a1 -cic-types==0.1.0a13 -crypto-dev-signer==0.4.14b7 +sarafu-faucet~=0.0.5a2 +cic-eth[tools]~=0.12.2a3 +cic-types~=0.1.0a14 +crypto-dev-signer~=0.4.14rc1 faker==4.17.1 -chainsyncer~=0.0.3a3 -chainlib-eth~=0.0.5a1 -eth-address-index~=0.1.2a1 -eth-contract-registry~=0.5.6a1 -eth-accounts-index~=0.0.12a1 -eth-erc20~=0.0.10a3 -erc20-faucet~=0.2.2a1 +chainsyncer~=0.0.6a1 +chainlib-eth~=0.0.7a1 +eth-address-index~=0.1.4a1 +eth-contract-registry~=0.5.8a1 +eth-accounts-index~=0.0.14a1 +eth-erc20~=0.0.12a1 +erc20-faucet~=0.2.4a1 psycopg2==2.8.6 diff --git a/docker-compose.yml b/docker-compose.yml index 3c9a4540..57b6be2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,16 +79,18 @@ services: args: pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} pip_extra_args: $PIP_EXTRA_ARGS + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - ETH_PROVIDER: http://eth:8545 + ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} # And these two are for wait-for-it (could parse this) DEV_USE_DOCKER_WAIT_SCRIPT: 1 ETH_PROVIDER_HOST: eth ETH_PROVIDER_PORT: 8545 - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PORT: ${DATABASE_PORT:-5432} @@ -131,9 +133,12 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -142,9 +147,9 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 CELERY_RESULT_URL: redis://redis:6379 deploy: @@ -169,6 +174,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} @@ -181,7 +188,6 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 DATABASE_POOL_SIZE: 0 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 @@ -208,6 +214,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -240,11 +248,13 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER - ETH_PROVIDER: http://eth:8545 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -258,11 +268,12 @@ services: REDIS_HOST: redis PGPASSWORD: ${DATABASE_PASSWORD:-tralala} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} @@ -290,8 +301,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: - ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -300,7 +314,8 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -320,7 +335,8 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tracker.sh -vv -c /usr/local/etc/cic-eth + #./start_tracker.sh -vv -c /usr/local/etc/cic-eth + ./start_tracker.sh -vv # command: "/root/start_manager.sh head -vv" @@ -329,8 +345,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -339,6 +358,7 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -369,8 +389,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -380,6 +403,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}