diff --git a/apps/cic-cache/.coveragerc b/apps/cic-cache/.coveragerc index a8162630..1fd1312c 100644 --- a/apps/cic-cache/.coveragerc +++ b/apps/cic-cache/.coveragerc @@ -4,3 +4,4 @@ omit = scripts/* cic_cache/db/migrations/* cic_cache/version.py + cic_cache/cli diff --git a/apps/cic-cache/MANIFEST.in b/apps/cic-cache/MANIFEST.in new file mode 100644 index 00000000..2630c39c --- /dev/null +++ b/apps/cic-cache/MANIFEST.in @@ -0,0 +1 @@ +include *requirements.txt cic_cache/data/config/* diff --git a/apps/cic-cache/cic_cache/api.py b/apps/cic-cache/cic_cache/api.py index 0340e65c..74083ba5 100644 --- a/apps/cic-cache/cic_cache/api.py +++ b/apps/cic-cache/cic_cache/api.py @@ -55,15 +55,37 @@ class Api: queue=callback_queue, ) - def list(self, offset, limit, address=None): + def list(self, offset=0, limit=100, address=None, oldest=False): s = celery.signature( 'cic_cache.tasks.tx.tx_filter', [ - 0, - 100, + offset, + limit, address, + oldest, ], - queue=None + queue=self.queue, + ) + if self.callback_param != None: + s.link(self.callback_success).on_error(self.callback_error) + + t = s.apply_async() + + return t + + + def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False): + s = celery.signature( + 'cic_cache.tasks.tx.tx_filter_content', + [ + offset, + limit, + address, + block_offset, + block_limit, + oldest, + ], + queue=self.queue, ) if self.callback_param != None: s.link(self.callback_success).on_error(self.callback_error) diff --git a/apps/cic-cache/cic_cache/cache.py b/apps/cic-cache/cic_cache/cache.py index 7c4d1040..10e8c458 100644 --- a/apps/cic-cache/cic_cache/cache.py +++ b/apps/cic-cache/cic_cache/cache.py @@ -10,12 +10,16 @@ from cic_cache.db.list import ( list_transactions_mined, list_transactions_account_mined, list_transactions_mined_with_data, + list_transactions_mined_with_data_index, + list_transactions_account_mined_with_data_index, + list_transactions_account_mined_with_data, ) logg = logging.getLogger() DEFAULT_FILTER_SIZE = 8192 * 8 +DEFAULT_LIMIT = 100 class Cache: @@ -32,7 +36,7 @@ class BloomCache(Cache): return n - def load_transactions(self, offset, limit): + def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False): """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions. Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index. @@ -49,7 +53,7 @@ class BloomCache(Cache): :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx :rtype: tuple """ - rows = list_transactions_mined(self.session, offset, limit) + rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) @@ -58,7 +62,12 @@ class BloomCache(Cache): for r in rows: if highest_block == -1: highest_block = r[0] - lowest_block = r[0] + lowest_block = r[0] + else: + if oldest: + highest_block = r[0] + else: + lowest_block = r[0] block = r[0].to_bytes(4, byteorder='big') tx = r[1].to_bytes(4, byteorder='big') f_block.add(block) @@ -67,7 +76,7 @@ class BloomCache(Cache): return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),) - def load_transactions_account(self, address, offset, limit): + def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False): """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient. :param address: Address to retrieve transactions for. @@ -79,7 +88,7 @@ class BloomCache(Cache): :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx :rtype: tuple """ - rows = list_transactions_account_mined(self.session, address, offset, limit) + rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) @@ -88,7 +97,12 @@ class BloomCache(Cache): for r in rows: if highest_block == -1: highest_block = r[0] - lowest_block = r[0] + lowest_block = r[0] + else: + if oldest: + highest_block = r[0] + else: + lowest_block = r[0] block = r[0].to_bytes(4, byteorder='big') tx = r[1].to_bytes(4, byteorder='big') f_block.add(block) @@ -99,8 +113,21 @@ class BloomCache(Cache): class DataCache(Cache): - def load_transactions_with_data(self, offset, end): - rows = list_transactions_mined_with_data(self.session, offset, end) + def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False): + if limit == 0: + limit = DEFAULT_LIMIT + rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) + return self.__process_rows(rows, oldest) + + + def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False): + if limit == 0: + limit = DEFAULT_LIMIT + rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) + return self.__process_rows(rows, oldest) + + + def __process_rows(self, rows, oldest): tx_cache = [] highest_block = -1; lowest_block = -1; @@ -108,7 +135,12 @@ class DataCache(Cache): for r in rows: if highest_block == -1: highest_block = r['block_number'] - lowest_block = r['block_number'] + lowest_block = r['block_number'] + else: + if oldest: + highest_block = r['block_number'] + else: + lowest_block = r['block_number'] tx_type = 'unknown' if r['value'] != None: diff --git a/apps/cic-cache/cic_cache/cli/__init__.py b/apps/cic-cache/cic_cache/cli/__init__.py new file mode 100644 index 00000000..4952837d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/__init__.py @@ -0,0 +1,15 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp +from .registry import ( + connect_registry, + connect_token_registry, + connect_declarator, + ) diff --git a/apps/cic-cache/cic_cache/cli/arg.py b/apps/cic-cache/cic_cache/cli/arg.py new file mode 100644 index 00000000..2d4e6e8a --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/arg.py @@ -0,0 +1,20 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') diff --git a/apps/cic-cache/cic_cache/cli/base.py b/apps/cic-cache/cic_cache/cli/base.py new file mode 100644 index 00000000..84e4ab8d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 +# REDIS = 16 +# REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-cache/cic_cache/cli/celery.py b/apps/cic-cache/cic_cache/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-cache/cic_cache/cli/chain.py b/apps/cic-cache/cic_cache/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-cache/cic_cache/cli/config.py b/apps/cic-cache/cic_cache/cli/config.py new file mode 100644 index 00000000..c21d46dc --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} +# if local_arg_flags & CICFlag.REDIS: +# local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') +# local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') +# local_args_override['REDIS_DB'] = getattr(args, 'redis_db') +# local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + +# if local_arg_flags & CICFlag.REDIS_CALLBACK: +# config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') +# config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-cache/cic_cache/cli/registry.py b/apps/cic-cache/cic_cache/cli/registry.py new file mode 100644 index 00000000..1d3e0d31 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/registry.py @@ -0,0 +1,33 @@ +# standard imports +import logging + +# external imports +from cic_eth_registry import CICRegistry +from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup +from cic_eth_registry.lookup.tokenindex import TokenIndexLookup +from chainlib.eth.constant import ZERO_ADDRESS + +logg = logging.getLogger() + + +def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address) + logg.debug('using token registry address {}'.format(token_registry_address)) + lookup = TokenIndexLookup(chain_spec, token_registry_address) + CICRegistry.add_lookup(lookup) + + +def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address) + logg.debug('using declarator address {}'.format(declarator_address)) + lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) + CICRegistry.add_lookup(lookup) + + +def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS): + CICRegistry.address = registry_address + registry = CICRegistry(chain_spec, conn) + registry_address = registry.by_name('ContractRegistry', sender_address=sender_address) + return registry diff --git a/apps/cic-cache/cic_cache/cli/rpc.py b/apps/cic-cache/cic_cache/cli/rpc.py new file mode 100644 index 00000000..7f6500ae --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/rpc.py @@ -0,0 +1,43 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import EthUnixSignerConnection +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return RPCConnection.connect(self.chain_spec, 'default') + + + @staticmethod + def from_config(config): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + if config.get('SIGNER_PROVIDER'): + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer') + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + + diff --git a/apps/cic-cache/cic_cache/data/config/celery.ini b/apps/cic-cache/cic_cache/data/config/celery.ini new file mode 100644 index 00000000..a799cb4a --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-cache +debug = 0 diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini new file mode 100644 index 00000000..b4275352 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/cic.ini @@ -0,0 +1,4 @@ +[cic] +registry_address = +trust_address = +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas diff --git a/apps/cic-cache/cic_cache/data/config/database.ini b/apps/cic-cache/cic_cache/data/config/database.ini new file mode 100644 index 00000000..e32e62f3 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = cic-cache +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-cache/cic_cache/data/config/signer.ini b/apps/cic-cache/cic_cache/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-cache/cic_cache/data/config/syncer.ini b/apps/cic-cache/cic_cache/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/cic_cache/db/list.py b/apps/cic-cache/cic_cache/db/list.py index 62e0a2e0..4052471e 100644 --- a/apps/cic-cache/cic_cache/db/list.py +++ b/apps/cic-cache/cic_cache/db/list.py @@ -13,6 +13,9 @@ def list_transactions_mined( session, offset, limit, + block_offset, + block_limit, + oldest=False, ): """Executes db query to return all confirmed transactions according to the specified offset and limit. @@ -23,15 +26,62 @@ def list_transactions_mined( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset) + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset) r = session.execute(s) return r def list_transactions_mined_with_data( + session, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit. + + :param block_offset: First block to include in search + :type block_offset: int + :param block_limit: Last block to include in search + :type block_limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset) + + + r = session.execute(s) + return r + + +def list_transactions_mined_with_data_index( session, offset, end, + block_offset, + block_limit, + oldest=False, ): """Executes db query to return all confirmed transactions according to the specified offset and limit. @@ -42,7 +92,87 @@ def list_transactions_mined_with_data( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end) + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end) + + r = session.execute(s) + return r + + +def list_transactions_account_mined_with_data_index( + session, + address, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address + + :param offset: Offset in data set to return transactions from + :type offset: int + :param limit: Max number of transactions to retrieve + :type limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) + + r = session.execute(s) + return r + +def list_transactions_account_mined_with_data( + session, + address, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit. + + :param block_offset: First block to include in search + :type block_offset: int + :param block_limit: Last block to include in search + :type block_limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) r = session.execute(s) return r @@ -53,6 +183,9 @@ def list_transactions_account_mined( address, offset, limit, + block_offset, + block_limit, + oldest=False, ): """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient. @@ -65,7 +198,20 @@ def list_transactions_account_mined( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset) + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + + else: + s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) + r = session.execute(s) return r diff --git a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py index 55a5ecca..1b845ff6 100644 --- a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py +++ b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py @@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243 """ from alembic import op import sqlalchemy as sa -from chainsyncer.db.migrations.sqlalchemy import ( +from chainsyncer.db.migrations.default.export import ( chainsyncer_upgrade, chainsyncer_downgrade, ) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py index a698a692..d7edf6ab 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/query.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -91,13 +91,14 @@ def process_transactions_all_data(session, env): if env.get('HTTP_X_CIC_CACHE_MODE') != 'all': return None - offset = r[1] - end = r[2] + logg.debug('got data request {}'.format(env)) + block_offset = r[1] + block_end = r[2] if int(r[2]) < int(r[1]): raise ValueError('cart before the horse, dude') c = DataCache(session) - (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end) + (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable for r in tx_cache: r['date_block'] = r['date_block'].timestamp() diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index ec03f6b7..50876e5f 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -8,15 +8,7 @@ import sys import re # external imports -import confini -import celery import sqlalchemy -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec @@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports +import cic_cache.cli from cic_cache.db import ( dsn_from_config, add_tag, @@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import ( FaucetFilter, ) -script_dir = os.path.realpath(os.path.dirname(__file__)) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process args +arg_flags = cic_cache.cli.argflag_std_read +local_arg_flags = cic_cache.cli.argflag_local_sync +argparser = cic_cache.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +# process config +config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# connect to database dsn = dsn_from_config(config) - SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_cache.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None +try: + registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) +except UnknownContractError as e: + logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) + sys.exit(1) +logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) def register_filter_tags(filters, session): @@ -95,14 +92,12 @@ def main(): syncers = [] - #if SQLBackend.first(chain_spec): - # backend = SQLBackend.initial(chain_spec, block_offset) syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.get('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -112,10 +107,10 @@ def main(): logg.info('resuming sync session {}'.format(syncer_backend)) for syncer_backend in syncer_backends: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface)) syncer_backend = SQLBackend.live(chain_spec, block_offset+1) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-cache/cic_cache/tasks/tx.py b/apps/cic-cache/cic_cache/tasks/tx.py index 0e2f7435..7524fe6f 100644 --- a/apps/cic-cache/cic_cache/tasks/tx.py +++ b/apps/cic-cache/cic_cache/tasks/tx.py @@ -2,14 +2,17 @@ import celery # local imports -from cic_cache.cache import BloomCache +from cic_cache.cache import ( + BloomCache, + DataCache, + ) from cic_cache.db.models.base import SessionBase celery_app = celery.current_app @celery_app.task(bind=True) -def tx_filter(self, offset, limit, address=None, encoding='hex'): +def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'): queue = self.request.delivery_info.get('routing_key') session = SessionBase.create_session() @@ -17,9 +20,9 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'): c = BloomCache(session) b = None if address == None: - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest) else: - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest) session.close() @@ -35,4 +38,17 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'): return o +@celery_app.task(bind=True) +def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'): + session = SessionBase.create_session() + c = DataCache(session) + b = None + if address == None: + (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) + else: + (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit) + + session.close() + + return (lowest_block, highest_block, tx_cache,) diff --git a/apps/cic-cache/cic_cache/version.py b/apps/cic-cache/cic_cache/version.py index 338bb831..b5293a59 100644 --- a/apps/cic-cache/cic_cache/version.py +++ b/apps/cic-cache/cic_cache/version.py @@ -4,8 +4,8 @@ import semver version = ( 0, 2, - 0, - 'alpha.2', + 1, + 'alpha.1', ) version_object = semver.VersionInfo( diff --git a/apps/cic-cache/config/bancor.ini b/apps/cic-cache/config/bancor.ini deleted file mode 100644 index c1591774..00000000 --- a/apps/cic-cache/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini index cbb3fb2e..48c32fad 100644 --- a/apps/cic-cache/config/cic.ini +++ b/apps/cic-cache/config/cic.ini @@ -1,4 +1,3 @@ [cic] registry_address = -chain_spec = trust_address = diff --git a/apps/cic-cache/config/docker/bancor.ini b/apps/cic-cache/config/docker/bancor.ini deleted file mode 100644 index 7e53c791..00000000 --- a/apps/cic-cache/config/docker/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = -dir = /usr/local/share/bancor diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini index 1fd98612..af112724 100644 --- a/apps/cic-cache/config/docker/cic.ini +++ b/apps/cic-cache/config/docker/cic.ini @@ -1,4 +1,3 @@ [cic] -chain_spec = registry_address = trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C diff --git a/apps/cic-cache/config/docker/eth.ini b/apps/cic-cache/config/docker/eth.ini deleted file mode 100644 index 321384f8..00000000 --- a/apps/cic-cache/config/docker/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = http://localhost:63545 diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini index c811bccc..255dc00f 100644 --- a/apps/cic-cache/config/docker/syncer.ini +++ b/apps/cic-cache/config/docker/syncer.ini @@ -1,3 +1,4 @@ [syncer] loop_interval = 1 -history_start = 0 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/config/eth.ini b/apps/cic-cache/config/eth.ini deleted file mode 100644 index 98b831e5..00000000 --- a/apps/cic-cache/config/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = ws://localhost:8545 diff --git a/apps/cic-cache/config/syncer.ini b/apps/cic-cache/config/syncer.ini deleted file mode 100644 index f26739c7..00000000 --- a/apps/cic-cache/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[syncer] -loop_interval = 5 -history_start = 0 diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 54b7a895..890594bf 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,17 @@ -cic-base~=0.2.0a4 alembic==1.4.2 -confini>=0.3.6rc3,<0.5.0 +confini>=0.3.6rc4,<0.5.0 uwsgi==2.0.19.1 -moolb~=0.1.0 -cic-eth-registry~=0.5.6a2 +moolb~=0.1.1b2 +cic-eth-registry~=0.5.8a1 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.3a5 -erc20-faucet~=0.2.2a2 +chainsyncer[sql]>=0.0.6a1,<0.1.0 +erc20-faucet>=0.2.4a2, <0.3.0 +#chainlib-eth==0.0.7a5,<0.1.0 +chainlib-eth==0.0.7a5 +#chainlib==0.0.7a4,<0.1.0 +chainlib==0.0.7a4 +eth-address-index>=0.1.4a1,<0.2.0 diff --git a/apps/cic-cache/setup.cfg b/apps/cic-cache/setup.cfg index 4616a475..f8393c92 100644 --- a/apps/cic-cache/setup.cfg +++ b/apps/cic-cache/setup.cfg @@ -23,11 +23,13 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_cache cic_cache.tasks cic_cache.db cic_cache.db.models + cic_cache.cli cic_cache.runnable cic_cache.runnable.daemons cic_cache.runnable.daemons.filters @@ -39,3 +41,4 @@ console_scripts = cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main cic-cache-serverd = cic_cache.runnable.daemons.server:main cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main + cic-cache-list = cic_cache.runable.list:main diff --git a/apps/cic-cache/test_requirements.txt b/apps/cic-cache/test_requirements.txt index f602fe01..9d46b791 100644 --- a/apps/cic-cache/test_requirements.txt +++ b/apps/cic-cache/test_requirements.txt @@ -6,5 +6,5 @@ sqlparse==0.4.1 pytest-celery==0.0.0a1 eth_tester==0.5.0b3 py-evm==0.3.0a20 -cic_base[full]==0.1.3a3+build.984b5cff -sarafu-faucet~=0.0.4a1 +sarafu-faucet~=0.0.5a2 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 diff --git a/apps/cic-cache/tests/cli/test_cli_args.py b/apps/cic-cache/tests/cli/test_cli_args.py new file mode 100644 index 00000000..6e0c8a26 --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_args.py @@ -0,0 +1,40 @@ +# standard imports +import os + +# external imports +import chainlib.cli + +# local imports +import cic_cache.cli + +script_dir = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(script_dir, '..', 'testdata', 'config') + + +def test_argumentparserto_config(): + + argparser = cic_cache.cli.ArgumentParser() + + local_flags = 0xffff + argparser.process_local_flags(local_flags) + argparser.add_argument('--foo', type=str) + args = argparser.parse_args([ + '-q', 'baz', + '--offset', '13', + '--no-history', + '-r','0xdeadbeef', + '-vv', + '--foo', 'bar', + ]) + + extra_args = { + 'foo': '_BARBARBAR', + } + config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir) + + assert config.get('_BARBARBAR') == 'bar' + assert config.get('CELERY_QUEUE') == 'baz' + assert config.get('SYNCER_NO_HISTORY') == True + assert config.get('SYNCER_OFFSET') == 13 + assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef' + diff --git a/apps/cic-cache/tests/cli/test_cli_celery.py b/apps/cic-cache/tests/cli/test_cli_celery.py new file mode 100644 index 00000000..81d975af --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_celery.py @@ -0,0 +1,17 @@ +# standard imports +import tempfile + +# local imports +import cic_cache.cli + + +def test_cli_celery(): + cf = tempfile.mkdtemp() + + config = { + 'CELERY_RESULT_URL': 'filesystem://' + cf, + } + cic_cache.cli.CeleryApp.from_config(config) + + config['CELERY_BROKER_URL'] = 'filesystem://' + cf + cic_cache.cli.CeleryApp.from_config(config) diff --git a/apps/cic-cache/tests/cli/test_cli_chain.py b/apps/cic-cache/tests/cli/test_cli_chain.py new file mode 100644 index 00000000..0cf9d5da --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_chain.py @@ -0,0 +1,68 @@ +# external imports +import pytest +from chainlib.eth.gas import ( + Gas, + RPCGasOracle, + ) +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.block import ( + block_latest, + Block, + ) +from chainlib.eth.pytest.fixtures_chain import default_chain_spec +from chainlib.eth.pytest.fixtures_ethtester import * +from cic_eth_registry.pytest.fixtures_contracts import * +from hexathon import add_0x + +# local imports +import cic_cache.cli + + +@pytest.mark.xfail() +def test_cli_rpc( + eth_rpc, + eth_signer, + default_chain_spec, + ): + config = { + 'CHAIN_SPEC': str(default_chain_spec), + 'RPC_HTTP_PROVIDER': 'http://localhost:8545', + } + rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') + conn = rpc.get_by_label('foo') + #o = block_latest() + #conn.do(o) + + +def test_cli_chain( + default_chain_spec, + eth_rpc, + eth_signer, + contract_roles, + ): + ifc = cic_cache.cli.EthChainInterface() + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc) + gas_oracle = RPCGasOracle(conn=eth_rpc) + c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer) + recipient = add_0x(os.urandom(20).hex()) + (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024) + r = eth_rpc.do(o) + + o = ifc.tx_receipt(r) + r = eth_rpc.do(o) + assert r['status'] == 1 + + o = ifc.block_by_number(1) + block_src = eth_rpc.do(o) + block = ifc.block_from_src(block_src) + assert block.number == 1 + + with pytest.raises(KeyError): + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + + block_src = ifc.src_normalize(block_src) + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + diff --git a/apps/cic-cache/tests/conftest.py b/apps/cic-cache/tests/conftest.py index 61db702f..478c261f 100644 --- a/apps/cic-cache/tests/conftest.py +++ b/apps/cic-cache/tests/conftest.py @@ -64,7 +64,6 @@ def txs( dt.timestamp(), ) - tx_number = 42 tx_hash_second = '0x' + os.urandom(32).hex() tx_signed_second = '0x' + os.urandom(128).hex() @@ -93,6 +92,44 @@ def txs( ] +@pytest.fixture(scope='function') +def more_txs( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + ): + + session = init_database + + tx_number = 666 + tx_hash = '0x' + os.urandom(32).hex() + tx_signed = '0x' + os.urandom(128).hex() + nonce = 3 + + dt = datetime.datetime.utcnow() + dt += datetime.timedelta(hours=1) + db.add_transaction( + session, + tx_hash, + list_defaults['block']+2, + tx_number, + list_actors['alice'], + list_actors['diane'], + list_tokens['bar'], + list_tokens['bar'], + 2048, + 4096, + False, + dt.timestamp(), + ) + + session.commit() + + return [tx_hash] + txs + + @pytest.fixture(scope='function') def tag_txs( init_database, diff --git a/apps/cic-cache/tests/test_cache.py b/apps/cic-cache/tests/test_cache.py index cdd038c1..a5fdcc0d 100644 --- a/apps/cic-cache/tests/test_cache.py +++ b/apps/cic-cache/tests/test_cache.py @@ -8,6 +8,7 @@ import json import pytest # local imports +from cic_cache import db from cic_cache import BloomCache from cic_cache.cache import DataCache @@ -18,7 +19,6 @@ def test_cache( init_database, list_defaults, list_actors, - list_tokens, txs, ): @@ -37,9 +37,6 @@ def test_cache( def test_cache_data( init_database, - list_defaults, - list_actors, - list_tokens, txs, tag_txs, ): @@ -47,10 +44,209 @@ def test_cache_data( session = init_database c = DataCache(session) - b = c.load_transactions_with_data(410000, 420000) + b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True) assert len(b[2]) == 2 - assert b[2][0]['tx_hash'] == txs[1] - assert b[2][1]['tx_type'] == 'unknown' - assert b[2][0]['tx_type'] == 'test.taag' + assert b[2][0]['tx_hash'] == txs[0] + assert b[2][0]['tx_type'] == 'unknown' + assert b[2][1]['tx_type'] == 'test.taag' + + +def test_cache_ranges( + init_database, + list_defaults, + list_actors, + list_tokens, + more_txs, + ): + + session = init_database + + oldest = list_defaults['block'] - 1 + mid = list_defaults['block'] + newest = list_defaults['block'] + 2 + + c = BloomCache(session) + b = c.load_transactions(0, 100) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions(1, 2) + assert b[0] == oldest + assert b[1] == mid + + b = c.load_transactions(0, 2) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions(0, 1) + assert b[0] == newest + assert b[1] == newest + + b = c.load_transactions(0, 100, oldest=True) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions(0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == mid + + b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True) + assert b[0] == oldest + assert b[1] == mid + + # now check when supplying account + b = c.load_transactions_account(list_actors['alice'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions_account(list_actors['bob'], 0, 100) + assert b[0] == mid + assert b[1] == mid + + b = c.load_transactions_account(list_actors['diane'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + + # add block filter to the mix + b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == mid + assert b[1] == mid + + b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == oldest + + +def test_cache_ranges_data( + init_database, + list_defaults, + list_actors, + list_tokens, + more_txs, + ): + + session = init_database + + oldest = list_defaults['block'] - 1 + mid = list_defaults['block'] + newest = list_defaults['block'] + 2 + + c = DataCache(session) + + b = c.load_transactions_with_data(0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][2]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(1, 2) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[1] + assert b[2][1]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(0, 2) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_with_data(0, 1) + assert b[0] == newest + assert b[1] == newest + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[0] + + b = c.load_transactions_with_data(0, 100, oldest=True) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[2] + assert b[2][1]['tx_hash'] == more_txs[1] + assert b[2][2]['tx_hash'] == more_txs[0] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[1] + assert b[2][1]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[2] + assert b[2][1]['tx_hash'] == more_txs[1] + + # now check when supplying account + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + assert b[2][2]['tx_hash'] == more_txs[2] + + b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100) + assert b[0] == mid + assert b[1] == mid + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[2] + + # add block filter to the mix + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == mid + assert b[1] == mid + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == oldest + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[2] diff --git a/apps/cic-cache/tests/testdata/config/test.ini b/apps/cic-cache/tests/testdata/config/test.ini new file mode 100644 index 00000000..70903618 --- /dev/null +++ b/apps/cic-cache/tests/testdata/config/test.ini @@ -0,0 +1,2 @@ +[foo] +bar_baz = xyzzy diff --git a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py index ff63b895..eef7ecda 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py +++ b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py @@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp): def aux_setup(rpc, config, sender_address=ZERO_ADDRESS): - chain_spec_str = config.get('CIC_CHAIN_SPEC') + chain_spec_str = config.get('CHAIN_SPEC') chain_spec = ChainSpec.from_chain_str(chain_spec_str) token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') diff --git a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt index 9d7f6f9a..764056bb 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt @@ -1,5 +1,5 @@ celery==4.4.7 -erc20-demurrage-token~=0.0.2a3 -cic-eth-registry~=0.5.6a1 -chainlib~=0.0.5a1 -cic_eth~=0.12.0a2 +erc20-demurrage-token~=0.0.3a1 +cic-eth-registry~=0.5.8a1 +chainlib~=0.0.7a1 +cic_eth~=0.12.2a4 diff --git a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg index 69a41b7c..0e32d37f 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg +++ b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = cic-eth-aux-erc20-demurrage-token -version = 0.0.2a4 +version = 0.0.2a6 description = cic-eth tasks supporting erc20 demurrage token author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt index 0faedff4..a273b82a 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt @@ -5,8 +5,7 @@ pytest-cov==2.10.1 eth-tester==0.5.0b3 py-evm==0.3.0a20 SQLAlchemy==1.3.20 -cic-eth~=0.12.0a1 liveness~=0.0.1a7 -eth-accounts-index==0.0.12a1 -eth-contract-registry==0.5.6a1 -eth-address-index==0.1.2a1 +eth-accounts-index==0.1.1a1 +eth-contract-registry==0.5.8a1 +eth-address-index==0.2.1a1 diff --git a/apps/cic-eth/.coveragerc b/apps/cic-eth/.coveragerc index 47acdac6..8c586e19 100644 --- a/apps/cic-eth/.coveragerc +++ b/apps/cic-eth/.coveragerc @@ -6,4 +6,5 @@ omit = cic_eth/sync/head.py cic_eth/sync/mempool.py cic_eth/queue/state.py + cic_eth/cli *redis*.py diff --git a/apps/cic-eth/MANIFEST.in b/apps/cic-eth/MANIFEST.in index 5a9554ac..52d3a6f4 100644 --- a/apps/cic-eth/MANIFEST.in +++ b/apps/cic-eth/MANIFEST.in @@ -1,2 +1,2 @@ -include *requirements.txt config/test/* +include *requirements.txt config/test/* cic_eth/data/config/* diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 64448bfb..79501d30 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -520,9 +520,9 @@ class Api(ApiBase): s_external_get = celery.signature( external_task, [ - address, offset, limit, + address, ], queue=external_queue, ) diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py index b1646fd6..4add981b 100644 --- a/apps/cic-eth/cic_eth/check/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -21,7 +21,7 @@ def health(*args, **kwargs): session = SessionBase.create_session() config = kwargs['config'] - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec)) try: diff --git a/apps/cic-eth/cic_eth/check/signer.py b/apps/cic-eth/cic_eth/check/signer.py index 87b7785f..0395f0a1 100644 --- a/apps/cic-eth/cic_eth/check/signer.py +++ b/apps/cic-eth/cic_eth/check/signer.py @@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__) def health(*args, **kwargs): blocked = True max_attempts = 5 - conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer') + conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer') for i in range(max_attempts): idx = i + 1 logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts)) diff --git a/apps/cic-eth/cic_eth/cli/__init__.py b/apps/cic-eth/cic_eth/cli/__init__.py new file mode 100644 index 00000000..60eb0fb1 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/__init__.py @@ -0,0 +1,10 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp diff --git a/apps/cic-eth/cic_eth/cli/arg.py b/apps/cic-eth/cic_eth/cli/arg.py new file mode 100644 index 00000000..91fcc071 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/arg.py @@ -0,0 +1,31 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.REDIS: + self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') + self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') + self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') + if local_arg_flags & CICFlag.REDIS_CALLBACK: + self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') + self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') + + + diff --git a/apps/cic-eth/cic_eth/cli/base.py b/apps/cic-eth/cic_eth/cli/base.py new file mode 100644 index 00000000..b8b3ef2f --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 + REDIS = 16 + REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-eth/cic_eth/cli/celery.py b/apps/cic-eth/cic_eth/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-eth/cic_eth/cli/chain.py b/apps/cic-eth/cic_eth/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-eth/cic_eth/cli/config.py b/apps/cic-eth/cic_eth/cli/config.py new file mode 100644 index 00000000..8bfadc67 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} + if local_arg_flags & CICFlag.REDIS: + local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') + local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') + local_args_override['REDIS_DB'] = getattr(args, 'redis_db') + local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') + config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-eth/cic_eth/cli/rpc.py b/apps/cic-eth/cic_eth/cli/rpc.py new file mode 100644 index 00000000..25e28f9b --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/rpc.py @@ -0,0 +1,90 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import ( + EthUnixSignerConnection, + EthHTTPSignerConnection, + ) +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return self.get_by_label('default') + + + def get_by_label(self, label): + return RPCConnection.connect(self.chain_spec, label) + + + @staticmethod + def from_config(config, use_signer=False, default_label='default', signer_label='signer'): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label) + if use_signer: + + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label) + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + +# TOOD: re-implement file backend option from omittec code: +#broker = config.get('CELERY_BROKER_URL') +#if broker[:4] == 'file': +# bq = tempfile.mkdtemp() +# bp = tempfile.mkdtemp() +# conf_update = { +# 'broker_url': broker, +# 'broker_transport_options': { +# 'data_folder_in': bq, +# 'data_folder_out': bq, +# 'data_folder_processed': bp, +# }, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) +#else: +# conf_update = { +# 'broker_url': broker, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# +#result = config.get('CELERY_RESULT_URL') +#if result[:4] == 'file': +# rq = tempfile.mkdtemp() +# current_app.conf.update({ +# 'result_backend': 'file://{}'.format(rq), +# }) +# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) +#else: +# current_app.conf.update({ +# 'result_backend': result, +# }) +# diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-eth/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini similarity index 66% rename from apps/cic-eth/config/cic.ini rename to apps/cic-eth/cic_eth/data/config/cic.ini index d3840fac..103566ff 100644 --- a/apps/cic-eth/config/cic.ini +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -1,8 +1,6 @@ [cic] registry_address = -chain_spec = evm:bloxberg:8996 -tx_retry_delay = trust_address = -default_token_symbol = GFT +default_token_symbol = health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas run_dir = /run diff --git a/apps/cic-eth/cic_eth/data/config/database.ini b/apps/cic-eth/cic_eth/data/config/database.ini new file mode 100644 index 00000000..7a295a56 --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-eth/config/lash/syncer.ini b/apps/cic-eth/cic_eth/data/config/dispatcher.ini similarity index 58% rename from apps/cic-eth/config/lash/syncer.ini rename to apps/cic-eth/cic_eth/data/config/dispatcher.ini index 97236743..40e1b86e 100644 --- a/apps/cic-eth/config/lash/syncer.ini +++ b/apps/cic-eth/cic_eth/data/config/dispatcher.ini @@ -1,2 +1,2 @@ -[SYNCER] +[dispatcher] loop_interval = 1 diff --git a/apps/cic-eth/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini similarity index 64% rename from apps/cic-eth/config/eth.ini rename to apps/cic-eth/cic_eth/data/config/eth.ini index 3c589cb2..fd58e918 100644 --- a/apps/cic-eth/config/eth.ini +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -1,3 +1,2 @@ [eth] -provider = http://localhost:8545 gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/docker/redis.ini b/apps/cic-eth/cic_eth/data/config/redis.ini similarity index 54% rename from apps/cic-eth/config/docker/redis.ini rename to apps/cic-eth/cic_eth/data/config/redis.ini index ea321378..f836ebfd 100644 --- a/apps/cic-eth/config/docker/redis.ini +++ b/apps/cic-eth/cic_eth/data/config/redis.ini @@ -1,4 +1,5 @@ [redis] host = localhost -port = 63379 +port = 6379 db = 0 +timeout = 20.0 diff --git a/apps/cic-eth/cic_eth/data/config/retry.ini b/apps/cic-eth/cic_eth/data/config/retry.ini new file mode 100644 index 00000000..4bec6b6b --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/retry.ini @@ -0,0 +1,3 @@ +[retry] +delay = +batch_size = diff --git a/apps/cic-eth/cic_eth/data/config/signer.ini b/apps/cic-eth/cic_eth/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-eth/cic_eth/data/config/syncer.ini b/apps/cic-eth/cic_eth/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-eth/config/docker/tasks.ini b/apps/cic-eth/cic_eth/data/config/tasks.ini similarity index 100% rename from apps/cic-eth/config/docker/tasks.ini rename to apps/cic-eth/cic_eth/data/config/tasks.ini diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py index 77083874..3adefa2e 100644 --- a/apps/cic-eth/cic_eth/ext/tx.py +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -12,9 +12,11 @@ from chainlib.eth.tx import ( transaction_by_block, receipt, ) +from chainlib.eth.error import RequestMismatchException from chainlib.eth.block import block_by_number from chainlib.eth.contract import abi_decode_single from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.tx import Tx from hexathon import strip_0x from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token @@ -23,6 +25,8 @@ from chainqueue.db.models.otx import Otx from chainqueue.db.enum import StatusEnum from chainqueue.sql.query import get_tx_cache from eth_erc20 import ERC20 +from erc20_faucet import Faucet +from potaahto.symbols import snake_and_camel # local imports from cic_eth.queue.time import tx_times @@ -35,6 +39,32 @@ logg = logging.getLogger() MAX_BLOCK_TX = 250 +def parse_transaction(chain_spec, rpc, tx, sender_address=None): + try: + transfer_data = ERC20.parse_transfer_request(tx['input']) + tx_address = transfer_data[0] + tx_token_value = transfer_data[1] + logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value)) + return (tx_address, tx_token_value) + except RequestMismatchException: + pass + + try: + transfer_data = Faucet.parse_give_to_request(tx['input']) + tx_address = transfer_data[0] + c = Faucet(chain_spec) + o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number']) + r = rpc.do(o) + tx_token_value = Faucet.parse_token_amount(r) + logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value)) + return (tx_address, tx_token_value) + + except RequestMismatchException: + pass + + return None + + # TODO: Make this method easier to read @celery_app.task(bind=True, base=BaseTask) def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): @@ -71,36 +101,39 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data) txs = {} + logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high'])) for block_height in range(bloomspec['low'], bloomspec['high']): block_height_bytes = block_height.to_bytes(4, 'big') if block_filter.check(block_height_bytes): logg.debug('filter matched block {}'.format(block_height)) o = block_by_number(block_height) block = rpc.do(o) - logg.debug('block {}'.format(block)) for tx_index in range(0, len(block['transactions'])): - composite = tx_index + block_height - tx_index_bytes = composite.to_bytes(4, 'big') - if tx_filter.check(tx_index_bytes): + tx_index_bytes = tx_index.to_bytes(4, 'big') + composite = block_height_bytes + tx_index_bytes + if tx_filter.check(composite): logg.debug('filter matched block {} tx {}'.format(block_height, tx_index)) + o = transaction_by_block(block['hash'], tx_index) try: - #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index) - o = transaction_by_block(block['hash'], tx_index) tx = rpc.do(o) except Exception as e: logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e)) continue + + tx = Tx(tx).src() + + logg.debug('got tx {}'.format(tx)) tx_address = None tx_token_value = 0 - try: - transfer_data = ERC20.parse_transfer_request(tx['data']) - tx_address = transfer_data[0] - tx_token_value = transfer_data[1] - except ValueError: - logg.debug('not a transfer transaction, skipping {}'.format(tx)) + + transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address) + if transfer_data == None: continue + tx_address = transfer_data[0] + tx_token_value = transfer_data[1] + if address == tx_address: status = StatusEnum.SENT try: @@ -136,6 +169,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): return txs + # TODO: Surely it must be possible to optimize this # TODO: DRY this with callback filter in cic_eth/runnable/manager # TODO: Remove redundant fields from end representation (timestamp, tx_hash) diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 973ac2f5..31dd1991 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -7,54 +7,30 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home +from chainlib.chain import ChainSpec # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) -logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) +logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'no_register': None, } -config.dict_override(args_override, 'cli') - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): redis_host = config.get('REDIS_HOST') @@ -68,20 +44,20 @@ def main(): ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE'), ) - register = not args.no_register + register = not config.get('_NO_REGISTER') logg.debug('register {}'.format(register)) t = api.create_account(register=register) ps.get_message() try: - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) except TimeoutError as e: sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e)) sys.exit(1) diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index fc35548e..5f260d16 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -12,64 +12,38 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.address import is_checksum_address # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi from cic_eth.db.enum import LockEnum logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task | cic_eth.cli.argflag_local_chain +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') +argparser.process_local_flags(local_arg_flags) def process_lock_args(argparser): argparser.add_argument('flags', type=str, help='Flags to manipulate') argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,') -sub = argparser.add_subparsers() +sub = argparser.add_subparsers(help='') sub.dest = "command" sub_lock = sub.add_parser('lock', help='Set or reset locks') sub_unlock = sub.add_parser('unlock', help='Set or reset locks') process_lock_args(sub_lock) process_lock_args(sub_unlock) - args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -# override args -config.dict_override(args_override, 'cli') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -queue = args.q - -chain_spec = None -if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::': - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) admin_api = AdminApi(None) @@ -100,7 +74,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('unlock {} on {} task {}'.format(flags, args.address, t)) @@ -119,7 +93,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('lock {} on {} task {}'.format(flags, args.address, t)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index 7e22e911..64b51937 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -8,8 +8,7 @@ import sys import re import datetime -# third-party imports -import confini +# external imports import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -24,7 +23,7 @@ from chainqueue.error import NotLocalTxError from chainqueue.sql.state import set_reserved # local imports -import cic_eth +import cic_eth.cli from cic_eth.db import SessionBase from cic_eth.db.enum import LockEnum from cic_eth.db import dsn_from_config @@ -39,51 +38,30 @@ from cic_eth.error import ( logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -config_dir = os.path.join('/usr/local/etc/cic-eth') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() run = True + class DispatchSyncer: yield_delay = 0.0005 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/retry.py b/apps/cic-eth/cic_eth/runnable/daemons/retry.py index 8974c33d..354fd5bc 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/retry.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/retry.py @@ -6,7 +6,6 @@ import argparse import re # external imports -import confini import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -14,6 +13,7 @@ from chainlib.connection import RPCConnection from chainsyncer.filter import SyncFilter # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.db import SessionBase from cic_eth.admin.ctrl import lock_send @@ -25,66 +25,41 @@ from cic_eth.stat import init_chain_stat logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--batch-size', dest='batch_size', type=int, default=50, help='max amount of txs to resend per iteration') -argparser.add_argument('--retry-delay', dest='retry_delay', type=int, help='seconds to wait for retrying a transaction that is marked as sent') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) +argparser.add_argument('--retry-delay', dest='retry_delay', type=int, default=20, help='seconds to wait for retrying a transaction that is marked as sent') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'), +extra_args = { + 'retry_delay': 'RETRY_DELAY', + 'batch_size': 'RETRY_BATCH_SIZE', } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.batch_size, '_BATCH_SIZE', True) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) - -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() + def main(): - conn = RPCConnection.connect(chain_spec, 'default') - - straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) + straggler_delay = int(config.get('RETRY_DELAY')) loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: stat = init_chain_stat(conn) loop_interval = stat.block_average() - syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE')) + syncer = RetrySyncer(conn, chain_spec, cic_eth.cli.chain_interface, straggler_delay, batch_size=config.get('RETRY_BATCH_SIZE')) syncer.backend.set(0, 0) - fltr = StragglerFilter(chain_spec, queue=queue) + fltr = StragglerFilter(chain_spec, queue=config.get('CELERY_QUEUE')) syncer.add_filter(fltr) syncer.loop(int(loop_interval), conn) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 837da5cb..fab470f3 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -21,14 +21,17 @@ from chainlib.eth.connection import ( EthUnixSignerConnection, EthHTTPSignerConnection, ) +from chainlib.eth.address import to_checksum_address from chainlib.chain import ChainSpec from chainqueue.db.models.otx import Otx from cic_eth_registry.error import UnknownContractError from cic_eth_registry.erc20 import ERC20Token +from hexathon import add_0x import liveness.linux # local imports +import cic_eth.cli from cic_eth.eth import ( erc20, tx, @@ -70,114 +73,53 @@ from cic_eth.task import BaseTask logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -script_dir = os.path.dirname(os.path.realpath(__file__)) - -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks') -argparser.add_argument('-r', type=str, help='CIC registry address') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use') argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path') argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'), - 'ETH_PROVIDER': getattr(args, 'p'), - 'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'), +# process config +extra_args = { + 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL', + 'aux_all': None, + 'aux': None, + 'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS', } -config.add(args.q, '_CELERY_QUEUE', True) -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(args.c, config)) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -health_modules = config.get('CIC_HEALTH_MODULES', []) -if len(health_modules) != 0: - health_modules = health_modules.split(',') -logg.debug('health mods {}'.format(health_modules)) +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config, use_signer=True) +conn = rpc.get_default() # connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) - - -# set up celery -current_app = celery.Celery(__name__) - -broker = config.get('CELERY_BROKER_URL') -if broker[:4] == 'file': - bq = tempfile.mkdtemp() - bp = tempfile.mkdtemp() - conf_update = { - 'broker_url': broker, - 'broker_transport_options': { - 'data_folder_in': bq, - 'data_folder_out': bq, - 'data_folder_processed': bp, - }, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) -else: - conf_update = { - 'broker_url': broker, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - -result = config.get('CELERY_RESULT_URL') -if result[:4] == 'file': - rq = tempfile.mkdtemp() - current_app.conf.update({ - 'result_backend': 'file://{}'.format(rq), - }) - logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) -else: - current_app.conf.update({ - 'result_backend': result, - }) - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) -RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default') -RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer') - Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') -#import cic_eth.checks.gas -#if not cic_eth.checks.gas.health(config=config): -# raise RuntimeError() + +# execute health checks +# TODO: health should be separate service with endpoint that can be queried +health_modules = config.get('CIC_HEALTH_MODULES', []) +if len(health_modules) != 0: + health_modules = health_modules.split(',') +logg.debug('health mods {}'.format(health_modules)) liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker') -rpc = RPCConnection.connect(chain_spec, 'default') + +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) @@ -188,15 +130,15 @@ if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') -for address in trusted_addresses: +for i, address in enumerate(trusted_addresses): + if config.get('_UNSAFE'): + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) +connect_declarator(conn, chain_spec, trusted_addresses) +connect_token_registry(conn, chain_spec) -connect_declarator(rpc, chain_spec, trusted_addresses) -connect_token_registry(rpc, chain_spec) - -# detect aux +# detect auxiliary task modules (plugins) # TODO: move to separate file -#aux_dir = os.path.join(script_dir, '..', '..', 'aux') aux = [] if args.aux_all: if len(args.aux) > 0: @@ -249,36 +191,24 @@ elif len(args.aux) > 0: for v in aux: mname = 'cic_eth_aux.' + v mod = importlib.import_module(mname) - mod.aux_setup(rpc, config) + mod.aux_setup(conn, config) logg.info('loaded aux module {}'.format(mname)) def main(): argv = ['worker'] - if args.vv: - argv.append('--loglevel=DEBUG') - elif args.v: - argv.append('--loglevel=INFO') + log_level = logg.getEffectiveLevel() + log_level_name = logging.getLevelName(log_level) + argv.append('--loglevel=' + log_level_name) argv.append('-Q') - argv.append(args.q) + argv.append(config.get('CELERY_QUEUE')) argv.append('-n') - argv.append(args.q) - -# if config.true('SSL_ENABLE_CLIENT'): -# Callback.ssl = True -# Callback.ssl_cert_file = config.get('SSL_CERT_FILE') -# Callback.ssl_key_file = config.get('SSL_KEY_FILE') -# Callback.ssl_password = config.get('SSL_PASSWORD') -# -# if config.get('SSL_CA_FILE') != '': -# Callback.ssl_ca_file = config.get('SSL_CA_FILE') - - rpc = RPCConnection.connect(chain_spec, 'default') + argv.append(config.get('CELERY_QUEUE')) BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) - default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address) - default_token.load(rpc) + default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address) + default_token.load(conn) BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name @@ -286,13 +216,13 @@ def main(): logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) liveness.linux.set(rundir=config.get('CIC_RUN_DIR')) - current_app.worker_main(argv) + celery_app.worker_main(argv) liveness.linux.reset(rundir=config.get('CIC_RUN_DIR')) @celery.signals.eventlet_pool_postshutdown.connect def shutdown(sender=None, headers=None, body=None, **kwargs): - logg.warning('in shudown event hook') + logg.warning('in shutdown event hook') if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 94b29883..5117537c 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -8,14 +8,6 @@ import sys import re # external imports -import confini -import celery -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS @@ -30,8 +22,13 @@ from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver.head import HeadSyncer from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase +from chainlib.eth.address import ( + is_checksum_address, + to_checksum_address, + ) # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.runnable.daemons.filters import ( CallbackFilter, @@ -47,61 +44,50 @@ from cic_eth.registry import ( connect_token_registry, ) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -script_dir = os.path.realpath(os.path.dirname(__file__)) +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process config +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) +# connect to celery +cic_eth.cli.CeleryApp.from_config(config) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) - -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.y, '_KEYSTORE_FILE', True) -config.add(args.q, '_CELERY_QUEUE', True) -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# set up database dsn = dsn_from_config(config) - SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) - -rpc = RPCConnection.connect(chain_spec, 'default') +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) - def main(): - # connect to celery - celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - # Connect to blockchain with chainlib o = block_latest() - r = rpc.do(o) + r = conn.do(o) block_current = int(r, 16) block_offset = block_current + 1 loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: - stat = init_chain_stat(rpc, block_start=block_current) + stat = init_chain_stat(conn, block_start=block_current) loop_interval = stat.block_average() logg.debug('current block height {}'.format(block_offset)) @@ -113,9 +99,9 @@ def main(): syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.true('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -128,40 +114,45 @@ def main(): for syncer_backend in syncer_backends: try: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface)) logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend)) except AttributeError: logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface)) - connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') - for address in trusted_addresses: + for i, address in enumerate(trusted_addresses): + if not config.get('_UNSAFE'): + if not is_checksum_address(address): + raise ValueError('address {} is not a valid checksum address'.format(address)) + else: + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) - connect_declarator(rpc, chain_spec, trusted_addresses) - connect_token_registry(rpc, chain_spec) + connect_declarator(conn, chain_spec, trusted_addresses) + connect_token_registry(conn, chain_spec) CallbackFilter.trusted_addresses = trusted_addresses callback_filters = [] for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','): task_split = cb.split(':') - task_queue = config.get('_CELERY_QUEUE') + task_queue = config.get('CELERY_QUEUE') if len(task_split) > 1: task_queue = task_split[0] callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue) callback_filters.append(callback_filter) - tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE')) + tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE')) account_registry_address = registry.by_name('AccountRegistry') - registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE')) + registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE')) - gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE')) + gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE')) #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) @@ -176,7 +167,7 @@ def main(): for cf in callback_filters: syncer.add_filter(cf) - r = syncer.loop(int(loop_interval), rpc) + r = syncer.loop(int(loop_interval), conn) sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) i += 1 diff --git a/apps/cic-eth/cic_eth/runnable/info.py b/apps/cic-eth/cic_eth/runnable/info.py index f8e7b50f..40e85103 100644 --- a/apps/cic-eth/cic_eth/runnable/info.py +++ b/apps/cic-eth/cic_eth/runnable/info.py @@ -12,50 +12,27 @@ import confini import celery # local imports +import cic_eth.cli from cic_eth.api import Api from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli args') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - -queue = args.q - -api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue) +api = Api(config.get('CHAIN_SPEC'), queue=config.get('CELERY_QUEUE')) admin_api = AdminApi(None) + def main(): t = admin_api.registry() registry_address = t.get() diff --git a/apps/cic-eth/cic_eth/runnable/resend.py b/apps/cic-eth/cic_eth/runnable/resend.py index 6901ef34..9861e293 100644 --- a/apps/cic-eth/cic_eth/runnable/resend.py +++ b/apps/cic-eth/cic_eth/runnable/resend.py @@ -5,65 +5,38 @@ import re import os # third-party imports -import celery -import confini from chainlib.chain import ChainSpec from chainlib.eth.connection import EthHTTPConnection # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -logging.getLogger('web3').setLevel(logging.WARNING) -logging.getLogger('urllib3').setLevel(logging.WARNING) - -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string') -argparser.add_argument('--unlock', action='store_true', help='Append task to unlock account') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('tx_hash', type=str, help='Transaction hash') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--unlock', action='store_true', help='Unlock account after resend') +argparser.add_positional('tx_hash', type=str, help='Transaction hash') +argparser.process_local_flags(local_arg_flags) +extra_args = { + 'unlock': None, + 'tx_hash': None, + } args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) -if args.vv: - logg.setLevel(logging.DEBUG) -elif args.v: - logg.setLevel(logging.INFO) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } +celery_app = cic_eth.cli.CeleryApp.from_config(config) -# override args -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.tx_hash, '_TX_HASH', True) -config.add(args.unlock, '_UNLOCK', True) - -chain_spec = ChainSpec.from_chain_str(args.i) - -rpc = EthHTTPConnection(config.get('ETH_PROVIDER')) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) def main(): - api = AdminApi(rpc) - tx_details = api.tx(chain_spec, args.tx_hash) + api = AdminApi(None) + tx_details = api.tx(chain_spec, config.get('_TX_HASH')) t = api.resend(args.tx_hash, chain_spec, unlock=config.get('_UNLOCK')) print(t.get_leaf()) diff --git a/apps/cic-eth/cic_eth/runnable/tag.py b/apps/cic-eth/cic_eth/runnable/tag.py index f7d6af11..6aeffc90 100644 --- a/apps/cic-eth/cic_eth/runnable/tag.py +++ b/apps/cic-eth/cic_eth/runnable/tag.py @@ -6,8 +6,7 @@ import argparse import re # external imports -import celery -import confini +import cic_eth.cli from chainlib.chain import ChainSpec from xdg.BaseDirectory import xdg_config_home @@ -19,43 +18,28 @@ from cic_eth.db.models.base import SessionBase logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_positional('tag', type=str, help='address tag') +argparser.add_positional('address', type=str, help='address') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -argparser.add_argument('tag', type=str, help='address tag') -argparser.add_argument('address', type=str, help='address') -args = argparser.parse_args(sys.argv[1:]) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +admin_api = AdminApi(None) -config = confini.Config(args.c) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}\n{}'.format(args.c, config)) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -chain_spec = ChainSpec.from_chain_str(args.i) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +celery_app = cic_eth.cli.CeleryApp.from_config(config) +api = AdminApi(None) def main(): - api = AdminApi(None) - api.tag_account(args.tag, args.address, chain_spec) + admin_api.tag_account(args.tag, args.address, chain_spec) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/transfer.py b/apps/cic-eth/cic_eth/runnable/transfer.py index c63c72a7..4bbaa306 100644 --- a/apps/cic-eth/cic_eth/runnable/transfer.py +++ b/apps/cic-eth/cic_eth/runnable/transfer.py @@ -7,88 +7,58 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home from chainlib.eth.address import to_checksum_address # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() -argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Symbol of token to transfer') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('sender', type=str, help='Transaction sender') -argparser.add_argument('recipient', type=str, help='Transaction recipient') -argparser.add_argument('value', type=int, help='Transaction value with decimals') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol') +argparser.add_positional('sender', type=str, help='Token transaction sender') +argparser.add_positional('recipient', type=str, help='Token transaction recipient') +argparser.add_positional('value', type=int, help='Token transaction value') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'token_symbol': None, + 'sender': None, + 'recipient': None, + 'value': None, } -config.dict_override(args_override, 'cli') -config.add(to_checksum_address(args.sender), '_SENDER', True) -config.add(to_checksum_address(args.recipient), '_RECIPIENT', True) -config.add(args.value, '_VALUE', True) -config.add(args.token_symbol, '_SYMBOL', True) -if config.get('_SYMBOL') == None: - raise ValueError('gas transfers not yet supported; token symbol required') -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) + +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): - redis_host = config.get('REDIS_HOST') - redis_port = config.get('REDIS_PORT') - redis_db = config.get('REDIS_DB') redis_channel = str(uuid.uuid4()) - r = redis.Redis(redis_host, redis_port, redis_db) + r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB')) ps = r.pubsub() ps.subscribe(redis_channel) ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE') ) - t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL')) + t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_TOKEN_SYMBOL')) ps.get_message() - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) m = json.loads(o['data']) print(m['result']) diff --git a/apps/cic-eth/cic_eth/sync/retry.py b/apps/cic-eth/cic_eth/sync/retry.py index b311964d..9118ebf4 100644 --- a/apps/cic-eth/cic_eth/sync/retry.py +++ b/apps/cic-eth/cic_eth/sync/retry.py @@ -3,7 +3,7 @@ import logging import datetime # external imports -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainsyncer.backend.memory import MemBackend from chainsyncer.error import NoBlockForYou from chainlib.eth.block import ( @@ -39,9 +39,9 @@ class DbSessionMemBackend(MemBackend): class RetrySyncer(HeadSyncer): - def __init__(self, conn, chain_spec, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): + def __init__(self, conn, chain_spec, chain_interface, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): backend = DbSessionMemBackend(chain_spec, None) - super(RetrySyncer, self).__init__(backend) + super(RetrySyncer, self).__init__(backend, chain_interface) self.chain_spec = chain_spec if failed_grace_seconds == None: failed_grace_seconds = stalled_grace_seconds diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 039b9baf..f9ac618b 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 2, - 'alpha.3', + 'alpha.4', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/config/bancor.ini b/apps/cic-eth/config/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/celery.ini b/apps/cic-eth/config/celery.ini deleted file mode 100644 index 038f83ad..00000000 --- a/apps/cic-eth/config/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis:// -result_url = redis:// -debug = 0 diff --git a/apps/cic-eth/config/custody.ini b/apps/cic-eth/config/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/database.ini b/apps/cic-eth/config/database.ini deleted file mode 100644 index 4517445c..00000000 --- a/apps/cic-eth/config/database.ini +++ /dev/null @@ -1,10 +0,0 @@ -[database] -NAME=cic-eth -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=postgresql -DRIVER=psycopg2 -POOL_SIZE=50 -DEBUG=0 diff --git a/apps/cic-eth/config/dispatcer.ini b/apps/cic-eth/config/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/bancor.ini b/apps/cic-eth/config/docker/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/docker/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/docker/celery.ini b/apps/cic-eth/config/docker/celery.ini deleted file mode 100644 index 646d6603..00000000 --- a/apps/cic-eth/config/docker/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis://localhost:63379 -result_url = redis://localhost:63379 -debug = 0 diff --git a/apps/cic-eth/config/docker/chain.ini b/apps/cic-eth/config/docker/chain.ini new file mode 100644 index 00000000..3a23481e --- /dev/null +++ b/apps/cic-eth/config/docker/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = evm:bloxberg:8996 diff --git a/apps/cic-eth/config/docker/cic.ini b/apps/cic-eth/config/docker/cic.ini deleted file mode 100644 index 4fdefca0..00000000 --- a/apps/cic-eth/config/docker/cic.ini +++ /dev/null @@ -1,8 +0,0 @@ -[cic] -registry_address = -chain_spec = evm:bloxberg:8996 -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -tx_retry_delay = 20 -default_token_symbol = GFT -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas -run_dir = /run diff --git a/apps/cic-eth/config/docker/custody.ini b/apps/cic-eth/config/docker/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/docker/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/docker/database.ini b/apps/cic-eth/config/docker/database.ini deleted file mode 100644 index 6448be0e..00000000 --- a/apps/cic-eth/config/docker/database.ini +++ /dev/null @@ -1,10 +0,0 @@ -[database] -NAME=cic_eth -USER=postgres -PASSWORD=tralala -HOST=localhost -PORT=63432 -ENGINE=postgresql -DRIVER=psycopg2 -POOL_SIZE=50 -DEBUG=0 diff --git a/apps/cic-eth/config/docker/dispatcer.ini b/apps/cic-eth/config/docker/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/docker/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/eth.ini b/apps/cic-eth/config/docker/eth.ini deleted file mode 100644 index 534a2eea..00000000 --- a/apps/cic-eth/config/docker/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://localhost:63545 -gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/docker/signer.ini b/apps/cic-eth/config/docker/signer.ini deleted file mode 100644 index fe72206f..00000000 --- a/apps/cic-eth/config/docker/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/docker/ssl.ini b/apps/cic-eth/config/docker/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/docker/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/docker/syncer.ini b/apps/cic-eth/config/docker/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/docker/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/lash/bancor.ini b/apps/cic-eth/config/lash/bancor.ini deleted file mode 100644 index 0bff1ce0..00000000 --- a/apps/cic-eth/config/lash/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549 -dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6 diff --git a/apps/cic-eth/config/lash/database.ini b/apps/cic-eth/config/lash/database.ini deleted file mode 100644 index 77ac57d4..00000000 --- a/apps/cic-eth/config/lash/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -NAME=cic-eth -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=sqlite -DRIVER=pysqlite diff --git a/apps/cic-eth/config/lash/eth.ini b/apps/cic-eth/config/lash/eth.ini deleted file mode 100644 index 324dba08..00000000 --- a/apps/cic-eth/config/lash/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -provider = http://localhost:8545 diff --git a/apps/cic-eth/config/lash/redis.ini b/apps/cic-eth/config/lash/redis.ini deleted file mode 100644 index d21df9f8..00000000 --- a/apps/cic-eth/config/lash/redis.ini +++ /dev/null @@ -1,3 +0,0 @@ -[celery] -broker_url = redis:// -result_url = file:// diff --git a/apps/cic-eth/config/lash/signer.ini b/apps/cic-eth/config/lash/signer.ini deleted file mode 100644 index 9222aef8..00000000 --- a/apps/cic-eth/config/lash/signer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[signer] -socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc diff --git a/apps/cic-eth/config/lash/ssl.ini b/apps/cic-eth/config/lash/ssl.ini deleted file mode 100644 index 746ff856..00000000 --- a/apps/cic-eth/config/lash/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = true -cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt -key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key -password = test -ca_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt diff --git a/apps/cic-eth/config/redis.ini b/apps/cic-eth/config/redis.ini deleted file mode 100644 index 1a7acb52..00000000 --- a/apps/cic-eth/config/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = -port = -db = diff --git a/apps/cic-eth/config/signer.ini b/apps/cic-eth/config/signer.ini deleted file mode 100644 index 0fa1f015..00000000 --- a/apps/cic-eth/config/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = /run/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/ssl.ini b/apps/cic-eth/config/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/syncer.ini b/apps/cic-eth/config/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/tasks.ini b/apps/cic-eth/config/tasks.ini deleted file mode 100644 index 732d797e..00000000 --- a/apps/cic-eth/config/tasks.ini +++ /dev/null @@ -1,3 +0,0 @@ -[tasks] -transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop -trace_queue_status = 1 diff --git a/apps/cic-eth/config/test/chain.ini b/apps/cic-eth/config/test/chain.ini new file mode 100644 index 00000000..9fda0988 --- /dev/null +++ b/apps/cic-eth/config/test/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 64ce7d1b..e627361c 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -27,6 +27,12 @@ RUN python setup.py install ENV PYTHONPATH . +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY \ + --extra-index-url $EXTRA_INDEX_URL \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + COPY docker/entrypoints/* ./ RUN chmod 755 *.sh diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 09811a0f..078438ae 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.6a1,<0.1.0 +chainlib-eth>=0.0.7a7,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/scripts/migrate.py b/apps/cic-eth/scripts/migrate.py index d3602989..a316967e 100644 --- a/apps/cic-eth/scripts/migrate.py +++ b/apps/cic-eth/scripts/migrate.py @@ -10,6 +10,7 @@ from alembic.config import Config as AlembicConfig import confini from cic_eth.db import dsn_from_config +import cic_eth.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -19,25 +20,20 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) dbdir = os.path.join(rootdir, 'cic_eth', 'db') migrationsdir = os.path.join(dbdir, 'migrations') -config_dir = os.path.join('/usr/local/etc/cic-eth') +arg_flags = cic_eth.cli.argflag_std_base -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') argparser.add_argument('-f', action='store_true', help='force action') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() +extra_args = { + 'migrations_dir': None, + 'reset': None, + 'f': '_FORCE_ACTION', + } +config = cic_eth.cli.Config.from_args(args, arg_flags, 0, extra_args=extra_args) config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug('config:\n{}'.format(config)) diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index c70193a6..80d8009c 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -1,15 +1,15 @@ chainqueue>=0.0.3a2,<0.1.0 -chainsyncer[sql]>=0.0.5a1,<0.1.0 +chainsyncer[sql]>=0.0.6a1,<0.1.0 alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 liveness~=0.0.1a7 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a5,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a2,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a2,<0.1.0 moolb~=0.1.1b2 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index a0354019..2b798095 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -24,8 +24,10 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_eth + cic_eth.cli cic_eth.admin cic_eth.eth cic_eth.api diff --git a/apps/cic-eth/test_requirements.txt b/apps/cic-eth/test_requirements.txt index cf7c693d..bf5ecdcc 100644 --- a/apps/cic-eth/test_requirements.txt +++ b/apps/cic-eth/test_requirements.txt @@ -6,4 +6,4 @@ pytest-redis==2.0.0 redis==3.5.3 eth-tester==0.5.0b3 py-evm==0.3.0a20 -eth-erc20~=0.0.11a1 +eth-erc20~=0.0.12a1 diff --git a/apps/cic-eth/tests/check/test_check_gas.py b/apps/cic-eth/tests/check/test_check_gas.py index ea0ffb88..056120c8 100644 --- a/apps/cic-eth/tests/check/test_check_gas.py +++ b/apps/cic-eth/tests/check/test_check_gas.py @@ -11,7 +11,7 @@ def test_check_gas( whoever, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tests/check/test_check_signer.py b/apps/cic-eth/tests/check/test_check_signer.py index 5572e195..63959de9 100644 --- a/apps/cic-eth/tests/check/test_check_signer.py +++ b/apps/cic-eth/tests/check/test_check_signer.py @@ -9,5 +9,5 @@ def test_check_signer( eth_rpc, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tests/unit/ext/test_ext_tx.py b/apps/cic-eth/tests/task/test_ext_tx.py similarity index 84% rename from apps/cic-eth/tests/unit/ext/test_ext_tx.py rename to apps/cic-eth/tests/task/test_ext_tx.py index 90241865..2740bf20 100644 --- a/apps/cic-eth/tests/unit/ext/test_ext_tx.py +++ b/apps/cic-eth/tests/task/test_ext_tx.py @@ -20,7 +20,6 @@ from cic_eth.db.models.nonce import ( logg = logging.getLogger() -# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state def test_filter_process( init_database, default_chain_spec, @@ -48,10 +47,10 @@ def test_filter_process( eth_rpc.do(o) o = receipt(tx_hash_hex) r = eth_rpc.do(o) - a = r['block_number'] - b.add(a.to_bytes(4, 'big')) - a = r['block_number'] + r['transaction_index'] - t.add(a.to_bytes(4, 'big')) + block_bytes = r['block_number'].to_bytes(4, 'big') + b.add(block_bytes) + tx_index_bytes = r['transaction_index'].to_bytes(4, 'big') + t.add(block_bytes + tx_index_bytes) tx_hashes.append(tx_hash_hex) # external tx @@ -61,10 +60,10 @@ def test_filter_process( eth_rpc.do(o) o = receipt(tx_hash_hex) r = eth_rpc.do(o) - a = r['block_number'] - b.add(a.to_bytes(4, 'big')) - a = r['block_number'] + r['transaction_index'] - t.add(a.to_bytes(4, 'big')) + block_bytes = r['block_number'].to_bytes(4, 'big') + b.add(block_bytes) + tx_index_bytes = r['transaction_index'].to_bytes(4, 'big') + t.add(block_bytes + tx_index_bytes) tx_hashes.append(tx_hash_hex) init_eth_tester.mine_blocks(10) diff --git a/apps/cic-eth/tests/testdata/config/test.ini b/apps/cic-eth/tests/testdata/config/test.ini new file mode 100644 index 00000000..70903618 --- /dev/null +++ b/apps/cic-eth/tests/testdata/config/test.ini @@ -0,0 +1,2 @@ +[foo] +bar_baz = xyzzy diff --git a/apps/cic-eth/tests/unit/cli/test_cli_args.py b/apps/cic-eth/tests/unit/cli/test_cli_args.py new file mode 100644 index 00000000..bda5f183 --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_args.py @@ -0,0 +1,56 @@ +# standard imports +import os +import logging + +# external imports +import chainlib.cli + +# local imports +import cic_eth.cli + +logg = logging.getLogger() + +script_dir = os.path.dirname(os.path.realpath(__file__)) +#config_dir = os.path.join(script_dir, '..', '..', 'testdata', 'config') + + +def test_argumentparser_to_config(): + + argparser = cic_eth.cli.ArgumentParser() + + local_flags = 0xffff + argparser.process_local_flags(local_flags) + argparser.add_argument('--foo', type=str) + args = argparser.parse_args([ + '--redis-host', 'foo', + '--redis-port', '123', + '--redis-db', '0', + '--redis-host-callback', 'bar', + '--redis-port-callback', '456', + '--redis-timeout', '10.0', + '-q', 'baz', + '--offset', '13', + '--no-history', + '-r','0xdeadbeef', + '-vv', + '--foo', 'bar', + ]) + + extra_args = { + 'foo': '_BARBARBAR', + } + #config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir) + config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args) + + assert config.get('_BARBARBAR') == 'bar' + assert config.get('REDIS_HOST') == 'foo' + assert config.get('REDIS_PORT') == 123 + assert config.get('REDIS_DB') == 0 + assert config.get('_REDIS_HOST_CALLBACK') == 'bar' + assert config.get('_REDIS_PORT_CALLBACK') == 456 + assert config.get('REDIS_TIMEOUT') == 10.0 + assert config.get('CELERY_QUEUE') == 'baz' + assert config.get('SYNCER_NO_HISTORY') == True + assert config.get('SYNCER_OFFSET') == 13 + assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef' + diff --git a/apps/cic-eth/tests/unit/cli/test_cli_celery.py b/apps/cic-eth/tests/unit/cli/test_cli_celery.py new file mode 100644 index 00000000..1727008b --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_celery.py @@ -0,0 +1,17 @@ +# standard imports +import tempfile + +# local imports +import cic_eth.cli + + +def test_cli_celery(): + cf = tempfile.mkdtemp() + + config = { + 'CELERY_RESULT_URL': 'filesystem://' + cf, + } + cic_eth.cli.CeleryApp.from_config(config) + + config['CELERY_BROKER_URL'] = 'filesystem://' + cf + cic_eth.cli.CeleryApp.from_config(config) diff --git a/apps/cic-eth/tests/unit/cli/test_cli_chain.py b/apps/cic-eth/tests/unit/cli/test_cli_chain.py new file mode 100644 index 00000000..8ebebc15 --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_chain.py @@ -0,0 +1,64 @@ +# external imports +import pytest +from chainlib.eth.gas import ( + Gas, + RPCGasOracle, + ) +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.block import ( + block_latest, + Block, + ) + +# local imports +import cic_eth.cli + + +@pytest.mark.xfail() +def test_cli_rpc( + eth_rpc, + eth_signer, + default_chain_spec, + ): + config = { + 'CHAIN_SPEC': str(default_chain_spec), + 'RPC_HTTP_PROVIDER': 'http://localhost:8545', + } + rpc = cic_eth.cli.RPC.from_config(config, default_label='foo') + conn = rpc.get_by_label('foo') + #o = block_latest() + #conn.do(o) + + +def test_cli_chain( + default_chain_spec, + eth_rpc, + eth_signer, + contract_roles, + agent_roles, + ): + ifc = cic_eth.cli.EthChainInterface() + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc) + gas_oracle = RPCGasOracle(conn=eth_rpc) + c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer) + (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], 1024) + r = eth_rpc.do(o) + + o = ifc.tx_receipt(r) + r = eth_rpc.do(o) + assert r['status'] == 1 + + o = ifc.block_by_number(1) + block_src = eth_rpc.do(o) + block = ifc.block_from_src(block_src) + assert block.number == 1 + + with pytest.raises(KeyError): + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + + block_src = ifc.src_normalize(block_src) + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 03a00145..9dcf8d51 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,7 +1,7 @@ crypto-dev-signer>=0.4.14b7,<=0.4.14 -chainqueue~=0.0.2b6 +chainqueue>=0.0.3a1,<0.1.0 confini>=0.3.6rc4,<0.5.0 -cic-eth-registry>=0.5.7a1,<0.6.0 +cic-eth-registry>=0.5.8a1,<0.6.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index b0b81f95..9d71a990 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -13,5 +13,11 @@ redis==3.5.3 semver==2.13.0 SQLAlchemy==1.3.20 tinydb==4.2.0 +phonenumbers==8.12.12 +redis==3.5.3 +celery==4.4.7 +python-i18n[YAML]==0.3.9 +pyxdg==0.27 +bcrypt==3.2.0 +uWSGI==2.0.19.1 transitions==0.8.4 -uWSGI==2.0.19.1 \ No newline at end of file diff --git a/apps/contract-migration/config_template/celery.ini b/apps/contract-migration/config_template/celery.ini index dac9a5f4..e69de29b 100644 --- a/apps/contract-migration/config_template/celery.ini +++ b/apps/contract-migration/config_template/celery.ini @@ -1,3 +0,0 @@ -[celery] -broker_url = redis://redis:6379 -result_url = redis://redis:6379 diff --git a/apps/contract-migration/config_template/chain.ini b/apps/contract-migration/config_template/chain.ini deleted file mode 100644 index 6a2a8cba..00000000 --- a/apps/contract-migration/config_template/chain.ini +++ /dev/null @@ -1,2 +0,0 @@ -[chain] -spec = evm:ethereum:1 diff --git a/apps/contract-migration/config_template/cic.ini b/apps/contract-migration/config_template/cic.ini deleted file mode 100644 index bf598b98..00000000 --- a/apps/contract-migration/config_template/cic.ini +++ /dev/null @@ -1,7 +0,0 @@ -[cic] -registry_address = -token_index_address = -accounts_index_address = -declarator_address = -approval_escrow_address = -chain_spec = Bloxberg:8996 diff --git a/apps/contract-migration/config_template/database.ini b/apps/contract-migration/config_template/database.ini deleted file mode 100644 index db8227bc..00000000 --- a/apps/contract-migration/config_template/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -user = postgres -password = tralala -name = -host = postgres -port = 5432 -engine = postgres -driver = psycopg2 diff --git a/apps/contract-migration/config_template/dev.ini b/apps/contract-migration/config_template/dev.ini deleted file mode 100644 index a2c663ea..00000000 --- a/apps/contract-migration/config_template/dev.ini +++ /dev/null @@ -1,33 +0,0 @@ -[dev] -mnemonic = "history stumble mystery avoid embark arrive mom foil pledge keep grain dice" -eth_reserve_address = -eth_accounts_index_address = -# 10m * 10^18 (10^7 * 10^18) -eth_reserve_amount = 1000000000000000000000000 - -eth_account_contract_deployer=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -eth_account_gas_provider=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_gas_gifter=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_reserve_owner=0xc14958CD9A605AB0d9A36850362AaD2b9D42DF97 -eth_account_reserve_minter=0xe3C4db5947409Aff0FF8D643047EA41515cA4B8e -eth_account_accounts_index_owner=0x1AB26f3AAaaa884B651E213508592F3Ec6BfCe74 -eth_account_accounts_index_writer=0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA -eth_account_sarafu_owner=0x5567139c7a1C2977A391f51D8cA45B1D6700f5F6 -eth_account_sarafu_gifter=0x1149D55B3495CdC8FcfaF2816384AafaeFfaEa24 -eth_account_approval_escrow_owner=0x8754122F5718dC02Db5062c62fD4a63A5448623E -eth_account_single_shot_faucet_owner=0x34C85E47f45b0bea09F37c83e2fb02ECBC3a395d - -eth_sarafu_token_address = -eth_token_index_address = -eth_sarafu_faucet_address = -eth_erc20_approval_escrow_address = - -eth_sarafu_token_name=Sarafu -eth_sarafu_token_symbol=SRF -eth_sarafu_token_decimals=18 - -pgp_publickeys_active_file = publickeys.asc -pgp_publickeys_trusted_file = -pgp_publickeys_encrypt_file = - -faucet_amount = 1000000 diff --git a/apps/contract-migration/config_template/empty/config.ini b/apps/contract-migration/config_template/empty/config.ini new file mode 100644 index 00000000..e69de29b diff --git a/apps/contract-migration/config_template/eth.ini b/apps/contract-migration/config_template/eth.ini deleted file mode 100644 index 5506851b..00000000 --- a/apps/contract-migration/config_template/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://eth:8545 -abi_dir = /usr/local/share/cic/solidity/abi diff --git a/apps/contract-migration/config_template/legacy.ini b/apps/contract-migration/config_template/legacy.ini new file mode 100644 index 00000000..02eb709c --- /dev/null +++ b/apps/contract-migration/config_template/legacy.ini @@ -0,0 +1,7 @@ +[celery] +broker_url = +result_url = +debug = + +[cic] +chain_spec = diff --git a/apps/contract-migration/config_template/meta.ini b/apps/contract-migration/config_template/meta.ini deleted file mode 100644 index cbcc16ac..00000000 --- a/apps/contract-migration/config_template/meta.ini +++ /dev/null @@ -1,2 +0,0 @@ -[meta] -provider = http://cic-meta-server:80 diff --git a/apps/contract-migration/config_template/pgp.ini b/apps/contract-migration/config_template/pgp.ini deleted file mode 100644 index fade0d5a..00000000 --- a/apps/contract-migration/config_template/pgp.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pgp] -exports_dir = /tmp/cic/pgp -privatekey_file = ge.priv.asc -passphrase = ge diff --git a/apps/contract-migration/config_template/redis.ini b/apps/contract-migration/config_template/redis.ini deleted file mode 100644 index ea321378..00000000 --- a/apps/contract-migration/config_template/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = localhost -port = 63379 -db = 0 diff --git a/apps/contract-migration/config_template/rpc.ini b/apps/contract-migration/config_template/rpc.ini deleted file mode 100644 index 31dc0614..00000000 --- a/apps/contract-migration/config_template/rpc.ini +++ /dev/null @@ -1,5 +0,0 @@ -[rpc] -http_provider = http://localhost:8545 -http_authentication = -http_username = -http_password = diff --git a/apps/contract-migration/config_template/signer.ini b/apps/contract-migration/config_template/signer.ini deleted file mode 100644 index 58ad0c0a..00000000 --- a/apps/contract-migration/config_template/signer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[signer] -socket_path = /tmp/cic/signer/jsonrpc.ipc -secret = deadbeef diff --git a/apps/contract-migration/config_template/syncer.ini b/apps/contract-migration/config_template/syncer.ini deleted file mode 100644 index 60c0c007..00000000 --- a/apps/contract-migration/config_template/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[syncer] -loop_interval = diff --git a/apps/contract-migration/config_template/tasks.ini b/apps/contract-migration/config_template/tasks.ini deleted file mode 100644 index b49ecad5..00000000 --- a/apps/contract-migration/config_template/tasks.ini +++ /dev/null @@ -1,5 +0,0 @@ -[tasks] -africastalking = cic_notify.tasks.sms.africastalking -sms_db = cic_notify.tasks.sms.db -log = cic_notify.tasks.sms.log -transfer_callbacks=cic-ussd:cic_ussd.tasks.callback_handler.process_incoming_transfer_callback diff --git a/apps/contract-migration/config_template/token.ini b/apps/contract-migration/config_template/token.ini deleted file mode 100644 index 865e7f84..00000000 --- a/apps/contract-migration/config_template/token.ini +++ /dev/null @@ -1,8 +0,0 @@ -[token] -name = -symbol = -decimals = -demurrage_level = -redistribution_period = -sink_address = -supply_limit = 0 diff --git a/apps/contract-migration/config_template/wallet.ini b/apps/contract-migration/config_template/wallet.ini deleted file mode 100644 index 696cee43..00000000 --- a/apps/contract-migration/config_template/wallet.ini +++ /dev/null @@ -1,3 +0,0 @@ -[wallet] -key_file = -passphrase = diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index e90e6995..5f2fd37c 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -30,13 +30,14 @@ COPY requirements.txt . ARG pip_index_url=https://pypi.org/simple ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG EXTRA_PIP_ARGS="" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" ARG pip_trusted_host=pypi.org RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url https://pypi.org/simple \ --force-reinstall \ --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt COPY . . diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 9d5cb9b2..f71cb33b 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,10 +1,10 @@ -cic-eth[tools]==0.12.2a3 -eth-erc20>=0.0.11a1,<0.1.0 -erc20-demurrage-token>=0.0.2a5,<0.1.0 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<=0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a4,<0.1.0 -chainlib-eth>=0.0.6a1,<0.1.0 +cic-eth[tools]==0.12.2a4 +eth-erc20>=0.0.12a1,<0.1.0 +erc20-demurrage-token>=0.0.3a4,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a1,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a7,<0.1.0 +chainlib-eth>=0.0.7a8,<0.1.0 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index ed6912c6..990e1dae 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -17,7 +17,9 @@ token_redistribution_period: $TOKEN_REDISTRIBUTION_PERIOD token_supply_limit: $TOKEN_SUPPLY_LIMIT EOF -CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995} +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000} @@ -30,6 +32,7 @@ DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $DEV_ETH_KEYSTORE_FILE | j if [ ! -z $DEV_ETH_GAS_PRICE ]; then gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE" + fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE" >&2 echo using static gas price $DEV_ETH_GAS_PRICE fi @@ -96,11 +99,15 @@ if [[ -n "${ETH_PROVIDER}" ]]; then >&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m" fi fi - DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww` + export _CONFINI_DIR=$CONFINI_DIR + unset CONFINI_DIR + DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` + export CONFINI_DIR=$_CONFINI_DIR else >&2 echo unknown token type $TOKEN_TYPE exit 1 fi + echo "giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT" giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT >&2 echo "deploy account index contract" @@ -133,13 +140,22 @@ if [[ -n "${ETH_PROVIDER}" ]]; then # Sarafu faucet contract >&2 echo "deploy token faucet contract" - DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS` + export _CONFINI_DIR=$CONFINI_DIR + unset CONFINI_DIR + DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s` + + >&2 echo "set token faucet amount" + sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT + + export CONFINI_DIR=$_CONFINI_DIR + + >&2 echo "register faucet in registry" eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS + >&2 echo "set faucet as token minter" giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS - >&2 echo "set token faucet amount" - sarafu-faucet-set $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS -vv $DEV_FAUCET_AMOUNT + else diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index aba7ea6c..29a80220 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -16,15 +16,18 @@ ETH_PASSPHRASE='' CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + # Debug flag DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C keystore_file=./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c debug='-vv' -abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} gas_amount=100000000000000000000000 token_amount=${gas_amount} env_out_file=${CIC_DATA_DIR}/.env_seed init_level_file=${CIC_DATA_DIR}/.init +empty_config_dir=$CONFINI_DIR/empty truncate $env_out_file -s 0 set -e @@ -33,9 +36,9 @@ set -a #pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL eth-address-index==0.1.1a7 # get required addresses from registries -DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` -DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` -DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` +DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` +DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` +DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` cat <&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER -DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_GAS_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER +cic-eth-tag -i $CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER >&2 echo "create account for sarafu gifter" -DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_SARAFU_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER +cic-eth-tag -i $CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER >&2 echo "create account for approval escrow owner" -DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER +cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER #>&2 echo "create account for faucet owner" #DEV_ETH_ACCOUNT_FAUCET_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` @@ -65,32 +68,32 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER >&2 echo "create account for accounts index writer" -DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >&2 echo "add acccounts index writer account as writer on contract" -eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +eth-accounts-index-writer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER # Transfer gas to custodial gas provider adddress _CONFINI_DIR=$CONFINI_DIR unset CONFINI_DIR >&2 echo gift gas to gas gifter ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount >&2 echo gift gas to sarafu token owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount >&2 echo gift gas to account index owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount # Send token to token creator >&2 echo "gift tokens to sarafu owner" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount # Send token to token gifter >&2 echo "gift tokens to keystore address" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)" echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file @@ -98,7 +101,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS # Transfer tokens to gifter address >&2 echo "transfer tokens to token gifter address" ->&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} +>&2 erc20-transfer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} #echo -n 0 > $init_level_file diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 10a752fd..7317ddf8 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,13 +1,13 @@ -sarafu-faucet==0.0.4a3 -cic-eth[tools]==0.12.1a1 -cic-types==0.1.0a13 -crypto-dev-signer==0.4.14b7 +sarafu-faucet~=0.0.5a2 +cic-eth[tools]~=0.12.2a3 +cic-types~=0.1.0a14 +crypto-dev-signer~=0.4.14rc1 faker==4.17.1 -chainsyncer~=0.0.3a3 -chainlib-eth~=0.0.5a1 -eth-address-index~=0.1.2a1 -eth-contract-registry~=0.5.6a1 -eth-accounts-index~=0.0.12a1 -eth-erc20~=0.0.10a3 -erc20-faucet~=0.2.2a1 +chainsyncer~=0.0.6a1 +chainlib-eth~=0.0.7a1 +eth-address-index~=0.1.4a1 +eth-contract-registry~=0.5.8a1 +eth-accounts-index~=0.0.14a1 +eth-erc20~=0.0.12a1 +erc20-faucet~=0.2.4a1 psycopg2==2.8.6 diff --git a/docker-compose.yml b/docker-compose.yml index 3d5242b6..5fcfe098 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -210,15 +210,20 @@ services: args: pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} pip_extra_args: $PIP_EXTRA_ARGS + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest env_file: - .env environment: - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - # And these two are for wait-for-it (could parse this) + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} DEV_USE_DOCKER_WAIT_SCRIPT: 1 ETH_PROVIDER_HOST: eth ETH_PROVIDER_PORT: 8545 + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth} DATABASE_USER: ${DATABASE_USER:-postgres} @@ -279,9 +284,12 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -290,9 +298,9 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 CELERY_RESULT_URL: redis://redis:6379 deploy: @@ -312,6 +320,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} @@ -324,7 +334,6 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 DATABASE_POOL_SIZE: 0 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 @@ -346,6 +355,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -363,7 +374,7 @@ services: - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \ + --wsgi-file /root/cic_cache/runnable/daemons/server.py \ --http :8000 \ --pyargv "-vv" deploy: @@ -385,11 +396,13 @@ services: build: context: apps/cic-eth dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER - ETH_PROVIDER: http://eth:8545 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -403,11 +416,12 @@ services: REDIS_HOST: redis PGPASSWORD: ${DATABASE_PASSWORD:-tralala} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} @@ -431,8 +445,11 @@ services: build: context: apps/cic-eth dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: - ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -441,7 +458,8 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -457,7 +475,8 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tracker.sh -vv -c /usr/local/etc/cic-eth + #./start_tracker.sh -vv -c /usr/local/etc/cic-eth + ./start_tracker.sh -vv # command: "/root/start_manager.sh head -vv" @@ -466,8 +485,11 @@ services: build: context: apps/cic-eth dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -476,6 +498,7 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -502,8 +525,11 @@ services: build: context: apps/cic-eth dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -513,6 +539,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}