From c569fe4b17dbd67f39b4c1ece51b432b7b8e2f46 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Thu, 20 May 2021 21:25:14 +0000 Subject: [PATCH] Rehabilitate traffic generator script --- apps/cic-cache/config/test/syncer.ini | 2 + apps/cic-eth/cic_eth/version.py | 2 +- .../scripts/cic_eth/traffic/cmd/cache.py | 76 +++++++++++++++++++ .../scripts/cic_eth/traffic/cmd/traffic.py | 23 ++++-- .../cic_eth/traffic/common/registry.py | 2 +- .../scripts/cic_eth/traffic/common/signer.py | 2 +- .../scripts/cic_eth/traffic/local/account.py | 2 +- .../scripts/cic_eth/traffic/local/noop.py | 6 +- .../scripts/cic_eth/traffic/local/transfer.py | 4 +- .../scripts/cic_eth/traffic/traffic.py | 70 ++++++++++++----- .../scripts/config/traffic.ini | 2 +- .../scripts/requirements.txt | 4 +- apps/contract-migration/scripts/verify.py | 10 ++- 13 files changed, 161 insertions(+), 44 deletions(-) create mode 100644 apps/cic-cache/config/test/syncer.ini create mode 100644 apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py diff --git a/apps/cic-cache/config/test/syncer.ini b/apps/cic-cache/config/test/syncer.ini new file mode 100644 index 00000000..452fb81c --- /dev/null +++ b/apps/cic-cache/config/test/syncer.ini @@ -0,0 +1,2 @@ +[syncer] +loop_interval = 1 diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 1382b9cf..18ac1dd1 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 11, 0, - 'beta.13', + 'beta.14', ) version_object = semver.VersionInfo( diff --git a/apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py b/apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py new file mode 100644 index 00000000..e112ae10 --- /dev/null +++ b/apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py @@ -0,0 +1,76 @@ +# external imports +from chainlib.jsonrpc import JSONRPCException +from eth_erc20 import ERC20 +from eth_accounts_index import AccountsIndex +from eth_token_index import TokenUniqueSymbolIndex + +class ERC20Token: + + def __init__(self, chain_spec, address, conn): + self.__address = address + + c = ERC20(chain_spec) + o = c.symbol(address) + r = conn.do(o) + self.__symbol = c.parse_symbol(r) + + o = c.decimals(address) + r = conn.do(o) + self.__decimals = c.parse_decimals(r) + + + def symbol(self): + return self.__symbol + + + def decimals(self): + return self.__decimals + + +class IndexCache: + + def __init__(self, chain_spec, address): + self.address = address + self.chain_spec = chain_spec + + + def parse(self, r): + return r + + + def get(self, conn): + entries = [] + i = 0 + while True: + o = self.o.entry(self.address, i) + try: + r = conn.do(o) + entries.append(self.parse(r, conn)) + except JSONRPCException: + return entries + i += 1 + + +class AccountRegistryCache(IndexCache): + + def __init__(self, chain_spec, address): + super(AccountRegistryCache, self).__init__(chain_spec, address) + self.o = AccountsIndex(chain_spec) + self.get_accounts = self.get + + + def parse(self, r, conn): + return self.o.parse_account(r) + + +class TokenRegistryCache(IndexCache): + + def __init__(self, chain_spec, address): + super(TokenRegistryCache, self).__init__(chain_spec, address) + self.o = TokenUniqueSymbolIndex(chain_spec) + self.get_tokens = self.get + + + def parse(self, r, conn): + token_address = self.o.parse_entry(r) + return ERC20Token(self.chain_spec, token_address, conn) diff --git a/apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py b/apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py index 1e5f58c4..11893240 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py @@ -163,9 +163,9 @@ class TrafficProvisioner: """Aux parameter template to be passed to the traffic generator module""" - def __init__(self): - self.tokens = self.oracles['token'].get_tokens() - self.accounts = self.oracles['account'].get_accounts() + def __init__(self, conn): + self.tokens = self.oracles['token'].get_tokens(conn) + self.accounts = self.oracles['account'].get_accounts(conn) self.aux = copy.copy(self.default_aux) self.__balances = {} for a in self.accounts: @@ -277,13 +277,14 @@ class TrafficSyncHandler: :type traffic_router: TrafficRouter :raises Exception: Any Exception redis may raise on connection attempt. """ - def __init__(self, config, traffic_router): + def __init__(self, config, traffic_router, conn): self.traffic_router = traffic_router self.redis_channel = str(uuid.uuid4()) self.pubsub = self.__connect_redis(self.redis_channel, config) self.traffic_items = {} self.config = config self.init = False + self.conn = conn # connects to redis @@ -307,7 +308,7 @@ class TrafficSyncHandler: :param tx_index: Syncer block transaction index at time of call. :type tx_index: number """ - traffic_provisioner = TrafficProvisioner() + traffic_provisioner = TrafficProvisioner(self.conn) traffic_provisioner.add_aux('redis_channel', self.redis_channel) refresh_accounts = None @@ -343,7 +344,7 @@ class TrafficSyncHandler: sender = traffic_provisioner.accounts[sender_index] #balance_full = balances[sender][token_pair[0].symbol()] if len(sender_indices) == 1: - sender_indices[m] = sender_sender_indices[len(senders)-1] + sender_indices[sender_index] = sender_indices[len(sender_indices)-1] sender_indices = sender_indices[:len(sender_indices)-1] balance_full = traffic_provisioner.balance(sender, token_pair[0]) @@ -351,7 +352,14 @@ class TrafficSyncHandler: recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1) recipient = traffic_provisioner.accounts[recipient_index] - logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}') + logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format( + traffic_item, + token_pair, + sender, + recipient, + balance_full, + ) + ) (e, t, balance_result,) = traffic_item.method( token_pair, sender, @@ -359,7 +367,6 @@ class TrafficSyncHandler: balance_full, traffic_provisioner.aux, block_number, - tx_index, ) traffic_provisioner.update_balance(sender, token_pair[0], balance_result) sender_indices.append(recipient_index) diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/registry.py b/apps/contract-migration/scripts/cic_eth/traffic/common/registry.py index 83bbf029..99145538 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/common/registry.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/common/registry.py @@ -3,7 +3,7 @@ import logging import copy # external imports -from cic_registry import CICRegistry +from cic_registry.registry import Registry from eth_token_index import TokenUniqueSymbolIndex from eth_accounts_index import AccountRegistry from chainlib.chain import ChainSpec diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/signer.py b/apps/contract-migration/scripts/cic_eth/traffic/common/signer.py index 34935137..fa47b79b 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/common/signer.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/common/signer.py @@ -3,7 +3,7 @@ import logging # external imports from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore import DictKeystore +from crypto_dev_signer.keystore.dict import DictKeystore logg = logging.getLogger(__name__) diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/account.py b/apps/contract-migration/scripts/cic_eth/traffic/local/account.py index 35666a0a..830da009 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/local/account.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/local/account.py @@ -11,7 +11,7 @@ queue = 'cic-eth' name = 'account' -def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): +def do(token_pair, sender, recipient, sender_balance, aux, block_number): """Triggers creation and registration of new account through the custodial cic-eth component. It expects the following aux parameters to exist: diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/noop.py b/apps/contract-migration/scripts/cic_eth/traffic/local/noop.py index 85f7cf23..d9c8eda4 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/local/noop.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/local/noop.py @@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): +def do(token_pair, sender, recipient, sender_balance, aux, block_number): """Defines the function signature for a traffic generator. The method itself only logs the input parameters. If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight. @@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde :type aux: dict :param block_number: Syncer block number position at time of method call :type block_number: number - :param tx_index: Syncer block transaction index position at time of method call - :type tx_index: number :raises KeyError: Missing required aux element :returns: Exception|None, task_id|None and adjusted_sender_balance respectively :rtype: tuple """ - logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index)) + logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number)) return (None, None, sender_balance, ) diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py b/apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py index e6a6e74d..7eea76f5 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py @@ -12,7 +12,7 @@ queue = 'cic-eth' name = 'erc20_transfer' -def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): +def do(token_pair, sender, recipient, sender_balance, aux, block_number): """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution. It expects the following aux parameters to exist: @@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde balance_units = int(sender_balance_value / decimals) if balance_units <= 0: - return (AttributeError('sender {} has zero balance'), None, 0,) + return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,) spend_units = random.randint(1, balance_units) spend_value = spend_units * decimals diff --git a/apps/contract-migration/scripts/cic_eth/traffic/traffic.py b/apps/contract-migration/scripts/cic_eth/traffic/traffic.py index 8dc4ba35..f0cbd7bb 100644 --- a/apps/contract-migration/scripts/cic_eth/traffic/traffic.py +++ b/apps/contract-migration/scripts/cic_eth/traffic/traffic.py @@ -8,16 +8,25 @@ import json # external imports import redis import celery -from chainsyncer.backend import MemBackend +from cic_eth_registry.registry import CICRegistry +from chainsyncer.backend.memory import MemBackend from chainsyncer.driver import HeadSyncer -from chainlib.eth.connection import HTTPConnection -from chainlib.eth.gas import DefaultGasOracle -from chainlib.eth.nonce import DefaultNonceOracle +from chainlib.eth.connection import EthHTTPConnection +from chainlib.chain import ChainSpec +from chainlib.eth.gas import RPCGasOracle +from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.block import block_latest from hexathon import strip_0x +from cic_base import ( + argparse, + config, + log, + rpc, + signer as signer_funcs, + ) # local imports -import common +#import common from cmd.traffic import ( TrafficItem, TrafficRouter, @@ -25,15 +34,19 @@ from cmd.traffic import ( TrafficSyncHandler, ) from cmd.traffic import add_args as add_traffic_args +from cmd.cache import ( + AccountRegistryCache, + TokenRegistryCache, + ) # common basics script_dir = os.path.realpath(os.path.dirname(__file__)) -logg = common.log.create() -argparser = common.argparse.create(script_dir, common.argparse.full_template) -argparser = common.argparse.add(argparser, add_traffic_args, 'traffic') -args = common.argparse.parse(argparser, logg) -config = common.config.create(args.c, args, args.env_prefix) +logg = log.create() +argparser = argparse.create(script_dir, argparse.full_template) +argparser = argparse.add(argparser, add_traffic_args, 'traffic') +args = argparse.parse(argparser, logg) +config = config.create(args.c, args, args.env_prefix) # map custom args to local config entries batchsize = args.batch_size @@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True) config.add(args.q, '_CELERY_QUEUE', True) -common.config.log(config) +logg.debug(config) +chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) def main(): # create signer (not currently in use, but needs to be accessible for custom traffic item generators) - (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE')) + (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE')) # connect to celery celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) # set up registry - w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored - registry = common.registry.init_legacy(config, w3) + rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored + conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + #registry = registry.init_legacy(config, w3) + CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS') + registry = CICRegistry(chain_spec, conn) # Connect to blockchain with chainlib - conn = HTTPConnection(config.get('ETH_PROVIDER')) - gas_oracle = DefaultGasOracle(conn) - nonce_oracle = DefaultNonceOracle(signer_address, conn) + gas_oracle = RPCGasOracle(conn) + nonce_oracle = RPCNonceOracle(signer_address, conn) # Set up magic traffic handler traffic_router = TrafficRouter() traffic_router.apply_import_dict(config.all(), config) - handler = TrafficSyncHandler(config, traffic_router) + handler = TrafficSyncHandler(config, traffic_router, conn) # Set up syncer syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0) @@ -80,9 +96,21 @@ def main(): block_offset = int(strip_0x(r), 16) + 1 syncer_backend.set(block_offset, 0) + # get relevant registry entries + token_registry = registry.lookup('TokenRegistry') + logg.info('using token registry {}'.format(token_registry)) + token_cache = TokenRegistryCache(chain_spec, token_registry) + + account_registry = registry.lookup('AccountRegistry') + logg.info('using account registry {}'.format(account_registry)) + account_cache = AccountRegistryCache(chain_spec, account_registry) + # Set up provisioner for common task input data - TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) - TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) + #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) + #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) + TrafficProvisioner.oracles['token'] = token_cache + TrafficProvisioner.oracles['account'] = account_cache + TrafficProvisioner.default_aux = { 'chain_spec': config.get('CIC_CHAIN_SPEC'), 'registry': registry, @@ -92,7 +120,7 @@ def main(): 'api_queue': config.get('_CELERY_QUEUE'), } - syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh) + syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/contract-migration/scripts/config/traffic.ini b/apps/contract-migration/scripts/config/traffic.ini index 99b572c8..df89d1aa 100644 --- a/apps/contract-migration/scripts/config/traffic.ini +++ b/apps/contract-migration/scripts/config/traffic.ini @@ -1,4 +1,4 @@ [traffic] #local.noop_traffic = 2 -local.account = 2 +#local.account = 2 local.transfer = 2 diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index f7487c3b..fcc526d4 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2b9 +cic-base[full_graph]==0.1.2b11 sarafu-faucet==0.0.3a3 -cic-eth==0.11.0b13 +cic-eth==0.11.0b14 cic-types==0.1.0a11 crypto-dev-signer==0.4.14b3 diff --git a/apps/contract-migration/scripts/verify.py b/apps/contract-migration/scripts/verify.py index 6a526e02..2e265264 100644 --- a/apps/contract-migration/scripts/verify.py +++ b/apps/contract-migration/scripts/verify.py @@ -72,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification') +argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error') @@ -101,6 +102,8 @@ config.censor('PASSWORD', 'SSL') config.add(args.meta_provider, '_META_PROVIDER', True) config.add(args.ussd_provider, '_USSD_PROVIDER', True) +token_symbol = args.token_symbol + logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) @@ -273,7 +276,10 @@ class Verifier: def verify_balance(self, address, balance): o = self.erc20_tx_factory.balance(self.token_address, address) r = self.conn.do(o) - actual_balance = int(strip_0x(r), 16) + try: + actual_balance = int(strip_0x(r), 16) + except ValueError: + actual_balance = int(r) balance = int(balance / 1000000) * 1000000 logg.debug('balance for {}: {}'.format(address, balance)) if balance != actual_balance: @@ -461,7 +467,7 @@ def main(): tx = txf.template(ZERO_ADDRESS, token_index_address) data = add_0x(registry_addressof_method) h = hashlib.new('sha256') - h.update(b'SRF') + h.update(token_symbol.encode('utf-8')) z = h.digest() data += eth_abi.encode_single('bytes32', z).hex() txf.set_code(tx, data)