Merge branch 'lash/rehabilitate-traffic-2' into 'master'

Rehabilitate traffic generator script

See merge request grassrootseconomics/cic-internal-integration!145

apps/cic-cache/config/test/syncer.ini

@@ -0,0 +1,2 @@
+[syncer]
+loop_interval = 1

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.13',
+        'beta.14',
         )
 
 version_object = semver.VersionInfo(

apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py

@@ -0,0 +1,76 @@
+# external imports
+from chainlib.jsonrpc import JSONRPCException
+from eth_erc20 import ERC20
+from eth_accounts_index import AccountsIndex
+from eth_token_index import TokenUniqueSymbolIndex
+
+class ERC20Token:
+
+    def __init__(self, chain_spec, address, conn):
+        self.__address = address
+
+        c = ERC20(chain_spec)
+        o = c.symbol(address)
+        r = conn.do(o)
+        self.__symbol = c.parse_symbol(r)
+
+        o = c.decimals(address)
+        r = conn.do(o)
+        self.__decimals = c.parse_decimals(r)
+
+
+    def symbol(self):
+        return self.__symbol
+
+
+    def decimals(self):
+        return self.__decimals
+
+
+class IndexCache:
+
+    def __init__(self, chain_spec, address):
+        self.address = address
+        self.chain_spec = chain_spec
+
+    
+    def parse(self, r):
+        return r
+
+
+    def get(self, conn):
+        entries = []
+        i = 0
+        while True:
+            o = self.o.entry(self.address, i)
+            try:
+                r = conn.do(o)
+                entries.append(self.parse(r, conn))
+            except JSONRPCException:
+                return entries
+            i += 1
+
+
+class AccountRegistryCache(IndexCache):
+    
+    def __init__(self, chain_spec, address):
+        super(AccountRegistryCache, self).__init__(chain_spec, address)
+        self.o = AccountsIndex(chain_spec)
+        self.get_accounts = self.get
+
+
+    def parse(self, r, conn):
+        return self.o.parse_account(r)
+
+    
+class TokenRegistryCache(IndexCache):
+
+    def __init__(self, chain_spec, address):
+        super(TokenRegistryCache, self).__init__(chain_spec, address)
+        self.o = TokenUniqueSymbolIndex(chain_spec)
+        self.get_tokens = self.get
+
+
+    def parse(self, r, conn):
+        token_address = self.o.parse_entry(r)
+        return ERC20Token(self.chain_spec, token_address, conn)

apps/data-seeding/cic_eth/traffic/cmd/traffic.py

@@ -163,9 +163,9 @@ class TrafficProvisioner:
     """Aux parameter template to be passed to the traffic generator module"""
 
 
-    def __init__(self):
+    def __init__(self, conn):
-        self.tokens = self.oracles['token'].get_tokens()
+        self.tokens = self.oracles['token'].get_tokens(conn)
-        self.accounts = self.oracles['account'].get_accounts()
+        self.accounts = self.oracles['account'].get_accounts(conn)
         self.aux = copy.copy(self.default_aux)
         self.__balances = {}
         for a in self.accounts:
@@ -277,13 +277,14 @@ class TrafficSyncHandler:
     :type traffic_router: TrafficRouter
     :raises Exception: Any Exception redis may raise on connection attempt.
     """
-    def __init__(self, config, traffic_router):
+    def __init__(self, config, traffic_router, conn):
         self.traffic_router = traffic_router
         self.redis_channel = str(uuid.uuid4())
         self.pubsub = self.__connect_redis(self.redis_channel, config)
         self.traffic_items = {}
         self.config = config
         self.init = False
+        self.conn = conn
 
 
     # connects to redis
@@ -307,7 +308,7 @@ class TrafficSyncHandler:
         :param tx_index: Syncer block transaction index at time of call.
         :type tx_index: number
         """
-        traffic_provisioner = TrafficProvisioner()
+        traffic_provisioner = TrafficProvisioner(self.conn)
         traffic_provisioner.add_aux('redis_channel', self.redis_channel)
 
         refresh_accounts = None
@@ -343,7 +344,7 @@ class TrafficSyncHandler:
             sender = traffic_provisioner.accounts[sender_index]
             #balance_full = balances[sender][token_pair[0].symbol()]
             if len(sender_indices) == 1:
-                sender_indices[m] = sender_sender_indices[len(senders)-1]
+                sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
             sender_indices = sender_indices[:len(sender_indices)-1]
 
             balance_full = traffic_provisioner.balance(sender, token_pair[0])
@@ -351,7 +352,14 @@ class TrafficSyncHandler:
             recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
             recipient = traffic_provisioner.accounts[recipient_index]
           
-            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
+            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
+                traffic_item,
+                token_pair,
+                sender,
+                recipient,
+                balance_full,
+                )
+                )
             (e, t, balance_result,) = traffic_item.method(
                     token_pair,
                     sender,
@@ -359,7 +367,6 @@ class TrafficSyncHandler:
                     balance_full,
                     traffic_provisioner.aux,
                     block_number,
-                    tx_index,
                     )
             traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
             sender_indices.append(recipient_index)

apps/data-seeding/cic_eth/traffic/common/registry.py

@@ -3,7 +3,7 @@ import logging
 import copy
 
 # external imports
-from cic_registry import CICRegistry
+from cic_registry.registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
 from eth_accounts_index import AccountRegistry
 from chainlib.chain import ChainSpec

apps/data-seeding/cic_eth/traffic/common/signer.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports 
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore import DictKeystore
+from crypto_dev_signer.keystore.dict import DictKeystore
 
 logg = logging.getLogger(__name__)
 

apps/data-seeding/cic_eth/traffic/local/account.py

@@ -11,7 +11,7 @@ queue = 'cic-eth'
 name = 'account'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Triggers creation and registration of new account through the custodial cic-eth component.
 
     It expects the following aux parameters to exist:

apps/data-seeding/cic_eth/traffic/local/noop.py

@@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
 
     If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
     :type aux: dict
     :param block_number: Syncer block number position at time of method call
     :type block_number: number
-    :param tx_index: Syncer block transaction index position at time of method call
-    :type tx_index: number
     :raises KeyError: Missing required aux element
     :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
     :rtype: tuple
     """
-    logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
+    logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
 
     return (None, None, sender_balance, )

apps/data-seeding/cic_eth/traffic/local/transfer.py

@@ -12,7 +12,7 @@ queue = 'cic-eth'
 name = 'erc20_transfer'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
 
     It expects the following aux parameters to exist:
@@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
     balance_units = int(sender_balance_value / decimals)
 
     if balance_units <= 0:
-        return (AttributeError('sender {} has zero balance'), None, 0,)
+        return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
 
     spend_units = random.randint(1, balance_units)
     spend_value = spend_units * decimals

apps/data-seeding/cic_eth/traffic/traffic.py

@@ -8,16 +8,25 @@ import json
 # external imports
 import redis
 import celery
-from chainsyncer.backend import MemBackend
+from cic_eth_registry.registry import CICRegistry
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
-from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.connection import EthHTTPConnection
-from chainlib.eth.gas import DefaultGasOracle
+from chainlib.chain import ChainSpec
-from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.gas import RPCGasOracle
+from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.block import block_latest
 from hexathon import strip_0x
+from cic_base import (
+        argparse,
+        config,
+        log,
+        rpc,
+        signer as signer_funcs,
+        )
 
 # local imports
-import common
+#import common
 from cmd.traffic import (
         TrafficItem,
         TrafficRouter,
@@ -25,15 +34,19 @@ from cmd.traffic import (
         TrafficSyncHandler,
         )
 from cmd.traffic import add_args as add_traffic_args
+from cmd.cache import (
+        AccountRegistryCache,
+        TokenRegistryCache,
+        )
 
 
 # common basics
 script_dir = os.path.realpath(os.path.dirname(__file__))
-logg = common.log.create()
+logg = log.create()
-argparser = common.argparse.create(script_dir, common.argparse.full_template)
+argparser = argparse.create(script_dir, argparse.full_template)
-argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
+argparser = argparse.add(argparser, add_traffic_args, 'traffic')
-args = common.argparse.parse(argparser, logg)
+args = argparse.parse(argparser, logg)
-config = common.config.create(args.c, args, args.env_prefix)
+config = config.create(args.c, args, args.env_prefix)
 
 # map custom args to local config entries
 batchsize = args.batch_size
@@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True)
 
 config.add(args.q, '_CELERY_QUEUE', True)
 
-common.config.log(config)
+logg.debug(config)
 
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 def main():
     # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
+    (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
 
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # set up registry
-    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
-    registry = common.registry.init_legacy(config, w3)
+    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    #registry = registry.init_legacy(config, w3)
+    CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
+    registry = CICRegistry(chain_spec, conn)
 
     # Connect to blockchain with chainlib
-    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = RPCGasOracle(conn)
-    gas_oracle = DefaultGasOracle(conn)
+    nonce_oracle = RPCNonceOracle(signer_address, conn)
-    nonce_oracle = DefaultNonceOracle(signer_address, conn)
 
     # Set up magic traffic handler
     traffic_router = TrafficRouter()
     traffic_router.apply_import_dict(config.all(), config)
-    handler = TrafficSyncHandler(config, traffic_router)
+    handler = TrafficSyncHandler(config, traffic_router, conn)
 
     # Set up syncer
     syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
@@ -80,9 +96,21 @@ def main():
     block_offset = int(strip_0x(r), 16) + 1
     syncer_backend.set(block_offset, 0)
 
+    # get relevant registry entries
+    token_registry = registry.lookup('TokenRegistry')
+    logg.info('using token registry {}'.format(token_registry))
+    token_cache = TokenRegistryCache(chain_spec, token_registry)
+
+    account_registry = registry.lookup('AccountRegistry')
+    logg.info('using account registry {}'.format(account_registry))
+    account_cache = AccountRegistryCache(chain_spec, account_registry)
+   
     # Set up provisioner for common task input data
-    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.oracles['token'] = token_cache
+    TrafficProvisioner.oracles['account'] = account_cache
+    
     TrafficProvisioner.default_aux = {
             'chain_spec': config.get('CIC_CHAIN_SPEC'),
             'registry': registry,
@@ -92,7 +120,7 @@ def main():
             'api_queue': config.get('_CELERY_QUEUE'),
             }
 
-    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
+    syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
 

apps/data-seeding/config/traffic.ini

@@ -1,4 +1,4 @@
 [traffic]
 #local.noop_traffic = 2
-local.account = 2
+#local.account = 2
 local.transfer = 2

apps/data-seeding/verify.py

@@ -72,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@@ -101,6 +102,8 @@ config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
 
+token_symbol = args.token_symbol
+
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
@@ -273,7 +276,10 @@ class Verifier:
     def verify_balance(self, address, balance):
         o = self.erc20_tx_factory.balance(self.token_address, address)
         r = self.conn.do(o)
+        try:
             actual_balance = int(strip_0x(r), 16)
+        except ValueError:
+            actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
@@ -461,7 +467,7 @@ def main():
     tx = txf.template(ZERO_ADDRESS, token_index_address)
     data = add_0x(registry_addressof_method)
     h = hashlib.new('sha256')
-    h.update(b'SRF')
+    h.update(token_symbol.encode('utf-8'))
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)