Merge branch 'lash/rehabilitate-traffic-2' into 'master'

Rehabilitate traffic generator script

See merge request grassrootseconomics/cic-internal-integration!145
This commit is contained in:
Louis Holbrook 2021-05-20 21:25:15 +00:00
commit d7113f3923
12 changed files with 159 additions and 42 deletions

View File

@ -0,0 +1,2 @@
[syncer]
loop_interval = 1

View File

@ -10,7 +10,7 @@ version = (
0, 0,
11, 11,
0, 0,
'beta.13', 'beta.14',
) )
version_object = semver.VersionInfo( version_object = semver.VersionInfo(

View File

@ -0,0 +1,76 @@
# external imports
from chainlib.jsonrpc import JSONRPCException
from eth_erc20 import ERC20
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
class ERC20Token:
def __init__(self, chain_spec, address, conn):
self.__address = address
c = ERC20(chain_spec)
o = c.symbol(address)
r = conn.do(o)
self.__symbol = c.parse_symbol(r)
o = c.decimals(address)
r = conn.do(o)
self.__decimals = c.parse_decimals(r)
def symbol(self):
return self.__symbol
def decimals(self):
return self.__decimals
class IndexCache:
def __init__(self, chain_spec, address):
self.address = address
self.chain_spec = chain_spec
def parse(self, r):
return r
def get(self, conn):
entries = []
i = 0
while True:
o = self.o.entry(self.address, i)
try:
r = conn.do(o)
entries.append(self.parse(r, conn))
except JSONRPCException:
return entries
i += 1
class AccountRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(AccountRegistryCache, self).__init__(chain_spec, address)
self.o = AccountsIndex(chain_spec)
self.get_accounts = self.get
def parse(self, r, conn):
return self.o.parse_account(r)
class TokenRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(TokenRegistryCache, self).__init__(chain_spec, address)
self.o = TokenUniqueSymbolIndex(chain_spec)
self.get_tokens = self.get
def parse(self, r, conn):
token_address = self.o.parse_entry(r)
return ERC20Token(self.chain_spec, token_address, conn)

View File

@ -163,9 +163,9 @@ class TrafficProvisioner:
"""Aux parameter template to be passed to the traffic generator module""" """Aux parameter template to be passed to the traffic generator module"""
def __init__(self): def __init__(self, conn):
self.tokens = self.oracles['token'].get_tokens() self.tokens = self.oracles['token'].get_tokens(conn)
self.accounts = self.oracles['account'].get_accounts() self.accounts = self.oracles['account'].get_accounts(conn)
self.aux = copy.copy(self.default_aux) self.aux = copy.copy(self.default_aux)
self.__balances = {} self.__balances = {}
for a in self.accounts: for a in self.accounts:
@ -277,13 +277,14 @@ class TrafficSyncHandler:
:type traffic_router: TrafficRouter :type traffic_router: TrafficRouter
:raises Exception: Any Exception redis may raise on connection attempt. :raises Exception: Any Exception redis may raise on connection attempt.
""" """
def __init__(self, config, traffic_router): def __init__(self, config, traffic_router, conn):
self.traffic_router = traffic_router self.traffic_router = traffic_router
self.redis_channel = str(uuid.uuid4()) self.redis_channel = str(uuid.uuid4())
self.pubsub = self.__connect_redis(self.redis_channel, config) self.pubsub = self.__connect_redis(self.redis_channel, config)
self.traffic_items = {} self.traffic_items = {}
self.config = config self.config = config
self.init = False self.init = False
self.conn = conn
# connects to redis # connects to redis
@ -307,7 +308,7 @@ class TrafficSyncHandler:
:param tx_index: Syncer block transaction index at time of call. :param tx_index: Syncer block transaction index at time of call.
:type tx_index: number :type tx_index: number
""" """
traffic_provisioner = TrafficProvisioner() traffic_provisioner = TrafficProvisioner(self.conn)
traffic_provisioner.add_aux('redis_channel', self.redis_channel) traffic_provisioner.add_aux('redis_channel', self.redis_channel)
refresh_accounts = None refresh_accounts = None
@ -343,7 +344,7 @@ class TrafficSyncHandler:
sender = traffic_provisioner.accounts[sender_index] sender = traffic_provisioner.accounts[sender_index]
#balance_full = balances[sender][token_pair[0].symbol()] #balance_full = balances[sender][token_pair[0].symbol()]
if len(sender_indices) == 1: if len(sender_indices) == 1:
sender_indices[m] = sender_sender_indices[len(senders)-1] sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
sender_indices = sender_indices[:len(sender_indices)-1] sender_indices = sender_indices[:len(sender_indices)-1]
balance_full = traffic_provisioner.balance(sender, token_pair[0]) balance_full = traffic_provisioner.balance(sender, token_pair[0])
@ -351,7 +352,14 @@ class TrafficSyncHandler:
recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1) recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
recipient = traffic_provisioner.accounts[recipient_index] recipient = traffic_provisioner.accounts[recipient_index]
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}') logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
traffic_item,
token_pair,
sender,
recipient,
balance_full,
)
)
(e, t, balance_result,) = traffic_item.method( (e, t, balance_result,) = traffic_item.method(
token_pair, token_pair,
sender, sender,
@ -359,7 +367,6 @@ class TrafficSyncHandler:
balance_full, balance_full,
traffic_provisioner.aux, traffic_provisioner.aux,
block_number, block_number,
tx_index,
) )
traffic_provisioner.update_balance(sender, token_pair[0], balance_result) traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
sender_indices.append(recipient_index) sender_indices.append(recipient_index)

View File

@ -3,7 +3,7 @@ import logging
import copy import copy
# external imports # external imports
from cic_registry import CICRegistry from cic_registry.registry import Registry
from eth_token_index import TokenUniqueSymbolIndex from eth_token_index import TokenUniqueSymbolIndex
from eth_accounts_index import AccountRegistry from eth_accounts_index import AccountRegistry
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec

View File

@ -3,7 +3,7 @@ import logging
# external imports # external imports
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore import DictKeystore from crypto_dev_signer.keystore.dict import DictKeystore
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)

View File

@ -11,7 +11,7 @@ queue = 'cic-eth'
name = 'account' name = 'account'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers creation and registration of new account through the custodial cic-eth component. """Triggers creation and registration of new account through the custodial cic-eth component.
It expects the following aux parameters to exist: It expects the following aux parameters to exist:

View File

@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Defines the function signature for a traffic generator. The method itself only logs the input parameters. """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight. If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
:type aux: dict :type aux: dict
:param block_number: Syncer block number position at time of method call :param block_number: Syncer block number position at time of method call
:type block_number: number :type block_number: number
:param tx_index: Syncer block transaction index position at time of method call
:type tx_index: number
:raises KeyError: Missing required aux element :raises KeyError: Missing required aux element
:returns: Exception|None, task_id|None and adjusted_sender_balance respectively :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
:rtype: tuple :rtype: tuple
""" """
logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index)) logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
return (None, None, sender_balance, ) return (None, None, sender_balance, )

View File

@ -12,7 +12,7 @@ queue = 'cic-eth'
name = 'erc20_transfer' name = 'erc20_transfer'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index): def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution. """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
It expects the following aux parameters to exist: It expects the following aux parameters to exist:
@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
balance_units = int(sender_balance_value / decimals) balance_units = int(sender_balance_value / decimals)
if balance_units <= 0: if balance_units <= 0:
return (AttributeError('sender {} has zero balance'), None, 0,) return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
spend_units = random.randint(1, balance_units) spend_units = random.randint(1, balance_units)
spend_value = spend_units * decimals spend_value = spend_units * decimals

View File

@ -8,16 +8,25 @@ import json
# external imports # external imports
import redis import redis
import celery import celery
from chainsyncer.backend import MemBackend from cic_eth_registry.registry import CICRegistry
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver import HeadSyncer from chainsyncer.driver import HeadSyncer
from chainlib.eth.connection import HTTPConnection from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.gas import DefaultGasOracle from chainlib.chain import ChainSpec
from chainlib.eth.nonce import DefaultNonceOracle from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.block import block_latest from chainlib.eth.block import block_latest
from hexathon import strip_0x from hexathon import strip_0x
from cic_base import (
argparse,
config,
log,
rpc,
signer as signer_funcs,
)
# local imports # local imports
import common #import common
from cmd.traffic import ( from cmd.traffic import (
TrafficItem, TrafficItem,
TrafficRouter, TrafficRouter,
@ -25,15 +34,19 @@ from cmd.traffic import (
TrafficSyncHandler, TrafficSyncHandler,
) )
from cmd.traffic import add_args as add_traffic_args from cmd.traffic import add_args as add_traffic_args
from cmd.cache import (
AccountRegistryCache,
TokenRegistryCache,
)
# common basics # common basics
script_dir = os.path.realpath(os.path.dirname(__file__)) script_dir = os.path.realpath(os.path.dirname(__file__))
logg = common.log.create() logg = log.create()
argparser = common.argparse.create(script_dir, common.argparse.full_template) argparser = argparse.create(script_dir, argparse.full_template)
argparser = common.argparse.add(argparser, add_traffic_args, 'traffic') argparser = argparse.add(argparser, add_traffic_args, 'traffic')
args = common.argparse.parse(argparser, logg) args = argparse.parse(argparser, logg)
config = common.config.create(args.c, args, args.env_prefix) config = config.create(args.c, args, args.env_prefix)
# map custom args to local config entries # map custom args to local config entries
batchsize = args.batch_size batchsize = args.batch_size
@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True)
config.add(args.q, '_CELERY_QUEUE', True) config.add(args.q, '_CELERY_QUEUE', True)
common.config.log(config) logg.debug(config)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
def main(): def main():
# create signer (not currently in use, but needs to be accessible for custom traffic item generators) # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
(signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE')) (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
# connect to celery # connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
# set up registry # set up registry
w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
registry = common.registry.init_legacy(config, w3) conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
#registry = registry.init_legacy(config, w3)
CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
registry = CICRegistry(chain_spec, conn)
# Connect to blockchain with chainlib # Connect to blockchain with chainlib
conn = HTTPConnection(config.get('ETH_PROVIDER')) gas_oracle = RPCGasOracle(conn)
gas_oracle = DefaultGasOracle(conn) nonce_oracle = RPCNonceOracle(signer_address, conn)
nonce_oracle = DefaultNonceOracle(signer_address, conn)
# Set up magic traffic handler # Set up magic traffic handler
traffic_router = TrafficRouter() traffic_router = TrafficRouter()
traffic_router.apply_import_dict(config.all(), config) traffic_router.apply_import_dict(config.all(), config)
handler = TrafficSyncHandler(config, traffic_router) handler = TrafficSyncHandler(config, traffic_router, conn)
# Set up syncer # Set up syncer
syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0) syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
@ -80,9 +96,21 @@ def main():
block_offset = int(strip_0x(r), 16) + 1 block_offset = int(strip_0x(r), 16) + 1
syncer_backend.set(block_offset, 0) syncer_backend.set(block_offset, 0)
# get relevant registry entries
token_registry = registry.lookup('TokenRegistry')
logg.info('using token registry {}'.format(token_registry))
token_cache = TokenRegistryCache(chain_spec, token_registry)
account_registry = registry.lookup('AccountRegistry')
logg.info('using account registry {}'.format(account_registry))
account_cache = AccountRegistryCache(chain_spec, account_registry)
# Set up provisioner for common task input data # Set up provisioner for common task input data
TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry) #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['token'] = token_cache
TrafficProvisioner.oracles['account'] = account_cache
TrafficProvisioner.default_aux = { TrafficProvisioner.default_aux = {
'chain_spec': config.get('CIC_CHAIN_SPEC'), 'chain_spec': config.get('CIC_CHAIN_SPEC'),
'registry': registry, 'registry': registry,
@ -92,7 +120,7 @@ def main():
'api_queue': config.get('_CELERY_QUEUE'), 'api_queue': config.get('_CELERY_QUEUE'),
} }
syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh) syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
syncer.add_filter(handler) syncer.add_filter(handler)
syncer.loop(1, conn) syncer.loop(1, conn)

View File

@ -1,4 +1,4 @@
[traffic] [traffic]
#local.noop_traffic = 2 #local.noop_traffic = 2
local.account = 2 #local.account = 2
local.transfer = 2 local.transfer = 2

View File

@ -72,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications') argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification') argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification') argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error') argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@ -101,6 +102,8 @@ config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True) config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True) config.add(args.ussd_provider, '_USSD_PROVIDER', True)
token_symbol = args.token_symbol
logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@ -273,7 +276,10 @@ class Verifier:
def verify_balance(self, address, balance): def verify_balance(self, address, balance):
o = self.erc20_tx_factory.balance(self.token_address, address) o = self.erc20_tx_factory.balance(self.token_address, address)
r = self.conn.do(o) r = self.conn.do(o)
actual_balance = int(strip_0x(r), 16) try:
actual_balance = int(strip_0x(r), 16)
except ValueError:
actual_balance = int(r)
balance = int(balance / 1000000) * 1000000 balance = int(balance / 1000000) * 1000000
logg.debug('balance for {}: {}'.format(address, balance)) logg.debug('balance for {}: {}'.format(address, balance))
if balance != actual_balance: if balance != actual_balance:
@ -461,7 +467,7 @@ def main():
tx = txf.template(ZERO_ADDRESS, token_index_address) tx = txf.template(ZERO_ADDRESS, token_index_address)
data = add_0x(registry_addressof_method) data = add_0x(registry_addressof_method)
h = hashlib.new('sha256') h = hashlib.new('sha256')
h.update(b'SRF') h.update(token_symbol.encode('utf-8'))
z = h.digest() z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex() data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data) txf.set_code(tx, data)