kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'lash/rehabilitate-traffic-2' into 'master'

Browse Source 
Rehabilitate traffic generator script

See merge request grassrootseconomics/cic-internal-integration!145
This commit is contained in:
Louis Holbrook 2021-05-20 21:25:15 +00:00
commit d7113f3923
12 changed files with 159 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/config/test/syncer.ini Normal file
View File

@ -0,0 +1,2 @@
[syncer]
loop_interval = 1

2
apps/cic-eth/cic_eth/version.py
View File

@ -10,7 +10,7 @@ version = (
0,
11,
0,
'beta.13',
'beta.14',
)
version_object = semver.VersionInfo(

76
apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py Normal file
View File

@ -0,0 +1,76 @@
# external imports
from chainlib.jsonrpc import JSONRPCException
from eth_erc20 import ERC20
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
class ERC20Token:
def __init__(self, chain_spec, address, conn):
self.__address = address
c = ERC20(chain_spec)
o = c.symbol(address)
r = conn.do(o)
self.__symbol = c.parse_symbol(r)
o = c.decimals(address)
r = conn.do(o)
self.__decimals = c.parse_decimals(r)
def symbol(self):
return self.__symbol
def decimals(self):
return self.__decimals
class IndexCache:
def __init__(self, chain_spec, address):
self.address = address
self.chain_spec = chain_spec
def parse(self, r):
return r
def get(self, conn):
entries = []
i = 0
while True:
o = self.o.entry(self.address, i)
try:
r = conn.do(o)
entries.append(self.parse(r, conn))
except JSONRPCException:
return entries
i += 1
class AccountRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(AccountRegistryCache, self).__init__(chain_spec, address)
self.o = AccountsIndex(chain_spec)
self.get_accounts = self.get
def parse(self, r, conn):
return self.o.parse_account(r)
class TokenRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(TokenRegistryCache, self).__init__(chain_spec, address)
self.o = TokenUniqueSymbolIndex(chain_spec)
self.get_tokens = self.get
def parse(self, r, conn):
token_address = self.o.parse_entry(r)
return ERC20Token(self.chain_spec, token_address, conn)

23
apps/data-seeding/cic_eth/traffic/cmd/traffic.py
View File

@ -163,9 +163,9 @@ class TrafficProvisioner:
"""Aux parameter template to be passed to the traffic generator module"""
def __init__(self):
self.tokens = self.oracles['token'].get_tokens()
self.accounts = self.oracles['account'].get_accounts()
def __init__(self, conn):
self.tokens = self.oracles['token'].get_tokens(conn)
self.accounts = self.oracles['account'].get_accounts(conn)
self.aux = copy.copy(self.default_aux)
self.__balances = {}
for a in self.accounts:
@ -277,13 +277,14 @@ class TrafficSyncHandler:
:type traffic_router: TrafficRouter
:raises Exception: Any Exception redis may raise on connection attempt.
"""
def __init__(self, config, traffic_router):
def __init__(self, config, traffic_router, conn):
self.traffic_router = traffic_router
self.redis_channel = str(uuid.uuid4())
self.pubsub = self.__connect_redis(self.redis_channel, config)
self.traffic_items = {}
self.config = config
self.init = False
self.conn = conn
# connects to redis
@ -307,7 +308,7 @@ class TrafficSyncHandler:
:param tx_index: Syncer block transaction index at time of call.
:type tx_index: number
"""
traffic_provisioner = TrafficProvisioner()
traffic_provisioner = TrafficProvisioner(self.conn)
traffic_provisioner.add_aux('redis_channel', self.redis_channel)
refresh_accounts = None
@ -343,7 +344,7 @@ class TrafficSyncHandler:
sender = traffic_provisioner.accounts[sender_index]
#balance_full = balances[sender][token_pair[0].symbol()]
if len(sender_indices) == 1:
sender_indices[m] = sender_sender_indices[len(senders)-1]
sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
sender_indices = sender_indices[:len(sender_indices)-1]
balance_full = traffic_provisioner.balance(sender, token_pair[0])
@ -351,7 +352,14 @@ class TrafficSyncHandler:
recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
recipient = traffic_provisioner.accounts[recipient_index]
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
traffic_item,
token_pair,
sender,
recipient,
balance_full,
)
)
(e, t, balance_result,) = traffic_item.method(
token_pair,
sender,
@ -359,7 +367,6 @@ class TrafficSyncHandler:
balance_full,
traffic_provisioner.aux,
block_number,
tx_index,
)
traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
sender_indices.append(recipient_index)

2
apps/data-seeding/cic_eth/traffic/common/registry.py
View File

@ -3,7 +3,7 @@ import logging
import copy
# external imports
from cic_registry import CICRegistry
from cic_registry.registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
from eth_accounts_index import AccountRegistry
from chainlib.chain import ChainSpec

2
apps/data-seeding/cic_eth/traffic/common/signer.py
View File

@ -3,7 +3,7 @@ import logging
# external imports
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore import DictKeystore
from crypto_dev_signer.keystore.dict import DictKeystore
logg = logging.getLogger(__name__)

2
apps/data-seeding/cic_eth/traffic/local/account.py
View File

@ -11,7 +11,7 @@ queue = 'cic-eth'
name = 'account'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers creation and registration of new account through the custodial cic-eth component.
It expects the following aux parameters to exist:

6
apps/data-seeding/cic_eth/traffic/local/noop.py
View File

@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Defines the function signature for a traffic generator. The method itself only logs the input parameters.
If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
:type aux: dict
:param block_number: Syncer block number position at time of method call
:type block_number: number
:param tx_index: Syncer block transaction index position at time of method call
:type tx_index: number
:raises KeyError: Missing required aux element
:returns: Exception|None, task_id|None and adjusted_sender_balance respectively
:rtype: tuple
"""
logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
return (None, None, sender_balance, )

4
apps/data-seeding/cic_eth/traffic/local/transfer.py
View File

@ -12,7 +12,7 @@ queue = 'cic-eth'
name = 'erc20_transfer'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
It expects the following aux parameters to exist:
@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
balance_units = int(sender_balance_value / decimals)
if balance_units <= 0:
return (AttributeError('sender {} has zero balance'), None, 0,)
return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
spend_units = random.randint(1, balance_units)
spend_value = spend_units * decimals

70
apps/data-seeding/cic_eth/traffic/traffic.py
View File

@ -8,16 +8,25 @@ import json
# external imports
import redis
import celery
from chainsyncer.backend import MemBackend
from cic_eth_registry.registry import CICRegistry
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver import HeadSyncer
from chainlib.eth.connection import HTTPConnection
from chainlib.eth.gas import DefaultGasOracle
from chainlib.eth.nonce import DefaultNonceOracle
from chainlib.eth.connection import EthHTTPConnection
from chainlib.chain import ChainSpec
from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.block import block_latest
from hexathon import strip_0x
from cic_base import (
argparse,
config,
log,
rpc,
signer as signer_funcs,
)
# local imports
import common
#import common
from cmd.traffic import (
TrafficItem,
TrafficRouter,
@ -25,15 +34,19 @@ from cmd.traffic import (
TrafficSyncHandler,
)
from cmd.traffic import add_args as add_traffic_args
from cmd.cache import (
AccountRegistryCache,
TokenRegistryCache,
)
# common basics
script_dir = os.path.realpath(os.path.dirname(__file__))
logg = common.log.create()
argparser = common.argparse.create(script_dir, common.argparse.full_template)
argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
args = common.argparse.parse(argparser, logg)
config = common.config.create(args.c, args, args.env_prefix)
logg = log.create()
argparser = argparse.create(script_dir, argparse.full_template)
argparser = argparse.add(argparser, add_traffic_args, 'traffic')
args = argparse.parse(argparser, logg)
config = config.create(args.c, args, args.env_prefix)
# map custom args to local config entries
batchsize = args.batch_size
@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True)
config.add(args.q, '_CELERY_QUEUE', True)
common.config.log(config)
logg.debug(config)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
def main():
# create signer (not currently in use, but needs to be accessible for custom traffic item generators)
(signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
(signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
# connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
# set up registry
w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
registry = common.registry.init_legacy(config, w3)
rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
#registry = registry.init_legacy(config, w3)
CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
registry = CICRegistry(chain_spec, conn)
# Connect to blockchain with chainlib
conn = HTTPConnection(config.get('ETH_PROVIDER'))
gas_oracle = DefaultGasOracle(conn)
nonce_oracle = DefaultNonceOracle(signer_address, conn)
gas_oracle = RPCGasOracle(conn)
nonce_oracle = RPCNonceOracle(signer_address, conn)
# Set up magic traffic handler
traffic_router = TrafficRouter()
traffic_router.apply_import_dict(config.all(), config)
handler = TrafficSyncHandler(config, traffic_router)
handler = TrafficSyncHandler(config, traffic_router, conn)
# Set up syncer
syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
@ -80,9 +96,21 @@ def main():
block_offset = int(strip_0x(r), 16) + 1
syncer_backend.set(block_offset, 0)
# get relevant registry entries
token_registry = registry.lookup('TokenRegistry')
logg.info('using token registry {}'.format(token_registry))
token_cache = TokenRegistryCache(chain_spec, token_registry)
account_registry = registry.lookup('AccountRegistry')
logg.info('using account registry {}'.format(account_registry))
account_cache = AccountRegistryCache(chain_spec, account_registry)
# Set up provisioner for common task input data
TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
#TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
#TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['token'] = token_cache
TrafficProvisioner.oracles['account'] = account_cache
TrafficProvisioner.default_aux = {
'chain_spec': config.get('CIC_CHAIN_SPEC'),
'registry': registry,
@ -92,7 +120,7 @@ def main():
'api_queue': config.get('_CELERY_QUEUE'),
}
syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
syncer.add_filter(handler)
syncer.loop(1, conn)

2
apps/data-seeding/config/traffic.ini
View File

@ -1,4 +1,4 @@
[traffic]
#local.noop_traffic = 2
local.account = 2
#local.account = 2
local.transfer = 2

10
apps/data-seeding/verify.py
View File

@ -72,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@ -101,6 +102,8 @@ config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True)
token_symbol = args.token_symbol
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@ -273,7 +276,10 @@ class Verifier:
def verify_balance(self, address, balance):
o = self.erc20_tx_factory.balance(self.token_address, address)
r = self.conn.do(o)
actual_balance = int(strip_0x(r), 16)
try:
actual_balance = int(strip_0x(r), 16)
except ValueError:
actual_balance = int(r)
balance = int(balance / 1000000) * 1000000
logg.debug('balance for {}: {}'.format(address, balance))
if balance != actual_balance:
@ -461,7 +467,7 @@ def main():
tx = txf.template(ZERO_ADDRESS, token_index_address)
data = add_0x(registry_addressof_method)
h = hashlib.new('sha256')
h.update(b'SRF')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)