Implement migration script with ussd and notify
This commit is contained in:
Louis Holbrook
417
apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py
Normal file
417
apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py
Normal file
@@ -0,0 +1,417 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import json
|
||||
import uuid
|
||||
import importlib
|
||||
import random
|
||||
import copy
|
||||
from argparse import RawTextHelpFormatter
|
||||
|
||||
# external imports
|
||||
import redis
|
||||
from cic_eth.api.api_task import Api
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def add_args(argparser):
|
||||
"""Parse script specific command line arguments
|
||||
|
||||
:param argparser: Top-level argument parser
|
||||
:type argparser: argparse.ArgumentParser
|
||||
"""
|
||||
argparser.formatter_class = formatter_class=RawTextHelpFormatter
|
||||
argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
|
||||
argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
|
||||
argparser.add_argument('--batch-size', dest='batch_size', default=10, type=int, help='number of events to process simultaneously')
|
||||
argparser.description = """Generates traffic on the cic network using dynamically loaded modules as event sources
|
||||
|
||||
"""
|
||||
return argparser
|
||||
|
||||
|
||||
class TrafficItem:
|
||||
"""Represents a single item of traffic meta that will be processed by a traffic generation method
|
||||
|
||||
The traffic generation module passed in the argument must implement a method "do" with interface conforming to local.noop_traffic.do.
|
||||
|
||||
:param item: Traffic generation module.
|
||||
:type item: function
|
||||
"""
|
||||
def __init__(self, item):
|
||||
self.method = item.do
|
||||
self.uuid = uuid.uuid4()
|
||||
self.ext = None
|
||||
self.result = None
|
||||
self.sender = None
|
||||
self.recipient = None
|
||||
self.source_token = None
|
||||
self.destination_token = None
|
||||
self.source_value = 0
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return 'traffic item method {} uuid {}'.format(self.method, self.uuid)
|
||||
|
||||
|
||||
class TrafficRouter:
|
||||
"""Holds and selects from the collection of traffic generator modules that will be used for the execution.
|
||||
|
||||
:params batch_size: Amount of simultaneous traffic items that can simultanously be in flight.
|
||||
:type batch_size: number
|
||||
:raises ValueError: If batch size is zero of negative
|
||||
"""
|
||||
def __init__(self, batch_size=1):
|
||||
if batch_size < 1:
|
||||
raise ValueError('batch size cannot be 0')
|
||||
self.items = []
|
||||
self.weights = []
|
||||
self.total_weights = 0
|
||||
self.batch_size = batch_size
|
||||
self.reserved = {}
|
||||
self.reserved_count = 0
|
||||
self.traffic = {}
|
||||
|
||||
|
||||
def add(self, item, weight):
|
||||
"""Add a traffic generator module to the list of modules to choose between for traffic item exectuion.
|
||||
|
||||
The probability that a module will be chosen for any single item is the ratio between the weight parameter and the accumulated weights for all items.
|
||||
|
||||
See local.noop for which criteria the generator module must fulfill.
|
||||
|
||||
:param item: Qualified class path to traffic generator module. Will be dynamically loaded.
|
||||
:type item: str
|
||||
:param weight: Selection probability weight
|
||||
:type weight: number
|
||||
:raises ModuleNotFound: Invalid item argument
|
||||
"""
|
||||
self.weights.append(self.total_weights)
|
||||
self.total_weights += weight
|
||||
m = importlib.import_module(item)
|
||||
self.items.append(m)
|
||||
|
||||
|
||||
def reserve(self):
|
||||
"""Selects the module to be used to execute the next traffic item, using the provided weights.
|
||||
|
||||
If the current number of calls to "reserve" without corresponding calls to "release" equals the set batch size limit, None will be returned. The calling code should allow a short grace period before trying the call again.
|
||||
:raises ValueError: No items have been added
|
||||
:returns: A traffic item with the selected module method as the method property.
|
||||
:rtype: TrafficItem|None
|
||||
"""
|
||||
if len(self.items) == 0:
|
||||
raise ValueError('Add at least one item first')
|
||||
|
||||
if len(self.reserved) == self.batch_size:
|
||||
return None
|
||||
|
||||
n = random.randint(0, self.total_weights)
|
||||
item = self.items[0]
|
||||
for i in range(len(self.weights)):
|
||||
if n <= self.weights[i]:
|
||||
item = self.items[i]
|
||||
break
|
||||
|
||||
ti = TrafficItem(item)
|
||||
self.reserved[ti.uuid] = ti
|
||||
return ti
|
||||
|
||||
|
||||
def release(self, traffic_item):
|
||||
"""Releases the traffic item from the list of simultaneous traffic items in flight.
|
||||
|
||||
:param traffic_item: Traffic item
|
||||
:type traffic_item: TrafficItem
|
||||
"""
|
||||
del self.reserved[traffic_item.uuid]
|
||||
|
||||
|
||||
def apply_import_dict(self, keys, dct):
|
||||
"""Convenience method to add traffic generator modules from a dictionary.
|
||||
|
||||
:param keys: Keys in dictionary to add
|
||||
:type keys: list of str
|
||||
:param dct: Dictionary to choose module strings from
|
||||
:type dct: dict
|
||||
:raises ModuleNotFoundError: If one of the module strings refer to an invalid module.
|
||||
"""
|
||||
# parse traffic items
|
||||
for k in keys:
|
||||
if len(k) > 8 and k[:8] == 'TRAFFIC_':
|
||||
v = int(dct.get(k))
|
||||
self.add(k[8:].lower(), v)
|
||||
logg.debug('found traffic item {} weight {}'.format(k, v))
|
||||
|
||||
|
||||
# TODO: This will not work well with big networks. The provisioner should use lazy loading and LRU instead.
|
||||
class TrafficProvisioner:
|
||||
"""Loads metadata necessary for traffic item execution.
|
||||
|
||||
Instantiation will by default trigger retrieval of accounts and tokens on the network.
|
||||
|
||||
It will also populate the aux property of the instance with the values from the static aux parameter template.
|
||||
"""
|
||||
|
||||
oracles = {
|
||||
'account': None,
|
||||
'token': None,
|
||||
}
|
||||
"""Data oracles to be used for traffic item generation"""
|
||||
default_aux = {
|
||||
}
|
||||
"""Aux parameter template to be passed to the traffic generator module"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.tokens = self.oracles['token'].get_tokens()
|
||||
self.accounts = self.oracles['account'].get_accounts()
|
||||
self.aux = copy.copy(self.default_aux)
|
||||
self.__balances = {}
|
||||
for a in self.accounts:
|
||||
self.__balances[a] = {}
|
||||
|
||||
|
||||
# Caches a single address' balance of a single token
|
||||
def __cache_balance(self, holder_address, token, value):
|
||||
if self.__balances.get(holder_address) == None:
|
||||
self.__balances[holder_address] = {}
|
||||
self.__balances[holder_address][token] = value
|
||||
logg.debug('setting cached balance of {} token {} to {}'.format(holder_address, token, value))
|
||||
|
||||
|
||||
def add_aux(self, k, v):
|
||||
"""Add a key-value pair to the aux parameter list.
|
||||
|
||||
Does not protect existing entries from being overwritten.
|
||||
|
||||
:param k: Key
|
||||
:type k: str
|
||||
:param v: Value
|
||||
:type v: any
|
||||
"""
|
||||
logg.debug('added {} = {} to traffictasker'.format(k, v))
|
||||
self.aux[k] = v
|
||||
|
||||
|
||||
# TODO: Balance list type should perhaps be a class (provided by cic-eth package) due to its complexity.
|
||||
def balances(self, refresh_accounts=None):
|
||||
"""Retrieves all token balances for the given account list.
|
||||
|
||||
If refresh_accounts is not None, the balance values for the given accounts will be retrieved from upstream. If the argument is an empty list, the balances will be updated for all tokens of all ccounts. If there are many accounts and/or tokens, this may be a VERY EXPENSIVE OPERATION. The "balance" method can be used instead to update individual account/token pair balances.
|
||||
|
||||
:param accounts: List of accounts to refresh balances for.
|
||||
:type accounts: list of str, 0x-hex
|
||||
:returns: Dict of dict of dicts; v[accounts][token] = {balance_types}
|
||||
:rtype: dict
|
||||
"""
|
||||
if refresh_accounts != None:
|
||||
accounts = refresh_accounts
|
||||
if len(accounts) == 0:
|
||||
accounts = self.accounts
|
||||
for account in accounts:
|
||||
for token in self.tokens:
|
||||
value = self.balance(account, token)
|
||||
self.__cache_balance(account, token.symbol(), value)
|
||||
logg.debug('balance sender {} token {} = {}'.format(account, token, value))
|
||||
else:
|
||||
logg.debug('returning cached balances')
|
||||
|
||||
return self.__balances
|
||||
|
||||
|
||||
# TODO: use proper redis callback
|
||||
def balance(self, account, token):
|
||||
"""Update balance for a single token of a single account from upstream.
|
||||
|
||||
The balance will be the spendable balance at the time of the call. This value may be less than the balance reported by the consensus network, if a previous outgoing transaction is still pending in the network or the custodial system queue.
|
||||
|
||||
:param account: Account to update
|
||||
:type account: str, 0x-hex
|
||||
:param token: Token to update balance for
|
||||
:type token: cic_registry.token.Token
|
||||
:returns: Updated balance
|
||||
:rtype: complex balance dict
|
||||
"""
|
||||
api = Api(
|
||||
str(self.aux['chain_spec']),
|
||||
queue=self.aux['api_queue'],
|
||||
#callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
|
||||
#callback_task='cic_eth.callbacks.redis.redis',
|
||||
#callback_queue=queue,
|
||||
)
|
||||
t = api.balance(account, token.symbol())
|
||||
r = t.get()
|
||||
for c in t.collect():
|
||||
r = c[1]
|
||||
assert t.successful()
|
||||
#return r[0]['balance_network'] - r[0]['balance_outgoing']
|
||||
return r[0]
|
||||
|
||||
|
||||
def update_balance(self, account, token, value):
|
||||
"""Manually set a token balance for an account.
|
||||
|
||||
:param account: Account to update
|
||||
:type account: str, 0x-hex
|
||||
:param token: Token to update balance for
|
||||
:type token: cic_registry.token.Token
|
||||
:param value: Balance value to set
|
||||
:type value: number
|
||||
:returns: Balance value (unchanged)
|
||||
:rtype: complex balance dict
|
||||
"""
|
||||
self.__cache_balance(account, token.symbol(), value)
|
||||
return value
|
||||
|
||||
|
||||
# TODO: Abstract redis with a generic pubsub adapter
|
||||
class TrafficSyncHandler:
|
||||
"""Encapsulates callback methods required by the chain syncer.
|
||||
|
||||
This implementation uses a redis subscription as backend to retrieve results from asynchronously executed tasks.
|
||||
|
||||
:param config: Configuration of current top-level execution
|
||||
:type config: object with dict get interface
|
||||
:param traffic_router: Traffic router instance to use for the syncer session.
|
||||
:type traffic_router: TrafficRouter
|
||||
:raises Exception: Any Exception redis may raise on connection attempt.
|
||||
"""
|
||||
def __init__(self, config, traffic_router):
|
||||
self.traffic_router = traffic_router
|
||||
self.redis_channel = str(uuid.uuid4())
|
||||
self.pubsub = self.__connect_redis(self.redis_channel, config)
|
||||
self.traffic_items = {}
|
||||
self.config = config
|
||||
self.init = False
|
||||
|
||||
|
||||
# connects to redis
|
||||
def __connect_redis(self, redis_channel, config):
|
||||
r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB'))
|
||||
redis_pubsub = r.pubsub()
|
||||
redis_pubsub.subscribe(redis_channel)
|
||||
logg.debug('redis connected on channel {}'.format(redis_channel))
|
||||
return redis_pubsub
|
||||
|
||||
|
||||
# TODO: This method is too long, split up
|
||||
# TODO: This method will not yet cache balances for newly created accounts
|
||||
def refresh(self, block_number, tx_index):
|
||||
"""Traffic method and item execution driver to be called on every loop execution of the chain syncer.
|
||||
|
||||
Implements the signature required by callbacks called from chainsyncer.driver.loop.
|
||||
|
||||
:param block_number: Syncer block height at time of call.
|
||||
:type block_number: number
|
||||
:param tx_index: Syncer block transaction index at time of call.
|
||||
:type tx_index: number
|
||||
"""
|
||||
traffic_provisioner = TrafficProvisioner()
|
||||
traffic_provisioner.add_aux('redis_channel', self.redis_channel)
|
||||
|
||||
refresh_accounts = None
|
||||
# Note! This call may be very expensive if there are a lot of accounts and/or tokens on the network
|
||||
if not self.init:
|
||||
refresh_accounts = traffic_provisioner.accounts
|
||||
balances = traffic_provisioner.balances(refresh_accounts=refresh_accounts)
|
||||
self.init = True
|
||||
|
||||
if len(traffic_provisioner.tokens) == 0:
|
||||
logg.error('patiently waiting for at least one registered token...')
|
||||
return
|
||||
|
||||
logg.debug('executing handler refresh with accounts {}'.format(traffic_provisioner.accounts))
|
||||
logg.debug('executing handler refresh with tokens {}'.format(traffic_provisioner.tokens))
|
||||
|
||||
sender_indices = [*range(0, len(traffic_provisioner.accounts))]
|
||||
# TODO: only get balances for the selection that we will be generating for
|
||||
|
||||
while True:
|
||||
traffic_item = self.traffic_router.reserve()
|
||||
if traffic_item == None:
|
||||
logg.debug('no traffic_items left to reserve {}'.format(traffic_item))
|
||||
break
|
||||
|
||||
# TODO: temporary selection
|
||||
token_pair = (
|
||||
traffic_provisioner.tokens[0],
|
||||
traffic_provisioner.tokens[0],
|
||||
)
|
||||
sender_index_index = random.randint(0, len(sender_indices)-1)
|
||||
sender_index = sender_indices[sender_index_index]
|
||||
sender = traffic_provisioner.accounts[sender_index]
|
||||
#balance_full = balances[sender][token_pair[0].symbol()]
|
||||
if len(sender_indices) == 1:
|
||||
sender_indices[m] = sender_sender_indices[len(senders)-1]
|
||||
sender_indices = sender_indices[:len(sender_indices)-1]
|
||||
|
||||
balance_full = traffic_provisioner.balance(sender, token_pair[0])
|
||||
|
||||
recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
|
||||
recipient = traffic_provisioner.accounts[recipient_index]
|
||||
|
||||
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
|
||||
(e, t, balance_result,) = traffic_item.method(
|
||||
token_pair,
|
||||
sender,
|
||||
recipient,
|
||||
balance_full,
|
||||
traffic_provisioner.aux,
|
||||
block_number,
|
||||
tx_index,
|
||||
)
|
||||
traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
|
||||
sender_indices.append(recipient_index)
|
||||
|
||||
if e != None:
|
||||
logg.info('failed {}: {}'.format(str(traffic_item), e))
|
||||
self.traffic_router.release(traffic_item)
|
||||
continue
|
||||
|
||||
if t == None:
|
||||
logg.info('traffic method {} completed immediately')
|
||||
self.traffic_router.release(traffic_item)
|
||||
traffic_item.ext = t
|
||||
self.traffic_items[traffic_item.ext] = traffic_item
|
||||
|
||||
|
||||
while True:
|
||||
m = self.pubsub.get_message(timeout=0.1)
|
||||
if m == None:
|
||||
break
|
||||
logg.debug('redis message {}'.format(m))
|
||||
if m['type'] == 'message':
|
||||
message_data = json.loads(m['data'])
|
||||
uu = message_data['root_id']
|
||||
match_item = self.traffic_items[uu]
|
||||
self.traffic_router.release(match_item)
|
||||
logg.debug('>>>>>>>>>>>>>>>>>>> match item {} {} {}'.format(match_item, match_item.result, dir(match_item)))
|
||||
if message_data['status'] != 0:
|
||||
logg.error('task item {} failed with error code {}'.format(match_item, message_data['status']))
|
||||
else:
|
||||
match_item.result = message_data['result']
|
||||
logg.debug('got callback result: {}'.format(match_item))
|
||||
|
||||
|
||||
def name(self):
|
||||
"""Returns the common name for the syncer callback implementation. Required by the chain syncer.
|
||||
"""
|
||||
return 'traffic_item_handler'
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, db_session):
|
||||
"""Callback for every transaction found in a block. Required by the chain syncer.
|
||||
|
||||
Currently performs no operation.
|
||||
|
||||
:param conn: A HTTPConnection object to the chain rpc provider.
|
||||
:type conn: chainlib.eth.rpc.HTTPConnection
|
||||
:param block: The block object of current transaction
|
||||
:type block: chainlib.eth.block.Block
|
||||
:param tx: The block transaction object
|
||||
:type tx: chainlib.eth.tx.Tx
|
||||
:param db_session: Syncer backend database session
|
||||
:type db_session: SQLAlchemy.Session
|
||||
"""
|
||||
logg.debug('handler get {}'.format(tx))
|
||||
@@ -0,0 +1,8 @@
|
||||
from . import (
|
||||
log,
|
||||
argparse,
|
||||
config,
|
||||
signer,
|
||||
rpc,
|
||||
registry,
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
default_config_dir = os.environ.get('CONFINI_DIR')
|
||||
full_template = {
|
||||
# (long arg and key name, short var, type, default, help,)
|
||||
'provider': ('p', str, None, 'RPC provider url',),
|
||||
'registry_address': ('r', str, None, 'CIC registry address',),
|
||||
'keystore_file': ('y', str, None, 'Keystore file',),
|
||||
'config_dir': ('c', str, default_config_dir, 'Configuration directory',),
|
||||
'queue': ('q', str, 'cic-eth', 'Celery task queue',),
|
||||
'chain_spec': ('i', str, None, 'Chain spec string',),
|
||||
'abi_dir': (None, str, None, 'Smart contract ABI search path',),
|
||||
'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
|
||||
}
|
||||
default_include_args = [
|
||||
'config_dir',
|
||||
'provider',
|
||||
'env_prefix',
|
||||
]
|
||||
|
||||
sub = None
|
||||
|
||||
def create(caller_dir, include_args=default_include_args):
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
|
||||
for k in include_args:
|
||||
a = full_template[k]
|
||||
long_flag = '--' + k.replace('_', '-')
|
||||
short_flag = None
|
||||
dest = None
|
||||
if a[0] != None:
|
||||
short_flag = '-' + a[0]
|
||||
dest = a[0]
|
||||
else:
|
||||
dest = k
|
||||
default = a[2]
|
||||
if default == None and k == 'config_dir':
|
||||
default = os.path.join(caller_dir, 'config')
|
||||
|
||||
if short_flag == None:
|
||||
argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
|
||||
else:
|
||||
argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
|
||||
|
||||
argparser.add_argument('-v', action='store_true', help='Be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='Be more verbose')
|
||||
|
||||
return argparser
|
||||
|
||||
|
||||
def add(argparser, processor, name, description=None):
|
||||
processor(argparser)
|
||||
|
||||
return argparser
|
||||
|
||||
|
||||
def parse(argparser, logger=None):
|
||||
|
||||
args = argparser.parse_args(sys.argv[1:])
|
||||
|
||||
# handle logging input
|
||||
if logger != None:
|
||||
if args.vv:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
return args
|
||||
@@ -0,0 +1,39 @@
|
||||
# external imports
|
||||
import logging
|
||||
import confini
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
default_arg_overrides = {
|
||||
'abi_dir': 'ETH_ABI_DIR',
|
||||
'p': 'ETH_PROVIDER',
|
||||
'i': 'CIC_CHAIN_SPEC',
|
||||
'r': 'CIC_REGISTRY_ADDRESS',
|
||||
}
|
||||
|
||||
|
||||
def override(config, override_dict, label):
|
||||
config.dict_override(override_dict, label)
|
||||
config.validate()
|
||||
return config
|
||||
|
||||
|
||||
def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
|
||||
# handle config input
|
||||
config = confini.Config(config_dir, env_prefix)
|
||||
config.process()
|
||||
if arg_overrides != None and args != None:
|
||||
override_dict = {}
|
||||
for k in arg_overrides:
|
||||
v = getattr(args, k)
|
||||
if v != None:
|
||||
override_dict[arg_overrides[k]] = v
|
||||
config = override(config, override_dict, 'args')
|
||||
else:
|
||||
config.validate()
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def log(config):
|
||||
logg.debug('config loaded:\n{}'.format(config))
|
||||
@@ -0,0 +1,18 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
|
||||
default_mutelist = [
|
||||
'urllib3',
|
||||
'websockets.protocol',
|
||||
'web3.RequestManager',
|
||||
'web3.providers.WebsocketProvider',
|
||||
'web3.providers.HTTPProvider',
|
||||
]
|
||||
|
||||
def create(name=None, mutelist=default_mutelist):
|
||||
logg = logging.getLogger(name)
|
||||
for m in mutelist:
|
||||
logging.getLogger(m).setLevel(logging.CRITICAL)
|
||||
return logg
|
||||
@@ -0,0 +1,86 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import copy
|
||||
|
||||
# external imports
|
||||
from cic_registry import CICRegistry
|
||||
from eth_token_index import TokenUniqueSymbolIndex
|
||||
from eth_accounts_index import AccountRegistry
|
||||
from chainlib.chain import ChainSpec
|
||||
from cic_registry.chain import ChainRegistry
|
||||
from cic_registry.helper.declarator import DeclaratorOracleAdapter
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TokenOracle:
|
||||
|
||||
def __init__(self, conn, chain_spec, registry):
|
||||
self.tokens = []
|
||||
self.chain_spec = chain_spec
|
||||
self.registry = registry
|
||||
|
||||
token_registry_contract = CICRegistry.get_contract(chain_spec, 'TokenRegistry', 'Registry')
|
||||
self.getter = TokenUniqueSymbolIndex(conn, token_registry_contract.address())
|
||||
|
||||
|
||||
def get_tokens(self):
|
||||
token_count = self.getter.count()
|
||||
if token_count == len(self.tokens):
|
||||
return self.tokens
|
||||
|
||||
for i in range(len(self.tokens), token_count):
|
||||
token_address = self.getter.get_index(i)
|
||||
t = self.registry.get_address(self.chain_spec, token_address)
|
||||
token_symbol = t.symbol()
|
||||
self.tokens.append(t)
|
||||
|
||||
logg.debug('adding token idx {} symbol {} address {}'.format(i, token_symbol, token_address))
|
||||
|
||||
return copy.copy(self.tokens)
|
||||
|
||||
|
||||
class AccountsOracle:
|
||||
|
||||
def __init__(self, conn, chain_spec, registry):
|
||||
self.accounts = []
|
||||
self.chain_spec = chain_spec
|
||||
self.registry = registry
|
||||
|
||||
accounts_registry_contract = CICRegistry.get_contract(chain_spec, 'AccountRegistry', 'Registry')
|
||||
self.getter = AccountRegistry(conn, accounts_registry_contract.address())
|
||||
|
||||
|
||||
def get_accounts(self):
|
||||
accounts_count = self.getter.count()
|
||||
if accounts_count == len(self.accounts):
|
||||
return self.accounts
|
||||
|
||||
for i in range(len(self.accounts), accounts_count):
|
||||
account = self.getter.get_index(i)
|
||||
self.accounts.append(account)
|
||||
logg.debug('adding account {}'.format(account))
|
||||
|
||||
return copy.copy(self.accounts)
|
||||
|
||||
|
||||
def init_legacy(config, w3):
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
CICRegistry.init(w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
|
||||
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
|
||||
|
||||
chain_registry = ChainRegistry(chain_spec)
|
||||
CICRegistry.add_chain_registry(chain_registry, True)
|
||||
|
||||
declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
|
||||
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
|
||||
if trusted_addresses_src == None:
|
||||
raise ValueError('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
|
||||
trusted_addresses = trusted_addresses_src.split(',')
|
||||
for address in trusted_addresses:
|
||||
logg.info('using trusted address {}'.format(address))
|
||||
|
||||
oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
|
||||
chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
|
||||
|
||||
return CICRegistry
|
||||
@@ -0,0 +1,18 @@
|
||||
# standard imports
|
||||
import re
|
||||
|
||||
# external imports
|
||||
import web3
|
||||
|
||||
def create(url):
|
||||
# web3 input
|
||||
# TODO: Replace with chainlib
|
||||
re_websocket = r'^wss?:'
|
||||
re_http = r'^https?:'
|
||||
blockchain_provider = None
|
||||
if re.match(re_websocket, url):
|
||||
blockchain_provider = web3.Web3.WebsocketProvider(url)
|
||||
elif re.match(re_http, url):
|
||||
blockchain_provider = web3.Web3.HTTPProvider(url)
|
||||
w3 = web3.Web3(blockchain_provider)
|
||||
return w3
|
||||
@@ -0,0 +1,23 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
|
||||
from crypto_dev_signer.keystore import DictKeystore
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
keystore = DictKeystore()
|
||||
|
||||
def from_keystore(keyfile):
|
||||
global keystore
|
||||
|
||||
# signer
|
||||
if keyfile == None:
|
||||
raise ValueError('please specify signer keystore file')
|
||||
|
||||
logg.debug('loading keystore file {}'.format(keyfile))
|
||||
address = keystore.import_keystore_file(keyfile)
|
||||
|
||||
signer = EIP155Signer(keystore)
|
||||
return (address, signer,)
|
||||
@@ -0,0 +1,37 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from cic_eth.api.api_task import Api
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
queue = 'cic-eth'
|
||||
name = 'account'
|
||||
|
||||
|
||||
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
|
||||
"""Triggers creation and registration of new account through the custodial cic-eth component.
|
||||
|
||||
It expects the following aux parameters to exist:
|
||||
- redis_host_callback: Redis host name exposed to cic-eth, for callback
|
||||
- redis_port_callback: Redis port exposed to cic-eth, for callback
|
||||
- redis_db: Redis db, for callback
|
||||
- redis_channel: Redis channel, for callback
|
||||
- chain_spec: Chain specification for the chain to execute the transfer on
|
||||
|
||||
See local.noop.do for details on parameters and return values.
|
||||
"""
|
||||
logg.debug('running {} {} {}'.format(__name__, token_pair, sender, recipient))
|
||||
api = Api(
|
||||
str(aux['chain_spec']),
|
||||
queue=queue,
|
||||
callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
|
||||
callback_task='cic_eth.callbacks.redis.redis',
|
||||
callback_queue=queue,
|
||||
)
|
||||
|
||||
t = api.create_account(register=True)
|
||||
|
||||
return (None, t, sender_balance, )
|
||||
@@ -0,0 +1,37 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
|
||||
"""Defines the function signature for a traffic generator. The method itself only logs the input parameters.
|
||||
|
||||
If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
|
||||
|
||||
If the task_id position in the return tuple is None, the calling code should interpret the traffic item to have been synchronously completed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
|
||||
|
||||
The balance element of the result is the balance dict passed as argument, with fields updated according to the expected delta as a result of the operation. However, in the event that the generator module dispatches an asynchronous event then there is no guarantee that this balance will actually be the correct result. The caller should take care to periodically re-sync balance from the upstream.
|
||||
|
||||
:param token_pair: Source and destination tokens for the traffic item.
|
||||
:type token_pair: 2-element tuple with cic_registry.token.Token
|
||||
:param sender: Sender address
|
||||
:type sender: str, 0x-hex
|
||||
:param recipient: Recipient address
|
||||
:type recipient: str, 0x-hex
|
||||
:param sender_balance: Sender balance in full decimal resolution
|
||||
:type sender_balance: complex balance dict
|
||||
:param aux: Custom parameters defined by traffic generation client code
|
||||
:type aux: dict
|
||||
:param block_number: Syncer block number position at time of method call
|
||||
:type block_number: number
|
||||
:param tx_index: Syncer block transaction index position at time of method call
|
||||
:type tx_index: number
|
||||
:raises KeyError: Missing required aux element
|
||||
:returns: Exception|None, task_id|None and adjusted_sender_balance respectively
|
||||
:rtype: tuple
|
||||
"""
|
||||
logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
|
||||
|
||||
return (None, None, sender_balance, )
|
||||
@@ -0,0 +1,51 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import random
|
||||
|
||||
# external imports
|
||||
from cic_eth.api.api_task import Api
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
queue = 'cic-eth'
|
||||
name = 'erc20_transfer'
|
||||
|
||||
|
||||
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
|
||||
"""Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
|
||||
|
||||
It expects the following aux parameters to exist:
|
||||
- redis_host_callback: Redis host name exposed to cic-eth, for callback
|
||||
- redis_port_callback: Redis port exposed to cic-eth, for callback
|
||||
- redis_db: Redis db, for callback
|
||||
- redis_channel: Redis channel, for callback
|
||||
- chain_spec: Chain specification for the chain to execute the transfer on
|
||||
|
||||
See local.noop.do for details on parameters and return values.
|
||||
"""
|
||||
logg.debug('running {} {} {} {}'.format(__name__, token_pair, sender, recipient))
|
||||
|
||||
decimals = token_pair[0].decimals()
|
||||
|
||||
sender_balance_value = sender_balance['balance_network'] - sender_balance['balance_outgoing']
|
||||
|
||||
balance_units = int(sender_balance_value / decimals)
|
||||
|
||||
if balance_units <= 0:
|
||||
return (AttributeError('sender {} has zero balance'), None, 0,)
|
||||
|
||||
spend_units = random.randint(1, balance_units)
|
||||
spend_value = spend_units * decimals
|
||||
|
||||
api = Api(
|
||||
str(aux['chain_spec']),
|
||||
queue=queue,
|
||||
callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
|
||||
callback_task='cic_eth.callbacks.redis.redis',
|
||||
callback_queue=queue,
|
||||
)
|
||||
t = api.transfer(sender, recipient, spend_value, token_pair[0].symbol())
|
||||
|
||||
sender_balance['balance_outgoing'] += spend_value
|
||||
return (None, t, sender_balance,)
|
||||
101
apps/contract-migration/scripts/cic_eth/traffic/traffic.py
Normal file
101
apps/contract-migration/scripts/cic_eth/traffic/traffic.py
Normal file
@@ -0,0 +1,101 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
|
||||
# external imports
|
||||
import redis
|
||||
import celery
|
||||
from chainsyncer.backend import MemBackend
|
||||
from chainsyncer.driver import HeadSyncer
|
||||
from chainlib.eth.connection import HTTPConnection
|
||||
from chainlib.eth.gas import DefaultGasOracle
|
||||
from chainlib.eth.nonce import DefaultNonceOracle
|
||||
from chainlib.eth.block import block_latest
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
import common
|
||||
from cmd.traffic import (
|
||||
TrafficItem,
|
||||
TrafficRouter,
|
||||
TrafficProvisioner,
|
||||
TrafficSyncHandler,
|
||||
)
|
||||
from cmd.traffic import add_args as add_traffic_args
|
||||
|
||||
|
||||
# common basics
|
||||
script_dir = os.path.realpath(os.path.dirname(__file__))
|
||||
logg = common.log.create()
|
||||
argparser = common.argparse.create(script_dir, common.argparse.full_template)
|
||||
argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
|
||||
args = common.argparse.parse(argparser, logg)
|
||||
config = common.config.create(args.c, args, args.env_prefix)
|
||||
|
||||
# map custom args to local config entries
|
||||
batchsize = args.batch_size
|
||||
if batchsize < 1:
|
||||
batchsize = 1
|
||||
logg.info('batch size {}'.format(batchsize))
|
||||
config.add(batchsize, '_BATCH_SIZE', True)
|
||||
|
||||
config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
|
||||
config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
|
||||
|
||||
config.add(args.y, '_KEYSTORE_FILE', True)
|
||||
|
||||
config.add(args.q, '_CELERY_QUEUE', True)
|
||||
|
||||
common.config.log(config)
|
||||
|
||||
|
||||
def main():
|
||||
# create signer (not currently in use, but needs to be accessible for custom traffic item generators)
|
||||
(signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
|
||||
|
||||
# connect to celery
|
||||
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
|
||||
|
||||
# set up registry
|
||||
w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
|
||||
registry = common.registry.init_legacy(config, w3)
|
||||
|
||||
# Connect to blockchain with chainlib
|
||||
conn = HTTPConnection(config.get('ETH_PROVIDER'))
|
||||
gas_oracle = DefaultGasOracle(conn)
|
||||
nonce_oracle = DefaultNonceOracle(signer_address, conn)
|
||||
|
||||
# Set up magic traffic handler
|
||||
traffic_router = TrafficRouter()
|
||||
traffic_router.apply_import_dict(config.all(), config)
|
||||
handler = TrafficSyncHandler(config, traffic_router)
|
||||
|
||||
# Set up syncer
|
||||
syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
|
||||
o = block_latest()
|
||||
r = conn.do(o)
|
||||
block_offset = int(strip_0x(r), 16) + 1
|
||||
syncer_backend.set(block_offset, 0)
|
||||
|
||||
# Set up provisioner for common task input data
|
||||
TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
|
||||
TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
|
||||
TrafficProvisioner.default_aux = {
|
||||
'chain_spec': config.get('CIC_CHAIN_SPEC'),
|
||||
'registry': registry,
|
||||
'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
|
||||
'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
|
||||
'redis_db': config.get('REDIS_DB'),
|
||||
'api_queue': config.get('_CELERY_QUEUE'),
|
||||
}
|
||||
|
||||
syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
|
||||
syncer.add_filter(handler)
|
||||
syncer.loop(1, conn)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user