grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: Split and improve contract migration steps

Browse Source
This commit is contained in:
Louis Holbrook
2021-10-20 15:02:36 +00:00
committed by Philip Wafula
parent 8f1afa094d
commit 13fb67d2d8
96 changed files with 2147 additions and 1327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
apps/data-seeding/cic_eth/import_balance.py
View File

@@ -47,12 +47,14 @@ from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
config_dir = './config'
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
@@ -71,22 +73,21 @@ if args.v == True:
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
# override args
config = None
if args.c != None:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
else:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CHAIN_SPEC': getattr(args, 'i'),
'RPC_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'KEYSTORE_FILE_PATH': getattr(args, 'y'),
'WALLET_KEY_FILE': getattr(args, 'y'),
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@@ -99,7 +100,7 @@ if args.y != None:
signer = EIP155Signer(keystore)
queue = args.q
chain_str = config.get('CIC_CHAIN_SPEC')
chain_str = config.get('CHAIN_SPEC')
block_offset = 0
if args.head:
block_offset = -1
@@ -192,7 +193,7 @@ def progress_callback(block_number, tx_index):
def main():
global chain_str, block_offset, user_dir
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
nonce_oracle = RPCNonceOracle(signer_address, conn)

75
apps/data-seeding/cic_eth/import_users.py
View File

@@ -10,7 +10,7 @@ import time
import phonenumbers
from glob import glob
# third-party imports
# external imports
import redis
import confini
import celery
@@ -23,15 +23,23 @@ from cic_types.models.person import Person
from cic_eth.api.api_task import Api
from chainlib.chain import ChainSpec
from cic_types.processor import generate_metadata_pointer
from cic_types import MetadataPointer
# local imports
from common.dirs import initialize_dirs
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
default_config_dir = '/usr/local/etc/cic'
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
argparser.add_argument('-f', action='store_true', help='force clear previous state')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
@@ -52,16 +60,21 @@ if args.v:
elif args.vv:
logg.setLevel(logging.DEBUG)
config_dir = args.c
config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config = None
if args.c != None:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
else:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CHAIN_SPEC': getattr(args, 'i'),
'REDIS_HOST': getattr(args, 'redis_host'),
'REDIS_PORT': getattr(args, 'redis_port'),
'REDIS_DB': getattr(args, 'redis_db'),
}
config.dict_override(args_override, 'cli')
config.add(args.user_dir, '_USERDIR', True)
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
redis_host = config.get('REDIS_HOST')
@@ -71,44 +84,25 @@ r = redis.Redis(redis_host, redis_port, redis_db)
ps = r.pubsub()
user_new_dir = os.path.join(args.user_dir, 'new')
os.makedirs(user_new_dir)
meta_dir = os.path.join(args.user_dir, 'meta')
os.makedirs(meta_dir)
custom_dir = os.path.join(args.user_dir, 'custom')
os.makedirs(custom_dir)
os.makedirs(os.path.join(custom_dir, 'new'))
os.makedirs(os.path.join(custom_dir, 'meta'))
phone_dir = os.path.join(args.user_dir, 'phone')
os.makedirs(os.path.join(phone_dir, 'meta'))
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
user_dir = args.user_dir
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
chain_str = str(chain_spec)
batch_size = args.batch_size
batch_delay = args.batch_delay
dirs = initialize_dirs(config.get('_USERDIR'), force_reset=args.f)
def register_eth(i, u):
redis_channel = str(uuid.uuid4())
ps.subscribe(redis_channel)
#ps.get_message()
api = Api(
config.get('CIC_CHAIN_SPEC'),
config.get('CHAIN_SPEC'),
queue=args.q,
callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
callback_task='cic_eth.callbacks.redis.redis',
@@ -145,7 +139,7 @@ def register_eth(i, u):
if __name__ == '__main__':
user_tags = {}
f = open(os.path.join(user_dir, 'tags.csv'), 'r')
f = open(os.path.join(config.get('_USERDIR'), 'tags.csv'), 'r')
while True:
r = f.readline().rstrip()
if len(r) == 0:
@@ -158,7 +152,8 @@ if __name__ == '__main__':
i = 0
j = 0
for x in os.walk(user_old_dir):
for x in os.walk(dirs['old']):
for y in x[2]:
if y[len(y)-5:] != '.json':
continue
@@ -182,7 +177,7 @@ if __name__ == '__main__':
new_address_clean = strip_0x(new_address)
filepath = os.path.join(
user_new_dir,
dirs['new'],
new_address_clean[:2].upper(),
new_address_clean[2:4].upper(),
new_address_clean.upper() + '.json',
@@ -194,17 +189,17 @@ if __name__ == '__main__':
f.write(json.dumps(o))
f.close()
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON)
meta_filepath = os.path.join(dirs['meta'], '{}.json'.format(new_address_clean.upper()))
os.symlink(os.path.realpath(filepath), meta_filepath)
phone_object = phonenumbers.parse(u.tel)
phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
meta_phone_filepath = os.path.join(dirs['phone'], 'meta', meta_phone_key)
filepath = os.path.join(
phone_dir,
dirs['phone'],
'new',
meta_phone_key[:2].upper(),
meta_phone_key[2:4].upper(),
@@ -220,11 +215,11 @@ if __name__ == '__main__':
# custom data
custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom')
custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.CUSTOM)
custom_filepath = os.path.join(dirs['custom'], 'meta', custom_key)
filepath = os.path.join(
custom_dir,
dirs['custom'],
'new',
custom_key[:2].upper(),
custom_key[2:4].upper(),

7
apps/data-seeding/cic_eth/traffic/cmd/cache.py
View File

@@ -3,6 +3,10 @@ from chainlib.jsonrpc import JSONRPCException
from eth_erc20 import ERC20
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
import logging
logg = logging.getLogger(__name__)
class ERC20Token:
@@ -46,7 +50,8 @@ class IndexCache:
try:
r = conn.do(o)
entries.append(self.parse(r, conn))
except JSONRPCException:
except JSONRPCException as e:
logg.debug('foo {}'.format(e))
return entries
i += 1

22
apps/data-seeding/cic_eth/traffic/local/account.py
View File

@@ -11,6 +11,18 @@ queue = 'cic-eth'
name = 'account'
def create_user(chain_spec, redis_host_callback, redis_port_callback, redis_db, redis_channel):
api = Api(
str(chain_spec),
queue=queue,
callback_param='{}:{}:{}:{}'.format(redis_host_callback, redis_port_callback, redis_db, redis_channel),
callback_task='cic_eth.callbacks.redis.redis',
callback_queue=queue,
)
return api.create_account(register=True)
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers creation and registration of new account through the custodial cic-eth component.
@@ -24,14 +36,6 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number):
See local.noop.do for details on parameters and return values.
"""
logg.debug('running {} {} {}'.format(__name__, token_pair, sender, recipient))
api = Api(
str(aux['chain_spec']),
queue=queue,
callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
callback_task='cic_eth.callbacks.redis.redis',
callback_queue=queue,
)
t = api.create_account(register=True)
t = create_user(aux['chain_spec'], aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel'])
return (None, t, sender_balance, )

30
apps/data-seeding/cic_eth/traffic/traffic.py
View File

@@ -21,6 +21,9 @@ import chainlib.eth.cli
import cic_eth.cli
from cic_eth.cli.chain import chain_interface
from chainlib.eth.constant import ZERO_ADDRESS
from eth_accounts_index import AccountsIndex
from erc20_faucet import Faucet
from cic_eth.api import Api
# local imports
#import common
@@ -108,6 +111,12 @@ def main():
raise NetworkError('AccountRegistry value missing from contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
logg.info('using account registry {}'.format(account_registry))
account_cache = AccountRegistryCache(chain_spec, account_registry)
faucet = registry.lookup('Faucet')
if faucet == ZERO_ADDRESS:
logg.warning('Faucet entry missing from value missing from contract registry {}. New account registrations will need external mechanism for initial token balances.'.format(config.get('CIC_REGISTRY_ADDRESS')))
else:
logg.info('using faucet {}'.format(faucet))
# Set up provisioner for common task input data
TrafficProvisioner.oracles['token'] = token_cache
@@ -124,6 +133,27 @@ def main():
syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=handler.refresh)
syncer.add_filter(handler)
# bootstrap two accounts if starting from scratch
c = AccountsIndex(chain_spec)
o = c.entry_count(account_registry)
r = conn.do(o)
logg.debug('entry count {}'.format(c.parse_entry_count(r)))
if c.parse_entry_count(r) == 0:
if faucet == ZERO_ADDRESS:
raise ValueError('No accounts exist in network and no faucet exists. It will be impossible for any created accounts to trade.')
c = Faucet(chain_spec)
o = c.token_amount(faucet)
r = conn.do(o)
if c.parse_token_amount(r) == 0:
raise ValueError('No accounts exist in network and faucet amount is set to 0. It will be impossible for any created accounts to trade.')
api = Api(str(chain_spec), queue=config.get('CELERY_QUEUE'))
api.create_account(register=True)
api.create_account(register=True)
syncer.loop(1, conn)