kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add data seeding preparation step, rehabilitation of non-custodial seeding

Browse Source
This commit is contained in:
nolash 2021-10-17 18:05:00 +02:00
parent ea3c75e755
commit 677fb346fd
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
10 changed files with 119 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/cic-ussd/requirements.txt
View File

@ -4,9 +4,9 @@ billiard==3.6.4.0
bcrypt==3.2.0
celery==4.4.7
cffi==1.14.6
cic-eth[services]~=0.12.4a11
cic-notify~=0.4.0a10
cic-types~=0.2.0a3
#cic-eth[services]~=0.12.4a11
#cic-notify~=0.4.0a10
#cic-types~=0.2.0a3
confini>=0.3.6rc4,<0.5.0
phonenumbers==8.12.12
psycopg2==2.8.6

2
apps/contract-migration/4_init_custodial.sh
View File

@ -36,7 +36,7 @@ cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer
# Assign system writer for accounts index
>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to writer to accounts index contract at $accounts_index_address\033[;39m"
>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to write to accounts index contract at $accounts_index_address\033[;39m"
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer`
add_pending_tx_hash $r

27
apps/contract-migration/5_data_seeding.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
. util.sh
set -a
. ${DEV_DATA_DIR}/env_reset
WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
set -e
if [ ! -z $DEV_ETH_GAS_PRICE ]; then
gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE"
fi
must_address "$CIC_REGISTRY_ADDRESS" "registry"
must_eth_rpc
accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`
>&2 echo -e "\033[;96mEnable default wallet $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER to write to accounts index contract at $accounts_index_address\033[;39m"
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
add_pending_tx_hash $r

4
apps/contract-migration/docker/Dockerfile
View File

@ -1,6 +1,6 @@
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
ARG DEV_DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
FROM $DEV_DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
WORKDIR /root

6
apps/contract-migration/run_job.sh
View File

@ -16,9 +16,9 @@ unset CONFINI_DIR
set +a
LAST_BIT_POS=4
files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial)
description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine")
LAST_BIT_POS=5
files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial data_seeding)
description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine" "data seeding for development")
>&2 echo -e "\033[;96mRUNNING configurations\033[;39m"
source ./config.sh

26
apps/data-seeding/eth/import_balance.py
View File

@ -47,12 +47,15 @@ from erc20_faucet import Faucet
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
config_dir = './config'
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
@ -71,21 +74,24 @@ if args.v == True:
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config = None
logg.debug('config dir {}'.format(base_config_dir))
if args.c != None:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
else:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
# override args
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CHAIN_SPEC': getattr(args, 'i'),
'RPC_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'KEYSTORE_FILE_PATH': getattr(args, 'y')
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@ -98,7 +104,7 @@ if args.y != None:
signer = EIP155Signer(keystore)
queue = args.q
chain_str = config.get('CIC_CHAIN_SPEC')
chain_str = config.get('CHAIN_SPEC')
block_offset = 0
if args.head:
block_offset = -1
@ -203,7 +209,7 @@ def progress_callback(block_number, tx_index):
def main():
global chain_str, block_offset, user_dir
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
nonce_oracle = RPCNonceOracle(signer_address, conn)

78
apps/data-seeding/eth/import_users.py
View File

@ -8,6 +8,7 @@ import uuid
import datetime
import time
import phonenumbers
import shutil
from glob import glob
# external imports
@ -23,6 +24,7 @@ from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from cic_types.processor import generate_metadata_pointer
from cic_types import MetadataPointer
from eth_accounts_index.registry import AccountRegistry
from eth_contract_registry import Registry
from crypto_dev_signer.keystore.dict import DictKeystore
@ -32,12 +34,15 @@ from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
default_config_dir = '/usr/local/etc/cic'
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('-f', action='store_true', help='force clear previous state')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
@ -53,8 +58,11 @@ if args.v:
elif args.vv:
logg.setLevel(logging.DEBUG)
config_dir = args.c
config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config = None
if args.c != None:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
else:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
args_override = {
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
@ -64,25 +72,26 @@ args_override = {
config.dict_override(args_override, 'cli')
config.add(args.user_dir, '_USERDIR', True)
user_new_dir = os.path.join(args.user_dir, 'new')
os.makedirs(user_new_dir)
meta_dir = os.path.join(args.user_dir, 'meta')
os.makedirs(meta_dir)
custom_dir = os.path.join(args.user_dir, 'custom')
os.makedirs(custom_dir)
os.makedirs(os.path.join(custom_dir, 'new'))
os.makedirs(os.path.join(custom_dir, 'meta'))
phone_dir = os.path.join(args.user_dir, 'phone')
os.makedirs(os.path.join(phone_dir, 'meta'))
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
user_new_dir = os.path.join(args.user_dir, 'new')
meta_dir = os.path.join(args.user_dir, 'meta')
custom_dir = os.path.join(args.user_dir, 'custom')
phone_dir = os.path.join(args.user_dir, 'phone')
txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
try:
os.stat(user_old_dir)
except FileNotFoundError:
sys.stderr.write('no users to import. please run create_import_users.py first\n')
sys.exit(1)
#os.makedirs(user_new_dir)
#os.makedirs(meta_dir)
#os.makedirs(custom_dir)
#os.makedirs(os.path.join(custom_dir, 'new'))
#os.makedirs(os.path.join(custom_dir, 'meta'))
#os.makedirs(os.path.join(phone_dir, 'meta'))
#os.makedirs(txs_dir)
user_dir = args.user_dir
@ -114,7 +123,26 @@ account_registry_address = registry.parse_address_of(r)
logg.info('using account registry {}'.format(account_registry_address))
keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
os.makedirs(keyfile_dir)
result_dirs = [
user_new_dir,
meta_dir,
custom_dir,
os.path.join(custom_dir, 'new'),
os.path.join(custom_dir, 'meta'),
os.path.join(phone_dir, 'meta'),
txs_dir,
keyfile_dir,
]
if args.f:
for d in result_dirs:
try:
shutil.rmtree(d)
except FileNotFoundError:
pass
for d in result_dirs:
os.makedirs(d)
def register_eth(i, u):
@ -190,14 +218,14 @@ if __name__ == '__main__':
f.write(json.dumps(o))
f.close()
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON)
meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
os.symlink(os.path.realpath(filepath), meta_filepath)
phone_object = phonenumbers.parse(u.tel)
phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
logg.debug('>>>>> Using phone {}'.format(phone))
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
filepath = os.path.join(
@ -217,7 +245,7 @@ if __name__ == '__main__':
# custom data
custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM)
custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
filepath = os.path.join(

1
apps/data-seeding/requirements.txt
View File

@ -12,3 +12,4 @@ eth-erc20~=0.1.2a3
erc20-faucet~=0.3.2a2
psycopg2==2.8.6
liveness~=0.0.1a7
confini>=0.4.2rc3,<0.5.0

21
apps/data-seeding/verify.py
View File

@ -39,7 +39,8 @@ from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
config_dir = '/usr/local/etc/cic-syncer'
script_dir = os.path.dirname(os.path.realpath(__file__))
base_config_dir = os.path.join(script_dir, 'config')
custodial_tests = [
'local_key',
@ -72,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-c', type=str, help='config override dir')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
@ -96,13 +97,17 @@ if args.v == True:
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config = None
logg.debug('config dir {}'.format(base_config_dir))
if args.c != None:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
else:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
config.process()
# override args
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
}
@ -114,11 +119,9 @@ config.add(args.ussd_provider, '_USSD_PROVIDER', True)
token_symbol = args.token_symbol
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
chain_str = str(chain_spec)
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec)

2
docker-compose_local.yml
View File

@ -48,7 +48,7 @@ services:
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}