Add data seeding preparation step, rehabilitation of non-custodial seeding

This commit is contained in:
nolash 2021-10-17 18:05:00 +02:00
parent ea3c75e755
commit 677fb346fd
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
10 changed files with 119 additions and 54 deletions

View File

@ -4,9 +4,9 @@ billiard==3.6.4.0
bcrypt==3.2.0 bcrypt==3.2.0
celery==4.4.7 celery==4.4.7
cffi==1.14.6 cffi==1.14.6
cic-eth[services]~=0.12.4a11 #cic-eth[services]~=0.12.4a11
cic-notify~=0.4.0a10 #cic-notify~=0.4.0a10
cic-types~=0.2.0a3 #cic-types~=0.2.0a3
confini>=0.3.6rc4,<0.5.0 confini>=0.3.6rc4,<0.5.0
phonenumbers==8.12.12 phonenumbers==8.12.12
psycopg2==2.8.6 psycopg2==2.8.6

View File

@ -36,7 +36,7 @@ cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer
# Assign system writer for accounts index # Assign system writer for accounts index
>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to writer to accounts index contract at $accounts_index_address\033[;39m" >&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to write to accounts index contract at $accounts_index_address\033[;39m"
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer` r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer`
add_pending_tx_hash $r add_pending_tx_hash $r

View File

@ -0,0 +1,27 @@
#!/bin/bash
. util.sh
set -a
. ${DEV_DATA_DIR}/env_reset
WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
set -e
if [ ! -z $DEV_ETH_GAS_PRICE ]; then
gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE"
fi
must_address "$CIC_REGISTRY_ADDRESS" "registry"
must_eth_rpc
accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`
>&2 echo -e "\033[;96mEnable default wallet $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER to write to accounts index contract at $accounts_index_address\033[;39m"
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
add_pending_tx_hash $r

View File

@ -1,6 +1,6 @@
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" ARG DEV_DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e FROM $DEV_DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
WORKDIR /root WORKDIR /root

View File

@ -16,9 +16,9 @@ unset CONFINI_DIR
set +a set +a
LAST_BIT_POS=4 LAST_BIT_POS=5
files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial) files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial data_seeding)
description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine") description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine" "data seeding for development")
>&2 echo -e "\033[;96mRUNNING configurations\033[;39m" >&2 echo -e "\033[;96mRUNNING configurations\033[;39m"
source ./config.sh source ./config.sh

View File

@ -47,12 +47,15 @@ from erc20_faucet import Faucet
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
config_dir = './config' script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
@ -71,21 +74,24 @@ if args.v == True:
elif args.vv == True: elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c) config = None
os.makedirs(config_dir, 0o777, True) logg.debug('config dir {}'.format(base_config_dir))
config = confini.Config(config_dir, args.env_prefix) if args.c != None:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
else:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
config.process() config.process()
# override args # override args
args_override = { args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'), 'CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'), 'RPC_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'KEYSTORE_FILE_PATH': getattr(args, 'y') 'KEYSTORE_FILE_PATH': getattr(args, 'y')
} }
config.dict_override(args_override, 'cli flag') config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL') config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) #app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@ -98,7 +104,7 @@ if args.y != None:
signer = EIP155Signer(keystore) signer = EIP155Signer(keystore)
queue = args.q queue = args.q
chain_str = config.get('CIC_CHAIN_SPEC') chain_str = config.get('CHAIN_SPEC')
block_offset = 0 block_offset = 0
if args.head: if args.head:
block_offset = -1 block_offset = -1
@ -203,7 +209,7 @@ def progress_callback(block_number, tx_index):
def main(): def main():
global chain_str, block_offset, user_dir global chain_str, block_offset, user_dir
conn = EthHTTPConnection(config.get('ETH_PROVIDER')) conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
nonce_oracle = RPCNonceOracle(signer_address, conn) nonce_oracle = RPCNonceOracle(signer_address, conn)

View File

@ -8,6 +8,7 @@ import uuid
import datetime import datetime
import time import time
import phonenumbers import phonenumbers
import shutil
from glob import glob from glob import glob
# external imports # external imports
@ -23,6 +24,7 @@ from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.gas import RPCGasOracle from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
from cic_types import MetadataPointer
from eth_accounts_index.registry import AccountRegistry from eth_accounts_index.registry import AccountRegistry
from eth_contract_registry import Registry from eth_contract_registry import Registry
from crypto_dev_signer.keystore.dict import DictKeystore from crypto_dev_signer.keystore.dict import DictKeystore
@ -32,12 +34,15 @@ from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
default_config_dir = '/usr/local/etc/cic' script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
base_config_dir = os.path.join(root_dir, 'config')
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') argparser.add_argument('-c', type=str, help='config override directory')
argparser.add_argument('-f', action='store_true', help='force clear previous state')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address') argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
@ -53,8 +58,11 @@ if args.v:
elif args.vv: elif args.vv:
logg.setLevel(logging.DEBUG) logg.setLevel(logging.DEBUG)
config_dir = args.c config = None
config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) if args.c != None:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
else:
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
config.process() config.process()
args_override = { args_override = {
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
@ -64,25 +72,26 @@ args_override = {
config.dict_override(args_override, 'cli') config.dict_override(args_override, 'cli')
config.add(args.user_dir, '_USERDIR', True) config.add(args.user_dir, '_USERDIR', True)
user_new_dir = os.path.join(args.user_dir, 'new')
os.makedirs(user_new_dir)
meta_dir = os.path.join(args.user_dir, 'meta')
os.makedirs(meta_dir)
custom_dir = os.path.join(args.user_dir, 'custom')
os.makedirs(custom_dir)
os.makedirs(os.path.join(custom_dir, 'new'))
os.makedirs(os.path.join(custom_dir, 'meta'))
phone_dir = os.path.join(args.user_dir, 'phone')
os.makedirs(os.path.join(phone_dir, 'meta'))
user_old_dir = os.path.join(args.user_dir, 'old') user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir) user_new_dir = os.path.join(args.user_dir, 'new')
meta_dir = os.path.join(args.user_dir, 'meta')
custom_dir = os.path.join(args.user_dir, 'custom')
phone_dir = os.path.join(args.user_dir, 'phone')
txs_dir = os.path.join(args.user_dir, 'txs') txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
try:
os.stat(user_old_dir)
except FileNotFoundError:
sys.stderr.write('no users to import. please run create_import_users.py first\n')
sys.exit(1)
#os.makedirs(user_new_dir)
#os.makedirs(meta_dir)
#os.makedirs(custom_dir)
#os.makedirs(os.path.join(custom_dir, 'new'))
#os.makedirs(os.path.join(custom_dir, 'meta'))
#os.makedirs(os.path.join(phone_dir, 'meta'))
#os.makedirs(txs_dir)
user_dir = args.user_dir user_dir = args.user_dir
@ -114,7 +123,26 @@ account_registry_address = registry.parse_address_of(r)
logg.info('using account registry {}'.format(account_registry_address)) logg.info('using account registry {}'.format(account_registry_address))
keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore') keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
os.makedirs(keyfile_dir)
result_dirs = [
user_new_dir,
meta_dir,
custom_dir,
os.path.join(custom_dir, 'new'),
os.path.join(custom_dir, 'meta'),
os.path.join(phone_dir, 'meta'),
txs_dir,
keyfile_dir,
]
if args.f:
for d in result_dirs:
try:
shutil.rmtree(d)
except FileNotFoundError:
pass
for d in result_dirs:
os.makedirs(d)
def register_eth(i, u): def register_eth(i, u):
@ -190,14 +218,14 @@ if __name__ == '__main__':
f.write(json.dumps(o)) f.write(json.dumps(o))
f.close() f.close()
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person') meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON)
meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
os.symlink(os.path.realpath(filepath), meta_filepath) os.symlink(os.path.realpath(filepath), meta_filepath)
phone_object = phonenumbers.parse(u.tel) phone_object = phonenumbers.parse(u.tel)
phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
logg.debug('>>>>> Using phone {}'.format(phone)) logg.debug('>>>>> Using phone {}'.format(phone))
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone') meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key) meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
filepath = os.path.join( filepath = os.path.join(
@ -217,7 +245,7 @@ if __name__ == '__main__':
# custom data # custom data
custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom') custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM)
custom_filepath = os.path.join(custom_dir, 'meta', custom_key) custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
filepath = os.path.join( filepath = os.path.join(

View File

@ -12,3 +12,4 @@ eth-erc20~=0.1.2a3
erc20-faucet~=0.3.2a2 erc20-faucet~=0.3.2a2
psycopg2==2.8.6 psycopg2==2.8.6
liveness~=0.0.1a7 liveness~=0.0.1a7
confini>=0.4.2rc3,<0.5.0

View File

@ -39,7 +39,8 @@ from eth_token_index import TokenUniqueSymbolIndex
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
config_dir = '/usr/local/etc/cic-syncer' script_dir = os.path.dirname(os.path.realpath(__file__))
base_config_dir = os.path.join(script_dir, 'config')
custodial_tests = [ custodial_tests = [
'local_key', 'local_key',
@ -72,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') argparser.add_argument('-c', type=str, help='config override dir')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
@ -96,13 +97,17 @@ if args.v == True:
elif args.vv == True: elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c) config = None
os.makedirs(config_dir, 0o777, True) logg.debug('config dir {}'.format(base_config_dir))
config = confini.Config(config_dir, args.env_prefix) if args.c != None:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
else:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
config.process() config.process()
# override args # override args
args_override = { args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'), 'CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'), 'ETH_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
} }
@ -114,11 +119,9 @@ config.add(args.ussd_provider, '_USSD_PROVIDER', True)
token_symbol = args.token_symbol token_symbol = args.token_symbol
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
chain_str = str(chain_spec) chain_str = str(chain_spec)
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec) old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec) old_chain_str = str(old_chain_spec)

View File

@ -48,7 +48,7 @@ services:
args: args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment: environment:
DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}