diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 512ec24d..8d697075 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -4,9 +4,9 @@ billiard==3.6.4.0 bcrypt==3.2.0 celery==4.4.7 cffi==1.14.6 -cic-eth[services]~=0.12.4a11 -cic-notify~=0.4.0a10 -cic-types~=0.2.0a3 +#cic-eth[services]~=0.12.4a11 +#cic-notify~=0.4.0a10 +#cic-types~=0.2.0a3 confini>=0.3.6rc4,<0.5.0 phonenumbers==8.12.12 psycopg2==2.8.6 diff --git a/apps/contract-migration/4_init_custodial.sh b/apps/contract-migration/4_init_custodial.sh index 96f74c26..ad003f17 100644 --- a/apps/contract-migration/4_init_custodial.sh +++ b/apps/contract-migration/4_init_custodial.sh @@ -36,7 +36,7 @@ cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer # Assign system writer for accounts index ->&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to writer to accounts index contract at $accounts_index_address\033[;39m" +>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to write to accounts index contract at $accounts_index_address\033[;39m" r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer` add_pending_tx_hash $r diff --git a/apps/contract-migration/5_data_seeding.sh b/apps/contract-migration/5_data_seeding.sh new file mode 100644 index 00000000..e465743e --- /dev/null +++ b/apps/contract-migration/5_data_seeding.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +. util.sh + +set -a + +. ${DEV_DATA_DIR}/env_reset + +WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60} + +set -e + +if [ ! -z $DEV_ETH_GAS_PRICE ]; then + gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE" + fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE" +fi + +must_address "$CIC_REGISTRY_ADDRESS" "registry" +must_eth_rpc + + +accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry` + + +>&2 echo -e "\033[;96mEnable default wallet $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER to write to accounts index contract at $accounts_index_address\033[;39m" +r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER` +add_pending_tx_hash $r diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 65f522f2..112546bc 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -1,6 +1,6 @@ -ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" +ARG DEV_DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" -FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e +FROM $DEV_DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e WORKDIR /root diff --git a/apps/contract-migration/run_job.sh b/apps/contract-migration/run_job.sh index dfd178c0..adeca52a 100644 --- a/apps/contract-migration/run_job.sh +++ b/apps/contract-migration/run_job.sh @@ -16,9 +16,9 @@ unset CONFINI_DIR set +a -LAST_BIT_POS=4 -files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial) -description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine") +LAST_BIT_POS=5 +files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial data_seeding) +description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine" "data seeding for development") >&2 echo -e "\033[;96mRUNNING configurations\033[;39m" source ./config.sh diff --git a/apps/data-seeding/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py index 55a08c7e..1a9080a9 100644 --- a/apps/data-seeding/eth/import_balance.py +++ b/apps/data-seeding/eth/import_balance.py @@ -47,12 +47,15 @@ from erc20_faucet import Faucet logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = './config' +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) +base_config_dir = os.path.join(root_dir, 'config') + argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') +argparser.add_argument('-c', type=str, help='config override directory') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') @@ -71,21 +74,24 @@ if args.v == True: elif args.vv == True: logging.getLogger().setLevel(logging.DEBUG) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) +config = None +logg.debug('config dir {}'.format(base_config_dir)) +if args.c != None: + config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c) +else: + config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX')) config.process() + # override args args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), + 'CHAIN_SPEC': getattr(args, 'i'), + 'RPC_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) #app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) @@ -98,7 +104,7 @@ if args.y != None: signer = EIP155Signer(keystore) queue = args.q -chain_str = config.get('CIC_CHAIN_SPEC') +chain_str = config.get('CHAIN_SPEC') block_offset = 0 if args.head: block_offset = -1 @@ -203,7 +209,7 @@ def progress_callback(block_number, tx_index): def main(): global chain_str, block_offset, user_dir - conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + conn = EthHTTPConnection(config.get('RPC_PROVIDER')) gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) nonce_oracle = RPCNonceOracle(signer_address, conn) diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index be04b2f2..2cc67caa 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -8,6 +8,7 @@ import uuid import datetime import time import phonenumbers +import shutil from glob import glob # external imports @@ -23,6 +24,7 @@ from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.gas import RPCGasOracle from chainlib.eth.nonce import RPCNonceOracle from cic_types.processor import generate_metadata_pointer +from cic_types import MetadataPointer from eth_accounts_index.registry import AccountRegistry from eth_contract_registry import Registry from crypto_dev_signer.keystore.dict import DictKeystore @@ -32,12 +34,15 @@ from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_config_dir = '/usr/local/etc/cic' +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) +base_config_dir = os.path.join(root_dir, 'config') argparser = argparse.ArgumentParser() argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') +argparser.add_argument('-c', type=str, help='config override directory') +argparser.add_argument('-f', action='store_true', help='force clear previous state') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address') @@ -53,8 +58,11 @@ if args.v: elif args.vv: logg.setLevel(logging.DEBUG) -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) +config = None +if args.c != None: + config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c) +else: + config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() args_override = { 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), @@ -64,25 +72,26 @@ args_override = { config.dict_override(args_override, 'cli') config.add(args.user_dir, '_USERDIR', True) -user_new_dir = os.path.join(args.user_dir, 'new') -os.makedirs(user_new_dir) - -meta_dir = os.path.join(args.user_dir, 'meta') -os.makedirs(meta_dir) - -custom_dir = os.path.join(args.user_dir, 'custom') -os.makedirs(custom_dir) -os.makedirs(os.path.join(custom_dir, 'new')) -os.makedirs(os.path.join(custom_dir, 'meta')) - -phone_dir = os.path.join(args.user_dir, 'phone') -os.makedirs(os.path.join(phone_dir, 'meta')) - user_old_dir = os.path.join(args.user_dir, 'old') -os.stat(user_old_dir) - +user_new_dir = os.path.join(args.user_dir, 'new') +meta_dir = os.path.join(args.user_dir, 'meta') +custom_dir = os.path.join(args.user_dir, 'custom') +phone_dir = os.path.join(args.user_dir, 'phone') txs_dir = os.path.join(args.user_dir, 'txs') -os.makedirs(txs_dir) + +try: + os.stat(user_old_dir) +except FileNotFoundError: + sys.stderr.write('no users to import. please run create_import_users.py first\n') + sys.exit(1) + +#os.makedirs(user_new_dir) +#os.makedirs(meta_dir) +#os.makedirs(custom_dir) +#os.makedirs(os.path.join(custom_dir, 'new')) +#os.makedirs(os.path.join(custom_dir, 'meta')) +#os.makedirs(os.path.join(phone_dir, 'meta')) +#os.makedirs(txs_dir) user_dir = args.user_dir @@ -114,7 +123,26 @@ account_registry_address = registry.parse_address_of(r) logg.info('using account registry {}'.format(account_registry_address)) keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore') -os.makedirs(keyfile_dir) + +result_dirs = [ + user_new_dir, + meta_dir, + custom_dir, + os.path.join(custom_dir, 'new'), + os.path.join(custom_dir, 'meta'), + os.path.join(phone_dir, 'meta'), + txs_dir, + keyfile_dir, + ] +if args.f: + for d in result_dirs: + try: + shutil.rmtree(d) + except FileNotFoundError: + pass +for d in result_dirs: + os.makedirs(d) + def register_eth(i, u): @@ -190,14 +218,14 @@ if __name__ == '__main__': f.write(json.dumps(o)) f.close() - meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person') + meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON) meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) os.symlink(os.path.realpath(filepath), meta_filepath) phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) logg.debug('>>>>> Using phone {}'.format(phone)) - meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone') + meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key) filepath = os.path.join( @@ -217,7 +245,7 @@ if __name__ == '__main__': # custom data - custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom') + custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM) custom_filepath = os.path.join(custom_dir, 'meta', custom_key) filepath = os.path.join( diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 962cefdf..f813c3d0 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -12,3 +12,4 @@ eth-erc20~=0.1.2a3 erc20-faucet~=0.3.2a2 psycopg2==2.8.6 liveness~=0.0.1a7 +confini>=0.4.2rc3,<0.5.0 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 216b26ef..56f6e7d0 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -39,7 +39,8 @@ from eth_token_index import TokenUniqueSymbolIndex logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = '/usr/local/etc/cic-syncer' +script_dir = os.path.dirname(os.path.realpath(__file__)) +base_config_dir = os.path.join(script_dir, 'config') custodial_tests = [ 'local_key', @@ -72,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') +argparser.add_argument('-c', type=str, help='config override dir') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') @@ -96,13 +97,17 @@ if args.v == True: elif args.vv == True: logging.getLogger().setLevel(logging.DEBUG) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) +config = None +logg.debug('config dir {}'.format(base_config_dir)) +if args.c != None: + config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c) +else: + config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX')) config.process() + # override args args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), } @@ -114,11 +119,9 @@ config.add(args.ussd_provider, '_USSD_PROVIDER', True) token_symbol = args.token_symbol -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) - celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) chain_str = str(chain_spec) old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec) old_chain_str = str(old_chain_spec) diff --git a/docker-compose_local.yml b/docker-compose_local.yml index d3da1c19..06d302c1 100644 --- a/docker-compose_local.yml +++ b/docker-compose_local.yml @@ -48,7 +48,7 @@ services: args: DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} - EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}