Add data seeding preparation step, rehabilitation of non-custodial seeding

apps/cic-ussd/requirements.txt

@@ -4,9 +4,9 @@ billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
 cffi==1.14.6
-cic-eth[services]~=0.12.4a11
+#cic-eth[services]~=0.12.4a11
-cic-notify~=0.4.0a10
+#cic-notify~=0.4.0a10
-cic-types~=0.2.0a3
+#cic-types~=0.2.0a3
 confini>=0.3.6rc4,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6

apps/contract-migration/4_init_custodial.sh

@@ -36,7 +36,7 @@ cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer
 
 
 # Assign system writer for accounts index
->&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to writer to accounts index contract at $accounts_index_address\033[;39m"
+>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to write to accounts index contract at $accounts_index_address\033[;39m"
 r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer`
 add_pending_tx_hash $r
 

apps/contract-migration/5_data_seeding.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+. util.sh
+
+set -a
+
+. ${DEV_DATA_DIR}/env_reset
+
+WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
+
+set -e
+
+if [ ! -z $DEV_ETH_GAS_PRICE ]; then
+	gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
+	fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE"
+fi
+
+must_address "$CIC_REGISTRY_ADDRESS" "registry"
+must_eth_rpc
+
+
+accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`
+
+
+>&2 echo -e "\033[;96mEnable default wallet $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER to write to accounts index contract at $accounts_index_address\033[;39m"
+r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
+add_pending_tx_hash $r

apps/contract-migration/docker/Dockerfile

@@ -1,6 +1,6 @@
-ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+ARG DEV_DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
 
-FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
+FROM $DEV_DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
 
 WORKDIR /root
 

apps/contract-migration/run_job.sh

@@ -16,9 +16,9 @@ unset CONFINI_DIR
 
 set +a
 
-LAST_BIT_POS=4
+LAST_BIT_POS=5
-files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial)
+files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial data_seeding)
-description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine")
+description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine" "data seeding for development")
 
 >&2 echo -e "\033[;96mRUNNING configurations\033[;39m"
 source ./config.sh

apps/data-seeding/eth/import_balance.py

@@ -47,12 +47,15 @@ from erc20_faucet import Faucet
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = './config'
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+base_config_dir = os.path.join(root_dir, 'config')
+
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
 argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-c', type=str, help='config override directory')
 argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
@@ -71,21 +74,24 @@ if args.v == True:
 elif args.vv == True:
     logging.getLogger().setLevel(logging.DEBUG)
 
-config_dir = os.path.join(args.c)
+config = None
-os.makedirs(config_dir, 0o777, True)
+logg.debug('config dir {}'.format(base_config_dir))
-config = confini.Config(config_dir, args.env_prefix)
+if args.c != None:
+    config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
+else:
+    config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
+
 # override args
 args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CHAIN_SPEC': getattr(args, 'i'),
-        'ETH_PROVIDER': getattr(args, 'p'),
+        'RPC_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
 #app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
 
@@ -98,7 +104,7 @@ if args.y != None:
 signer = EIP155Signer(keystore)
 
 queue = args.q
-chain_str = config.get('CIC_CHAIN_SPEC')
+chain_str = config.get('CHAIN_SPEC')
 block_offset = 0
 if args.head:
     block_offset = -1
@@ -203,7 +209,7 @@ def progress_callback(block_number, tx_index):
 def main():
     global chain_str, block_offset, user_dir
     
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
     gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
     nonce_oracle = RPCNonceOracle(signer_address, conn)
 

apps/data-seeding/eth/import_users.py

@@ -8,6 +8,7 @@ import uuid
 import datetime
 import time
 import phonenumbers
+import shutil
 from glob import glob
 
 # external imports
@@ -23,6 +24,7 @@ from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.gas import RPCGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from cic_types.processor import generate_metadata_pointer
+from cic_types import MetadataPointer
 from eth_accounts_index.registry import AccountRegistry
 from eth_contract_registry import Registry
 from crypto_dev_signer.keystore.dict import DictKeystore
@@ -32,12 +34,15 @@ from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-default_config_dir = '/usr/local/etc/cic'
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+base_config_dir = os.path.join(root_dir, 'config')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
 argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-c', type=str, help='config override directory')
+argparser.add_argument('-f', action='store_true', help='force clear previous state')
 argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
 argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
@@ -53,8 +58,11 @@ if args.v:
 elif args.vv:
     logg.setLevel(logging.DEBUG)
 
-config_dir = args.c
+config = None
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+if args.c != None:
+    config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
+else:
+    config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
@@ -64,25 +72,26 @@ args_override = {
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)
 
-user_new_dir = os.path.join(args.user_dir, 'new')
-os.makedirs(user_new_dir)
-
-meta_dir = os.path.join(args.user_dir, 'meta')
-os.makedirs(meta_dir)
-
-custom_dir = os.path.join(args.user_dir, 'custom')
-os.makedirs(custom_dir)
-os.makedirs(os.path.join(custom_dir, 'new'))
-os.makedirs(os.path.join(custom_dir, 'meta'))
-
-phone_dir = os.path.join(args.user_dir, 'phone')
-os.makedirs(os.path.join(phone_dir, 'meta'))
-
 user_old_dir = os.path.join(args.user_dir, 'old')
-os.stat(user_old_dir)
+user_new_dir = os.path.join(args.user_dir, 'new')
-
+meta_dir = os.path.join(args.user_dir, 'meta')
+custom_dir = os.path.join(args.user_dir, 'custom')
+phone_dir = os.path.join(args.user_dir, 'phone')
 txs_dir = os.path.join(args.user_dir, 'txs')
-os.makedirs(txs_dir)
+
+try:
+    os.stat(user_old_dir)
+except FileNotFoundError:
+    sys.stderr.write('no users to import. please run create_import_users.py first\n')
+    sys.exit(1)
+
+#os.makedirs(user_new_dir)
+#os.makedirs(meta_dir)
+#os.makedirs(custom_dir)
+#os.makedirs(os.path.join(custom_dir, 'new'))
+#os.makedirs(os.path.join(custom_dir, 'meta'))
+#os.makedirs(os.path.join(phone_dir, 'meta'))
+#os.makedirs(txs_dir)
 
 user_dir = args.user_dir
 
@@ -114,7 +123,26 @@ account_registry_address = registry.parse_address_of(r)
 logg.info('using account registry {}'.format(account_registry_address))
 
 keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
-os.makedirs(keyfile_dir)
+
+result_dirs = [
+        user_new_dir,
+        meta_dir,
+        custom_dir,
+        os.path.join(custom_dir, 'new'),
+        os.path.join(custom_dir, 'meta'),
+        os.path.join(phone_dir, 'meta'),
+        txs_dir,
+        keyfile_dir,
+        ]
+if args.f:
+    for d in result_dirs:
+        try:
+            shutil.rmtree(d)
+        except FileNotFoundError:
+            pass
+for d in result_dirs:
+    os.makedirs(d)
+
 
 def register_eth(i, u):
 
@@ -190,14 +218,14 @@ if __name__ == '__main__':
             f.write(json.dumps(o))
             f.close()
 
-            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON)
             meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
             os.symlink(os.path.realpath(filepath), meta_filepath)
 
             phone_object = phonenumbers.parse(u.tel)
             phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
             logg.debug('>>>>> Using phone {}'.format(phone))
-            meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
+            meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
             meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
 
             filepath = os.path.join(
@@ -217,7 +245,7 @@ if __name__ == '__main__':
 
 
             # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM)
             custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
 
             filepath = os.path.join(

apps/data-seeding/requirements.txt

@@ -12,3 +12,4 @@ eth-erc20~=0.1.2a3
 erc20-faucet~=0.3.2a2
 psycopg2==2.8.6
 liveness~=0.0.1a7
+confini>=0.4.2rc3,<0.5.0

apps/data-seeding/verify.py

@@ -39,7 +39,8 @@ from eth_token_index import TokenUniqueSymbolIndex
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = '/usr/local/etc/cic-syncer'
+script_dir = os.path.dirname(os.path.realpath(__file__))
+base_config_dir = os.path.join(script_dir, 'config')
 
 custodial_tests = [
         'local_key',
@@ -72,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-c', type=str, help='config override dir')
 argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
@@ -96,13 +97,17 @@ if args.v == True:
 elif args.vv == True:
     logging.getLogger().setLevel(logging.DEBUG)
 
-config_dir = os.path.join(args.c)
+config = None
-os.makedirs(config_dir, 0o777, True)
+logg.debug('config dir {}'.format(base_config_dir))
-config = confini.Config(config_dir, args.env_prefix)
+if args.c != None:
+    config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
+else:
+    config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
+
 # override args
 args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         }
@@ -114,11 +119,9 @@ config.add(args.ussd_provider, '_USSD_PROVIDER', True)
 
 token_symbol = args.token_symbol
 
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 chain_str = str(chain_spec)
 old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
 old_chain_str = str(old_chain_spec)

docker-compose_local.yml

@@ -48,7 +48,7 @@ services:
       args:
           DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
           PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
-          EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
+          EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
           EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     environment:
       DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}