diff --git a/apps/data-seeding/README.md b/apps/data-seeding/README.md index 02009a56..682af50c 100644 --- a/apps/data-seeding/README.md +++ b/apps/data-seeding/README.md @@ -136,13 +136,13 @@ First, make a note of the **block height** before running anything: To import, run to _completion_: -`python eth/import_users.py -v -c config -p -r -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` +`python eth/import_users.py -v -p -r -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` After the script completes, keystore files for all generated accouts will be found in `/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one). Then run: -`python eth/import_balance.py -v -c config -r -p --token-symbol --offset -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` +`python eth/import_balance.py -v -r -p --token-symbol --offset -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` diff --git a/apps/data-seeding/cic_eth/import_users.py b/apps/data-seeding/cic_eth/import_users.py index 8faa152c..1322c638 100644 --- a/apps/data-seeding/cic_eth/import_users.py +++ b/apps/data-seeding/cic_eth/import_users.py @@ -10,7 +10,7 @@ import time import phonenumbers from glob import glob -# third-party imports +# external imports import redis import confini import celery @@ -23,15 +23,23 @@ from cic_types.models.person import Person from cic_eth.api.api_task import Api from chainlib.chain import ChainSpec from cic_types.processor import generate_metadata_pointer +from cic_types import MetadataPointer + +# local imports +from common.dirs import initialize_dirs + logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_config_dir = '/usr/local/etc/cic' +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) +base_config_dir = os.path.join(root_dir, 'config') argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') +argparser.add_argument('-c', type=str, help='config override directory') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') +argparser.add_argument('-f', action='store_true', help='force clear previous state') argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') @@ -52,16 +60,21 @@ if args.v: elif args.vv: logg.setLevel(logging.DEBUG) -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) +config = None +if args.c != None: + config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c) +else: + config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX')) config.process() args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'CHAIN_SPEC': getattr(args, 'i'), 'REDIS_HOST': getattr(args, 'redis_host'), 'REDIS_PORT': getattr(args, 'redis_port'), 'REDIS_DB': getattr(args, 'redis_db'), } config.dict_override(args_override, 'cli') +config.add(args.user_dir, '_USERDIR', True) + celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) redis_host = config.get('REDIS_HOST') @@ -71,44 +84,25 @@ r = redis.Redis(redis_host, redis_port, redis_db) ps = r.pubsub() -user_new_dir = os.path.join(args.user_dir, 'new') -os.makedirs(user_new_dir) - -meta_dir = os.path.join(args.user_dir, 'meta') -os.makedirs(meta_dir) - -custom_dir = os.path.join(args.user_dir, 'custom') -os.makedirs(custom_dir) -os.makedirs(os.path.join(custom_dir, 'new')) -os.makedirs(os.path.join(custom_dir, 'meta')) - -phone_dir = os.path.join(args.user_dir, 'phone') -os.makedirs(os.path.join(phone_dir, 'meta')) - -user_old_dir = os.path.join(args.user_dir, 'old') -os.stat(user_old_dir) - -txs_dir = os.path.join(args.user_dir, 'txs') -os.makedirs(txs_dir) - -user_dir = args.user_dir old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec) old_chain_str = str(old_chain_spec) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) chain_str = str(chain_spec) batch_size = args.batch_size batch_delay = args.batch_delay +dirs = initialize_dirs(config.get('_USERDIR'), force_reset=args.f) + def register_eth(i, u): redis_channel = str(uuid.uuid4()) ps.subscribe(redis_channel) #ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), + config.get('CHAIN_SPEC'), queue=args.q, callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), callback_task='cic_eth.callbacks.redis.redis', @@ -145,7 +139,7 @@ def register_eth(i, u): if __name__ == '__main__': user_tags = {} - f = open(os.path.join(user_dir, 'tags.csv'), 'r') + f = open(os.path.join(config.get('_USERDIR'), 'tags.csv'), 'r') while True: r = f.readline().rstrip() if len(r) == 0: @@ -158,7 +152,8 @@ if __name__ == '__main__': i = 0 j = 0 - for x in os.walk(user_old_dir): + + for x in os.walk(dirs['old']): for y in x[2]: if y[len(y)-5:] != '.json': continue @@ -182,7 +177,7 @@ if __name__ == '__main__': new_address_clean = strip_0x(new_address) filepath = os.path.join( - user_new_dir, + dirs['new'], new_address_clean[:2].upper(), new_address_clean[2:4].upper(), new_address_clean.upper() + '.json', @@ -194,17 +189,17 @@ if __name__ == '__main__': f.write(json.dumps(o)) f.close() - meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person') - meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) + meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON) + meta_filepath = os.path.join(dirs['meta'], '{}.json'.format(new_address_clean.upper())) os.symlink(os.path.realpath(filepath), meta_filepath) phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) - meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone') - meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key) + meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) + meta_phone_filepath = os.path.join(dirs['phone'], 'meta', meta_phone_key) filepath = os.path.join( - phone_dir, + dirs['phone'], 'new', meta_phone_key[:2].upper(), meta_phone_key[2:4].upper(), @@ -220,11 +215,11 @@ if __name__ == '__main__': # custom data - custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom') - custom_filepath = os.path.join(custom_dir, 'meta', custom_key) + custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.CUSTOM) + custom_filepath = os.path.join(dirs['custom'], 'meta', custom_key) filepath = os.path.join( - custom_dir, + dirs['custom'], 'new', custom_key[:2].upper(), custom_key[2:4].upper(), diff --git a/apps/data-seeding/common/dirs.py b/apps/data-seeding/common/dirs.py new file mode 100644 index 00000000..8fa19c9d --- /dev/null +++ b/apps/data-seeding/common/dirs.py @@ -0,0 +1,41 @@ +# standard imports +import os +import shutil +import sys + + +def initialize_dirs(user_dir, force_reset=False): + + dirs = {} + + dirs['old'] = os.path.join(user_dir, 'old') + dirs['new'] = os.path.join(user_dir, 'new') + dirs['meta'] = os.path.join(user_dir, 'meta') + dirs['custom'] = os.path.join(user_dir, 'custom') + dirs['phone'] = os.path.join(user_dir, 'phone') + dirs['txs'] = os.path.join(user_dir, 'txs') + dirs['keyfile'] = os.path.join(user_dir, 'keystore') + dirs['custom_new'] = os.path.join(dirs['custom'], 'new') + dirs['custom_meta'] = os.path.join(dirs['custom'], 'meta') + dirs['phone_meta'] = os.path.join(dirs['phone'], 'meta') + + try: + os.stat(dirs['old']) + except FileNotFoundError: + sys.stderr.write('no users to import. please run create_import_users.py first\n') + sys.exit(1) + + if force_reset: + for d in dirs.keys(): + if d == 'old': + continue + try: + shutil.rmtree(dirs[d]) + except FileNotFoundError: + pass + for d in dirs.keys(): + if d == 'old': + continue + os.makedirs(dirs[d]) + + return dirs diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index 2cc67caa..c46b3c04 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -31,6 +31,10 @@ from crypto_dev_signer.keystore.dict import DictKeystore from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict +# local imports +from common.dirs import initialize_dirs + + logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -72,28 +76,7 @@ args_override = { config.dict_override(args_override, 'cli') config.add(args.user_dir, '_USERDIR', True) -user_old_dir = os.path.join(args.user_dir, 'old') -user_new_dir = os.path.join(args.user_dir, 'new') -meta_dir = os.path.join(args.user_dir, 'meta') -custom_dir = os.path.join(args.user_dir, 'custom') -phone_dir = os.path.join(args.user_dir, 'phone') -txs_dir = os.path.join(args.user_dir, 'txs') - -try: - os.stat(user_old_dir) -except FileNotFoundError: - sys.stderr.write('no users to import. please run create_import_users.py first\n') - sys.exit(1) - -#os.makedirs(user_new_dir) -#os.makedirs(meta_dir) -#os.makedirs(custom_dir) -#os.makedirs(os.path.join(custom_dir, 'new')) -#os.makedirs(os.path.join(custom_dir, 'meta')) -#os.makedirs(os.path.join(phone_dir, 'meta')) -#os.makedirs(txs_dir) - -user_dir = args.user_dir +#user_dir = args.user_dir chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_str = str(chain_spec) @@ -122,26 +105,7 @@ r = rpc.do(o) account_registry_address = registry.parse_address_of(r) logg.info('using account registry {}'.format(account_registry_address)) -keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore') - -result_dirs = [ - user_new_dir, - meta_dir, - custom_dir, - os.path.join(custom_dir, 'new'), - os.path.join(custom_dir, 'meta'), - os.path.join(phone_dir, 'meta'), - txs_dir, - keyfile_dir, - ] -if args.f: - for d in result_dirs: - try: - shutil.rmtree(d) - except FileNotFoundError: - pass -for d in result_dirs: - os.makedirs(d) +dirs = initialize_dirs(config.get('_USERDIR'), force_reset=args.f) def register_eth(i, u): @@ -157,7 +121,7 @@ def register_eth(i, u): pk = keystore.get(address) keyfile_content = to_keyfile_dict(pk, 'foo') - keyfile_path = os.path.join(keyfile_dir, '{}.json'.format(address)) + keyfile_path = os.path.join(dirs['keyfile'], '{}.json'.format(address)) f = open(keyfile_path, 'w') json.dump(keyfile_content, f) f.close() @@ -170,7 +134,7 @@ def register_eth(i, u): if __name__ == '__main__': user_tags = {} - f = open(os.path.join(user_dir, 'tags.csv'), 'r') + f = open(os.path.join(config.get('_USERDIR'), 'tags.csv'), 'r') while True: r = f.readline().rstrip() if len(r) == 0: @@ -182,7 +146,7 @@ if __name__ == '__main__': i = 0 j = 0 - for x in os.walk(user_old_dir): + for x in os.walk(dirs['old']): for y in x[2]: if y[len(y)-5:] != '.json': continue @@ -206,7 +170,7 @@ if __name__ == '__main__': new_address_clean = strip_0x(new_address) filepath = os.path.join( - user_new_dir, + dirs['new'], new_address_clean[:2].upper(), new_address_clean[2:4].upper(), new_address_clean.upper() + '.json', @@ -219,17 +183,17 @@ if __name__ == '__main__': f.close() meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON) - meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) + meta_filepath = os.path.join(dirs['meta'], '{}.json'.format(new_address_clean.upper())) os.symlink(os.path.realpath(filepath), meta_filepath) phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) logg.debug('>>>>> Using phone {}'.format(phone)) meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) - meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key) + meta_phone_filepath = os.path.join(dirs['phone'], 'meta', meta_phone_key) filepath = os.path.join( - phone_dir, + dirs['phone'], 'new', meta_phone_key[:2].upper(), meta_phone_key[2:4].upper(), @@ -246,10 +210,10 @@ if __name__ == '__main__': # custom data custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM) - custom_filepath = os.path.join(custom_dir, 'meta', custom_key) + custom_filepath = os.path.join(dirs['custom'], 'meta', custom_key) filepath = os.path.join( - custom_dir, + dirs['custom'], 'new', custom_key[:2].upper(), custom_key[2:4].upper(),