diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 8fb487a3..6f7f25e0 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -57,8 +57,8 @@ elif args.v: config = Config(args.c, args.env_prefix) config.process() args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), + 'CHAIN_SPEC': getattr(args, 'i'), + 'RPC_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'REDIS_HOST': getattr(args, 'redis_host'), 'REDIS_PORT': getattr(args, 'redis_port'), @@ -90,7 +90,7 @@ signer = EIP155Signer(keystore) block_offset = -1 if args.head else args.offset -chain_str = config.get('CIC_CHAIN_SPEC') +chain_str = config.get('CHAIN_SPEC') chain_spec = ChainSpec.from_chain_str(chain_str) ImportTask.chain_spec = chain_spec old_chain_spec_str = args.old_chain_spec @@ -99,16 +99,12 @@ old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str) MetadataTask.meta_host = config.get('META_HOST') MetadataTask.meta_port = config.get('META_PORT') -txs_dir = os.path.join(args.import_dir, 'txs') -os.makedirs(txs_dir, exist_ok=True) -sys.stdout.write(f'created txs dir: {txs_dir}') - celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) get_celery_worker_status(celery_app) def main(): - conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + conn = EthHTTPConnection(config.get('RPC_PROVIDER')) ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), diff --git a/apps/data-seeding/cic_ussd/import_task.py b/apps/data-seeding/cic_ussd/import_task.py index 00e72f1a..aa702031 100644 --- a/apps/data-seeding/cic_ussd/import_task.py +++ b/apps/data-seeding/cic_ussd/import_task.py @@ -14,7 +14,9 @@ from celery import Task from chainlib.chain import ChainSpec from chainlib.eth.address import to_checksum_address from chainlib.eth.tx import raw, unpack -from cic_types.models.person import Person, generate_metadata_pointer +from cic_types.models.person import Person, identity_tag +from cic_types.processor import generate_metadata_pointer +from cic_types.condiments import MetadataPointer from hexathon import add_0x, strip_0x # local imports @@ -55,7 +57,7 @@ class MetadataTask(ImportTask): def old_address_from_phone(base_path: str, phone_number: str): - pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone') + pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE) phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}') with open(phone_idx_path, 'r') as f: old_address = f.read() @@ -73,9 +75,13 @@ def generate_person_metadata(self, blockchain_address: str, phone_number: str): person = Person.deserialize(person_metadata) if not person.identities.get('evm'): person.identities['evm'] = {} - sub_chain_str = f'{self.chain_spec.common_name()}:{self.chain_spec.network_id()}' - person.identities['evm'][sub_chain_str] = [add_0x(blockchain_address)] - blockchain_address = strip_0x(blockchain_address) + chain_spec = self.chain_spec.asdict() + arch = chain_spec.get('arch') + fork = chain_spec.get('fork') + tag = identity_tag(chain_spec) + person.identities[arch][fork] = { + tag: [blockchain_address] + } file_path = os.path.join( self.import_dir, 'new', @@ -102,7 +108,7 @@ def generate_preferences_data(self, data: tuple): blockchain_address: str = data[0] preferences = data[1] preferences_dir = os.path.join(self.import_dir, 'preferences') - preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), ':cic.preferences') + preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES) preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key) filepath = os.path.join( preferences_dir, @@ -137,7 +143,7 @@ def generate_ussd_data(self, blockchain_address: str, phone_number: str): preferred_language = random.sample(["en", "sw"], 1)[0] preferences = {'preferred_language': preferred_language} with open(ussd_data_file, file_op) as ussd_data_file: - ussd_data_file.write(f'{phone_number}, { 1}, {preferred_language}, {False}\n') + ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n') logg.debug(f'written ussd data for address: {blockchain_address}') return blockchain_address, preferences @@ -163,7 +169,7 @@ def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial: @celery_app.task(bind=True, base=MetadataTask) def resolve_phone(self, phone_number: str): - identifier = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone') + identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE) url = parse.urljoin(self.meta_url(), identifier) logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}') r = request.urlopen(url) diff --git a/apps/data-seeding/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py index 72013737..a5ccc286 100644 --- a/apps/data-seeding/cic_ussd/import_users.py +++ b/apps/data-seeding/cic_ussd/import_users.py @@ -17,6 +17,7 @@ from cic_types.models.person import Person from confini import Config # local imports +from common.dirs import initialize_dirs from import_util import get_celery_worker_status default_config_dir = './config' @@ -37,6 +38,7 @@ arg_parser.add_argument('--env-prefix', dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration.') +arg_parser.add_argument('-f', action='store_true', help='force clear previous state') arg_parser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit data seeding tasks to.') arg_parser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') @@ -70,21 +72,7 @@ config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug(f'config loaded from {args.c}:\n{config}') -old_account_dir = os.path.join(args.import_dir, 'old') -os.stat(old_account_dir) -logg.debug(f'created old system data dir: {old_account_dir}') - -new_account_dir = os.path.join(args.import_dir, 'new') -os.makedirs(new_account_dir, exist_ok=True) -logg.debug(f'created new system data dir: {new_account_dir}') - -person_metadata_dir = os.path.join(args.import_dir, 'meta') -os.makedirs(person_metadata_dir, exist_ok=True) -logg.debug(f'created person metadata dir: {person_metadata_dir}') - -preferences_dir = os.path.join(args.import_dir, 'preferences') -os.makedirs(os.path.join(preferences_dir, 'meta'), exist_ok=True) -logg.debug(f'created preferences metadata dir: {preferences_dir}') +dirs = initialize_dirs(args.import_dir, force_reset=args.f) valid_service_codes = config.get('USSD_SERVICE_CODE').split(",") @@ -157,7 +145,7 @@ def register_account(person: Person): if __name__ == '__main__': i = 0 j = 0 - for x in os.walk(old_account_dir): + for x in os.walk(dirs['old']): for y in x[2]: if y[len(y) - 5:] != '.json': continue diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 56f6e7d0..fd59df45 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -25,10 +25,9 @@ from chainlib.eth.gas import ( from chainlib.eth.tx import TxFactory from chainlib.hash import keccak256_string_to_hex from chainlib.jsonrpc import JSONRPCRequest -from cic_types.models.person import ( - Person, - generate_metadata_pointer, -) +from cic_types.models.person import Person, identity_tag +from cic_types.condiments import MetadataPointer +from cic_types.processor import generate_metadata_pointer from erc20_faucet import Faucet from eth_erc20 import ERC20 from hexathon.parse import strip_0x, add_0x @@ -74,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-c', type=str, help='config override dir') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') @@ -108,7 +107,7 @@ config.process() # override args args_override = { 'CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), + 'RPC_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), } config.dict_override(args_override, 'cli flag') @@ -307,7 +306,7 @@ class Verifier: def verify_gas(self, address, balance_token=None): - o = balance(address) + o = balance(add_0x(address)) r = self.conn.do(o) logg.debug('wtf {}'.format(r)) actual_balance = int(strip_0x(r), 16) @@ -323,7 +322,7 @@ class Verifier: def verify_metadata(self, address, balance=None): - k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person') + k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), MetadataPointer.PERSON) url = os.path.join(config.get('_META_PROVIDER'), k) logg.debug('verify metadata url {}'.format(url)) try: @@ -367,7 +366,7 @@ class Verifier: p = Person.deserialize(o) - k = generate_metadata_pointer(p.tel.encode('utf-8'), ':cic.phone') + k = generate_metadata_pointer(p.tel.encode('utf-8'), MetadataPointer.PHONE) url = os.path.join(config.get('_META_PROVIDER'), k) logg.debug('verify metadata phone url {}'.format(url)) try: @@ -427,7 +426,7 @@ class Verifier: def main(): global chain_str, block_offset, user_dir - conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + conn = EthHTTPConnection(config.get('RPC_PROVIDER')) gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) # Get Token registry address @@ -505,10 +504,17 @@ def main(): u = Person.deserialize(o) #logg.debug('data {}'.format(u.identities['evm'])) - subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) - new_address = u.identities['evm'][subchain_str][0] - subchain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) - old_address = u.identities['evm'][subchain_str][0] + new_chain_spec = chain_spec.asdict() + arch = new_chain_spec.get('arch') + fork = new_chain_spec.get('fork') + tag = identity_tag(new_chain_spec) + new_address = u.identities[arch][fork][tag][0] + + old_chainspec = old_chain_spec.asdict() + arch = old_chainspec.get('arch') + fork = old_chainspec.get('fork') + tag = identity_tag(old_chainspec) + old_address = u.identities[arch][fork][tag][0] balance = 0 try: balance = balances[old_address]