Clean ups.

apps/data-seeding/cic_ussd/import_balance.py

@@ -57,8 +57,8 @@ elif args.v:
 config = Config(args.c, args.env_prefix)
 config.process()
 args_override = {
-    'CIC_CHAIN_SPEC': getattr(args, 'i'),
+    'CHAIN_SPEC': getattr(args, 'i'),
-    'ETH_PROVIDER': getattr(args, 'p'),
+    'RPC_PROVIDER': getattr(args, 'p'),
     'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
     'REDIS_HOST': getattr(args, 'redis_host'),
     'REDIS_PORT': getattr(args, 'redis_port'),
@@ -90,7 +90,7 @@ signer = EIP155Signer(keystore)
 
 block_offset = -1 if args.head else args.offset
 
-chain_str = config.get('CIC_CHAIN_SPEC')
+chain_str = config.get('CHAIN_SPEC')
 chain_spec = ChainSpec.from_chain_str(chain_str)
 ImportTask.chain_spec = chain_spec
 old_chain_spec_str = args.old_chain_spec
@@ -99,16 +99,12 @@ old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str)
 MetadataTask.meta_host = config.get('META_HOST')
 MetadataTask.meta_port = config.get('META_PORT')
 
-txs_dir = os.path.join(args.import_dir, 'txs')
-os.makedirs(txs_dir, exist_ok=True)
-sys.stdout.write(f'created txs dir: {txs_dir}')
-
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 get_celery_worker_status(celery_app)
 
 
 def main():
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
     ImportTask.balance_processor = BalanceProcessor(conn,
                                                     chain_spec,
                                                     config.get('CIC_REGISTRY_ADDRESS'),

apps/data-seeding/cic_ussd/import_task.py

@@ -14,7 +14,9 @@ from celery import Task
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.tx import raw, unpack
-from cic_types.models.person import Person, generate_metadata_pointer
+from cic_types.models.person import Person, identity_tag
+from cic_types.processor import generate_metadata_pointer
+from cic_types.condiments import MetadataPointer
 from hexathon import add_0x, strip_0x
 
 # local imports
@@ -55,7 +57,7 @@ class MetadataTask(ImportTask):
 
 
 def old_address_from_phone(base_path: str, phone_number: str):
-    pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
+    pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
     phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}')
     with open(phone_idx_path, 'r') as f:
         old_address = f.read()
@@ -73,9 +75,13 @@ def generate_person_metadata(self, blockchain_address: str, phone_number: str):
     person = Person.deserialize(person_metadata)
     if not person.identities.get('evm'):
         person.identities['evm'] = {}
-    sub_chain_str = f'{self.chain_spec.common_name()}:{self.chain_spec.network_id()}'
+    chain_spec = self.chain_spec.asdict()
-    person.identities['evm'][sub_chain_str] = [add_0x(blockchain_address)]
+    arch = chain_spec.get('arch')
-    blockchain_address = strip_0x(blockchain_address)
+    fork = chain_spec.get('fork')
+    tag = identity_tag(chain_spec)
+    person.identities[arch][fork] = {
+        tag: [blockchain_address]
+    }
     file_path = os.path.join(
         self.import_dir,
         'new',
@@ -102,7 +108,7 @@ def generate_preferences_data(self, data: tuple):
     blockchain_address: str = data[0]
     preferences = data[1]
     preferences_dir = os.path.join(self.import_dir, 'preferences')
-    preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), ':cic.preferences')
+    preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES)
     preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
     filepath = os.path.join(
         preferences_dir,
@@ -137,7 +143,7 @@ def generate_ussd_data(self, blockchain_address: str, phone_number: str):
     preferred_language = random.sample(["en", "sw"], 1)[0]
     preferences = {'preferred_language': preferred_language}
     with open(ussd_data_file, file_op) as ussd_data_file:
-        ussd_data_file.write(f'{phone_number}, { 1}, {preferred_language}, {False}\n')
+        ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n')
         logg.debug(f'written ussd data for address: {blockchain_address}')
     return blockchain_address, preferences
 
@@ -163,7 +169,7 @@ def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial:
 
 @celery_app.task(bind=True, base=MetadataTask)
 def resolve_phone(self, phone_number: str):
-    identifier = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
+    identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
     url = parse.urljoin(self.meta_url(), identifier)
     logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}')
     r = request.urlopen(url)

apps/data-seeding/cic_ussd/import_users.py

@@ -17,6 +17,7 @@ from cic_types.models.person import Person
 from confini import Config
 
 # local imports
+from common.dirs import initialize_dirs
 from import_util import get_celery_worker_status
 
 default_config_dir = './config'
@@ -37,6 +38,7 @@ arg_parser.add_argument('--env-prefix',
                         dest='env_prefix',
                         type=str,
                         help='environment prefix for variables to overwrite configuration.')
+arg_parser.add_argument('-f', action='store_true', help='force clear previous state')
 arg_parser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit data seeding tasks to.')
 arg_parser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -70,21 +72,7 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug(f'config loaded from {args.c}:\n{config}')
 
-old_account_dir = os.path.join(args.import_dir, 'old')
+dirs = initialize_dirs(args.import_dir, force_reset=args.f)
-os.stat(old_account_dir)
-logg.debug(f'created old system data dir: {old_account_dir}')
-
-new_account_dir = os.path.join(args.import_dir, 'new')
-os.makedirs(new_account_dir, exist_ok=True)
-logg.debug(f'created new system data dir: {new_account_dir}')
-
-person_metadata_dir = os.path.join(args.import_dir, 'meta')
-os.makedirs(person_metadata_dir, exist_ok=True)
-logg.debug(f'created person metadata dir: {person_metadata_dir}')
-
-preferences_dir = os.path.join(args.import_dir, 'preferences')
-os.makedirs(os.path.join(preferences_dir, 'meta'), exist_ok=True)
-logg.debug(f'created preferences metadata dir: {preferences_dir}')
 
 valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
 
@@ -157,7 +145,7 @@ def register_account(person: Person):
 if __name__ == '__main__':
     i = 0
     j = 0
-    for x in os.walk(old_account_dir):
+    for x in os.walk(dirs['old']):
         for y in x[2]:
             if y[len(y) - 5:] != '.json':
                 continue

apps/data-seeding/verify.py

@@ -25,10 +25,9 @@ from chainlib.eth.gas import (
 from chainlib.eth.tx import TxFactory
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.jsonrpc import JSONRPCRequest
-from cic_types.models.person import (
+from cic_types.models.person import Person, identity_tag
-    Person,
+from cic_types.condiments import MetadataPointer
-    generate_metadata_pointer,
+from cic_types.processor import generate_metadata_pointer
-)
 from erc20_faucet import Faucet
 from eth_erc20 import ERC20
 from hexathon.parse import strip_0x, add_0x
@@ -74,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
 argparser.add_argument('-c', type=str, help='config override dir')
-argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
 argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
@@ -108,7 +107,7 @@ config.process()
 # override args
 args_override = {
         'CHAIN_SPEC': getattr(args, 'i'),
-        'ETH_PROVIDER': getattr(args, 'p'),
+        'RPC_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         }
 config.dict_override(args_override, 'cli flag')
@@ -307,7 +306,7 @@ class Verifier:
 
 
     def verify_gas(self, address, balance_token=None):
-        o = balance(address)
+        o = balance(add_0x(address))
         r = self.conn.do(o)
         logg.debug('wtf {}'.format(r))
         actual_balance = int(strip_0x(r), 16)
@@ -323,7 +322,7 @@ class Verifier:
 
 
     def verify_metadata(self, address, balance=None):
-        k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person')
+        k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), MetadataPointer.PERSON)
         url = os.path.join(config.get('_META_PROVIDER'), k)
         logg.debug('verify metadata url {}'.format(url))
         try:
@@ -367,7 +366,7 @@ class Verifier:
 
         p = Person.deserialize(o) 
 
-        k = generate_metadata_pointer(p.tel.encode('utf-8'), ':cic.phone')
+        k = generate_metadata_pointer(p.tel.encode('utf-8'), MetadataPointer.PHONE)
         url = os.path.join(config.get('_META_PROVIDER'), k)
         logg.debug('verify metadata phone url {}'.format(url))
         try:
@@ -427,7 +426,7 @@ class Verifier:
 def main():
     global chain_str, block_offset, user_dir
     
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
     gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
 
     # Get Token registry address
@@ -505,10 +504,17 @@ def main():
             u = Person.deserialize(o)
             #logg.debug('data {}'.format(u.identities['evm']))
 
-            subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
+            new_chain_spec = chain_spec.asdict()
-            new_address = u.identities['evm'][subchain_str][0]
+            arch = new_chain_spec.get('arch')
-            subchain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
+            fork = new_chain_spec.get('fork')
-            old_address = u.identities['evm'][subchain_str][0]
+            tag = identity_tag(new_chain_spec)
+            new_address = u.identities[arch][fork][tag][0]
+
+            old_chainspec = old_chain_spec.asdict()
+            arch = old_chainspec.get('arch')
+            fork = old_chainspec.get('fork')
+            tag = identity_tag(old_chainspec)
+            old_address = u.identities[arch][fork][tag][0]
             balance = 0
             try:
                 balance = balances[old_address]