feat: Split and improve contract migration steps

apps/data-seeding/cic_ussd/import_balance.py

@@ -57,8 +57,8 @@ elif args.v:
 config = Config(args.c, args.env_prefix)
 config.process()
 args_override = {
-    'CIC_CHAIN_SPEC': getattr(args, 'i'),
-    'ETH_PROVIDER': getattr(args, 'p'),
+    'CHAIN_SPEC': getattr(args, 'i'),
+    'RPC_PROVIDER': getattr(args, 'p'),
     'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
     'REDIS_HOST': getattr(args, 'redis_host'),
     'REDIS_PORT': getattr(args, 'redis_port'),
@@ -90,7 +90,7 @@ signer = EIP155Signer(keystore)
 
 block_offset = -1 if args.head else args.offset
 
-chain_str = config.get('CIC_CHAIN_SPEC')
+chain_str = config.get('CHAIN_SPEC')
 chain_spec = ChainSpec.from_chain_str(chain_str)
 ImportTask.chain_spec = chain_spec
 old_chain_spec_str = args.old_chain_spec
@@ -99,16 +99,12 @@ old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str)
 MetadataTask.meta_host = config.get('META_HOST')
 MetadataTask.meta_port = config.get('META_PORT')
 
-txs_dir = os.path.join(args.import_dir, 'txs')
-os.makedirs(txs_dir, exist_ok=True)
-sys.stdout.write(f'created txs dir: {txs_dir}')
-
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 get_celery_worker_status(celery_app)
 
 
 def main():
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
     ImportTask.balance_processor = BalanceProcessor(conn,
                                                     chain_spec,
                                                     config.get('CIC_REGISTRY_ADDRESS'),

apps/data-seeding/cic_ussd/import_task.py

@@ -14,7 +14,9 @@ from celery import Task
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.tx import raw, unpack
-from cic_types.models.person import Person, generate_metadata_pointer
+from cic_types.models.person import Person, identity_tag
+from cic_types.processor import generate_metadata_pointer
+from cic_types.condiments import MetadataPointer
 from hexathon import add_0x, strip_0x
 
 # local imports
@@ -55,7 +57,7 @@ class MetadataTask(ImportTask):
 
 
 def old_address_from_phone(base_path: str, phone_number: str):
-    pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
+    pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
     phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}')
     with open(phone_idx_path, 'r') as f:
         old_address = f.read()
@@ -73,9 +75,13 @@ def generate_person_metadata(self, blockchain_address: str, phone_number: str):
     person = Person.deserialize(person_metadata)
     if not person.identities.get('evm'):
         person.identities['evm'] = {}
-    sub_chain_str = f'{self.chain_spec.common_name()}:{self.chain_spec.network_id()}'
-    person.identities['evm'][sub_chain_str] = [add_0x(blockchain_address)]
-    blockchain_address = strip_0x(blockchain_address)
+    chain_spec = self.chain_spec.asdict()
+    arch = chain_spec.get('arch')
+    fork = chain_spec.get('fork')
+    tag = identity_tag(chain_spec)
+    person.identities[arch][fork] = {
+        tag: [blockchain_address]
+    }
     file_path = os.path.join(
         self.import_dir,
         'new',
@@ -102,7 +108,7 @@ def generate_preferences_data(self, data: tuple):
     blockchain_address: str = data[0]
     preferences = data[1]
     preferences_dir = os.path.join(self.import_dir, 'preferences')
-    preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), ':cic.preferences')
+    preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES)
     preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
     filepath = os.path.join(
         preferences_dir,
@@ -137,7 +143,7 @@ def generate_ussd_data(self, blockchain_address: str, phone_number: str):
     preferred_language = random.sample(["en", "sw"], 1)[0]
     preferences = {'preferred_language': preferred_language}
     with open(ussd_data_file, file_op) as ussd_data_file:
-        ussd_data_file.write(f'{phone_number}, { 1}, {preferred_language}, {False}\n')
+        ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n')
         logg.debug(f'written ussd data for address: {blockchain_address}')
     return blockchain_address, preferences
 
@@ -163,7 +169,7 @@ def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial:
 
 @celery_app.task(bind=True, base=MetadataTask)
 def resolve_phone(self, phone_number: str):
-    identifier = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
+    identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
     url = parse.urljoin(self.meta_url(), identifier)
     logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}')
     r = request.urlopen(url)

apps/data-seeding/cic_ussd/import_users.py

@@ -17,6 +17,7 @@ from cic_types.models.person import Person
 from confini import Config
 
 # local imports
+from common.dirs import initialize_dirs
 from import_util import get_celery_worker_status
 
 default_config_dir = './config'
@@ -37,6 +38,7 @@ arg_parser.add_argument('--env-prefix',
                         dest='env_prefix',
                         type=str,
                         help='environment prefix for variables to overwrite configuration.')
+arg_parser.add_argument('-f', action='store_true', help='force clear previous state')
 arg_parser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
 arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit data seeding tasks to.')
 arg_parser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -70,21 +72,7 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug(f'config loaded from {args.c}:\n{config}')
 
-old_account_dir = os.path.join(args.import_dir, 'old')
-os.stat(old_account_dir)
-logg.debug(f'created old system data dir: {old_account_dir}')
-
-new_account_dir = os.path.join(args.import_dir, 'new')
-os.makedirs(new_account_dir, exist_ok=True)
-logg.debug(f'created new system data dir: {new_account_dir}')
-
-person_metadata_dir = os.path.join(args.import_dir, 'meta')
-os.makedirs(person_metadata_dir, exist_ok=True)
-logg.debug(f'created person metadata dir: {person_metadata_dir}')
-
-preferences_dir = os.path.join(args.import_dir, 'preferences')
-os.makedirs(os.path.join(preferences_dir, 'meta'), exist_ok=True)
-logg.debug(f'created preferences metadata dir: {preferences_dir}')
+dirs = initialize_dirs(args.import_dir, force_reset=args.f)
 
 valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
 
@@ -157,7 +145,7 @@ def register_account(person: Person):
 if __name__ == '__main__':
     i = 0
     j = 0
-    for x in os.walk(old_account_dir):
+    for x in os.walk(dirs['old']):
         for y in x[2]:
             if y[len(y) - 5:] != '.json':
                 continue