move files out of scripts folder to their own dir

apps/data-seeding/cic_ussd/import_balance.py

@@ -0,0 +1,157 @@
+# standard imports
+import os
+import sys
+import logging
+import argparse
+import hashlib
+import redis
+import celery
+
+# external imports
+import confini
+from chainlib.eth.connection import EthHTTPConnection
+from chainlib.chain import ChainSpec
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainlib.eth.address import to_checksum_address
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore.dict import DictKeystore
+from cic_types.models.person import Person
+
+# local imports
+from import_util import BalanceProcessor
+from import_task import *
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_dir = './config'
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
+argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
+argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
+argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
+argparser.add_argument('--meta-host', dest='meta_host', type=str, help='metadata server host')
+argparser.add_argument('--meta-port', dest='meta_port', type=int, help='metadata server host')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--offset', type=int, default=0, help='block offset to start syncer from')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('user_dir', default='out', type=str, help='user export directory')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        'META_HOST': getattr(args, 'meta_host'),
+        'META_PORT': getattr(args, 'meta_port'),
+        }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+r = redis.Redis(redis_host, redis_port, redis_db)
+celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+signer_address = None
+keystore = DictKeystore()
+if args.y != None:
+    logg.debug('loading keystore file {}'.format(args.y))
+    signer_address = keystore.import_keystore_file(args.y)
+    logg.debug('now have key for signer address {}'.format(signer_address))
+signer = EIP155Signer(keystore)
+
+queue = args.q
+chain_str = config.get('CIC_CHAIN_SPEC')
+block_offset = 0
+if args.head:
+    block_offset = -1
+else:
+    block_offset = args.offset
+
+chain_spec = ChainSpec.from_chain_str(chain_str)
+old_chain_spec_str = args.old_chain_spec
+old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str)
+
+user_dir = args.user_dir # user_out_dir from import_users.py
+
+token_symbol = args.token_symbol
+
+MetadataTask.meta_host = config.get('META_HOST')
+MetadataTask.meta_port = config.get('META_PORT')
+ImportTask.chain_spec = chain_spec
+
+def main():
+    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+   
+    ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer)
+    ImportTask.balance_processor.init()
+
+    # TODO get decimals from token
+    balances = {}
+    f = open('{}/balances.csv'.format(user_dir, 'r'))
+    remove_zeros = 10**6
+    i = 0
+    while True:
+        l = f.readline()
+        if l == None:
+            break
+        r = l.split(',')
+        try:
+            address = to_checksum_address(r[0])
+            sys.stdout.write('loading balance {} {} {}'.format(i, address, r[1]).ljust(200) + "\r")
+        except ValueError:
+            break
+        balance = int(int(r[1].rstrip()) / remove_zeros)
+        balances[address] = balance
+        i += 1
+
+    f.close()
+
+    ImportTask.balances = balances
+    ImportTask.count = i
+
+    s = celery.signature(
+            'import_task.send_txs',
+            [
+                MetadataTask.balance_processor.nonce_offset,
+                ],
+            queue='cic-import-ussd',
+            )
+    s.apply_async()
+
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
+
+if __name__ == '__main__':
+    main()

apps/data-seeding/cic_ussd/import_pins.py

@@ -0,0 +1,70 @@
+# standard import
+import argparse
+import csv
+import logging
+import os
+
+# third-party imports
+import celery
+import confini
+
+# local imports
+from import_task import *
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = './config'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit transaction tasks to')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('pins_dir', default='out', type=str, help='user export directory')
+args = arg_parser.parse_args()
+
+# set log levels
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+# process configs
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+
+
+def main():
+    with open(f'{args.pins_dir}/pins.csv') as pins_file:
+        phone_to_pins = [tuple(row) for row in csv.reader(pins_file)]
+
+    s_import_pins = celery.signature(
+        'import_task.set_pins',
+        (db_configs, phone_to_pins),
+        queue=args.q
+    )
+    s_import_pins.apply_async()
+
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
+
+if __name__ == '__main__':
+    main()

apps/data-seeding/cic_ussd/import_task.py

@@ -0,0 +1,276 @@
+# standard imports
+import os
+import logging
+import urllib.parse
+import urllib.error
+import urllib.request
+import json
+
+# external imports
+import celery
+import psycopg2
+from psycopg2 import extras
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.tx import (
+        unpack,
+        raw,
+        )
+from cic_types.processor import generate_metadata_pointer
+from cic_types.models.person import Person
+
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
+
+celery_app = celery.current_app
+
+
+class ImportTask(celery.Task):
+
+    balances = None
+    import_dir = 'out'
+    count = 0
+    chain_spec = None
+    balance_processor = None
+    max_retries = None
+
+class MetadataTask(ImportTask):
+
+    meta_host = None
+    meta_port = None
+    meta_path = ''
+    meta_ssl = False
+    autoretry_for = (
+            urllib.error.HTTPError,
+            OSError,
+            )
+    retry_jitter = True
+    retry_backoff = True
+    retry_backoff_max = 60
+
+    @classmethod
+    def meta_url(self):
+        scheme = 'http'
+        if self.meta_ssl:
+            scheme += 's'
+        url = urllib.parse.urlparse('{}://{}:{}/{}'.format(scheme, self.meta_host, self.meta_port, self.meta_path))
+        return urllib.parse.urlunparse(url)
+
+
+def old_address_from_phone(base_path, phone):
+    pidx = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
+    phone_idx_path = os.path.join('{}/phone/{}/{}/{}'.format(
+            base_path,
+            pidx[:2],
+            pidx[2:4],
+            pidx,
+            )
+        )
+    f = open(phone_idx_path, 'r')
+    old_address = f.read()
+    f.close()
+
+    return old_address
+
+
+@celery_app.task(bind=True, base=MetadataTask)
+def resolve_phone(self, phone):
+    identifier = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
+    url = urllib.parse.urljoin(self.meta_url(), identifier)
+    logg.debug('attempt getting phone pointer at {} for phone {}'.format(url, phone))
+    r = urllib.request.urlopen(url)
+    address = json.load(r)
+    address = address.replace('"', '')
+    logg.debug('address {} for phone {}'.format(address, phone))
+
+    return address
+
+
+@celery_app.task(bind=True, base=MetadataTask)
+def generate_metadata(self, address, phone):
+    old_address = old_address_from_phone(self.import_dir, phone)
+
+    logg.debug('address {}'.format(address))
+    old_address_upper = strip_0x(old_address).upper()
+    metadata_path = '{}/old/{}/{}/{}.json'.format(
+            self.import_dir,
+            old_address_upper[:2],
+            old_address_upper[2:4],
+            old_address_upper,
+            )
+
+    f = open(metadata_path, 'r')
+    o = json.load(f)
+    f.close()
+
+    u = Person.deserialize(o)
+
+    if u.identities.get('evm') == None:
+        u.identities['evm'] = {}
+    sub_chain_str = '{}:{}'.format(self.chain_spec.common_name(), self.chain_spec.network_id())
+    u.identities['evm'][sub_chain_str] = [add_0x(address)]
+
+    new_address_clean = strip_0x(address)
+    filepath = os.path.join(
+            self.import_dir,
+            'new',
+            new_address_clean[:2].upper(),
+            new_address_clean[2:4].upper(),
+            new_address_clean.upper() + '.json',
+            )
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
+    o = u.serialize()
+    f = open(filepath, 'w')
+    f.write(json.dumps(o))
+    f.close()
+
+    meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
+    meta_filepath = os.path.join(
+            self.import_dir,
+            'meta',
+            '{}.json'.format(new_address_clean.upper()),
+            )
+    os.symlink(os.path.realpath(filepath), meta_filepath)
+
+    logg.debug('found metadata {} for phone {}'.format(o, phone))
+
+    return address
+
+
+@celery_app.task(bind=True, base=MetadataTask)
+def opening_balance_tx(self, address, phone, serial):
+
+
+    old_address = old_address_from_phone(self.import_dir, phone)
+  
+    k = to_checksum_address(strip_0x(old_address))
+    balance = self.balances[k]
+    logg.debug('found balance {} for address {} phone {}'.format(balance, old_address, phone))
+
+    decimal_balance = self.balance_processor.get_decimal_amount(int(balance))
+
+    (tx_hash_hex, o) = self.balance_processor.get_rpc_tx(address, decimal_balance, serial)
+
+    tx = unpack(bytes.fromhex(strip_0x(o)), self.chain_spec)
+    logg.debug('generated tx token value {} to {} tx hash {}'.format(decimal_balance, address, tx_hash_hex))
+
+    tx_path = os.path.join(
+            self.import_dir,
+            'txs',
+            strip_0x(tx_hash_hex),
+            )
+            
+    f = open(tx_path, 'w')
+    f.write(strip_0x(o))
+    f.close()
+
+    tx_nonce_path = os.path.join(
+            self.import_dir,
+            'txs',
+            '.' + str(tx['nonce']),
+            )
+    os.symlink(os.path.realpath(tx_path), tx_nonce_path)
+
+    return tx['hash']
+
+
+@celery_app.task(bind=True, base=ImportTask, autoretry_for=(FileNotFoundError,), max_retries=None, default_retry_delay=0.1)
+def send_txs(self, nonce):
+
+    if nonce == self.count + self.balance_processor.nonce_offset:
+        logg.info('reached nonce {} (offset {} + count {}) exiting'.format(nonce, self.balance_processor.nonce_offset, self.count))
+        return
+
+
+    logg.debug('attempt to open symlink for nonce {}'.format(nonce))
+    tx_nonce_path = os.path.join(
+            self.import_dir,
+            'txs',
+            '.' + str(nonce),
+            )
+    f = open(tx_nonce_path, 'r')
+    tx_signed_raw_hex = f.read()
+    f.close()
+
+    os.unlink(tx_nonce_path)
+
+    o = raw(add_0x(tx_signed_raw_hex))
+    tx_hash_hex = self.balance_processor.conn.do(o)
+
+    logg.info('sent nonce {} tx hash {}'.format(nonce, tx_hash_hex)) #tx_signed_raw_hex))
+
+    nonce += 1
+
+    queue = self.request.delivery_info.get('routing_key')
+    s = celery.signature(
+            'import_task.send_txs',
+            [
+                nonce,
+                ],
+            queue=queue,
+            )
+    s.apply_async()
+
+
+    return nonce
+
+
+@celery_app.task
+def set_pins(config: dict, phone_to_pins: list):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # update db
+    for element in phone_to_pins:
+        sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
+        db_cursor.execute(sql, (element[1], element[0]))
+        logg.debug(f'Updating: {element[0]} with: {element[1]}')
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+
+
+@celery_app.task
+def set_ussd_data(config: dict, ussd_data: dict):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # process ussd_data
+    account_status = 1
+    if ussd_data['is_activated'] == 1:
+        account_status = 2
+    preferred_language = ussd_data['preferred_language']
+    phone_number = ussd_data['phone']
+
+    sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s'
+    db_cursor.execute(sql, (account_status, preferred_language, phone_number))
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+

apps/data-seeding/cic_ussd/import_users.py

@@ -0,0 +1,201 @@
+# standard imports
+import os
+import sys
+import json
+import logging
+import argparse
+import uuid
+import datetime
+import time
+import urllib.request
+from glob import glob
+
+# third-party imports
+import redis
+import confini
+import celery
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.eth.address import to_checksum
+from cic_types.models.person import Person
+from cic_eth.api.api_task import Api
+from chainlib.chain import ChainSpec
+from cic_types.processor import generate_metadata_pointer
+import phonenumbers
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--batch-size', dest='batch_size', default=100, type=int, help='burst size of sending transactions to node') # batch size should be slightly below cumulative gas limit worth, eg 80000 gas txs with 8000000 limit is a bit less than 100 batch size
+argparser.add_argument('--batch-delay', dest='batch_delay', default=3, type=int, help='seconds delay between batches')
+argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = argparser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        }
+config.dict_override(args_override, 'cli')
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+r = redis.Redis(redis_host, redis_port, redis_db)
+
+ps = r.pubsub()
+
+user_new_dir = os.path.join(args.user_dir, 'new')
+os.makedirs(user_new_dir)
+
+meta_dir = os.path.join(args.user_dir, 'meta')
+os.makedirs(meta_dir)
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+txs_dir = os.path.join(args.user_dir, 'txs')
+os.makedirs(txs_dir)
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_str = str(chain_spec)
+
+batch_size = args.batch_size
+batch_delay = args.batch_delay
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+  
+
+def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
+    url = 'http'
+    if ssl:
+        url += 's'
+    url += '://{}:{}'.format(host, port)
+    url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS'))
+
+    logg.info('ussd service url {}'.format(url))
+    logg.info('ussd phone {}'.format(phone))
+
+    session = uuid.uuid4().hex
+    data = {
+            'sessionId': session,
+            'serviceCode': service_code,
+            'phoneNumber': phone,
+            'text': service_code,
+        }
+    req = urllib.request.Request(url)
+    data_str = json.dumps(data)
+    data_bytes = data_str.encode('utf-8')
+    req.add_header('Content-Type', 'application/json')
+    req.data = data_bytes
+
+    return req
+
+
+def register_ussd(i, u):
+    phone_object = phonenumbers.parse(u.tel)
+    phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+    logg.debug('tel {} {}'.format(u.tel, phone))
+    req = build_ussd_request(phone, 'localhost', 63315, '*483*46#', '', '') 
+    response = urllib.request.urlopen(req)
+    response_data = response.read().decode('utf-8')
+    state = response_data[:3]
+    out = response_data[4:]
+    logg.debug('ussd reponse: {}'.format(out))
+
+
+if __name__ == '__main__':
+
+    i = 0
+    j = 0
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            if y[len(y)-5:] != '.json':
+                continue
+            # handle json containing person object
+            filepath = None
+            if y[:15] != '_ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+                u = Person.deserialize(o)
+
+                new_address = register_ussd(i, u)
+
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+
+                s_phone = celery.signature(
+                        'import_task.resolve_phone',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
+
+                s_meta = celery.signature(
+                        'import_task.generate_metadata',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
+
+                s_balance = celery.signature(
+                        'import_task.opening_balance_tx',
+                        [
+                            phone,
+                            i,
+                            ],
+                        queue='cic-import-ussd',
+                        )
+
+                s_meta.link(s_balance)
+                s_phone.link(s_meta)
+                # block time plus a bit of time for ussd processing
+                s_phone.apply_async(countdown=7)
+
+                i += 1
+                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+
+                j += 1
+                if j == batch_size:
+                    time.sleep(batch_delay)
+                    j = 0
+

apps/data-seeding/cic_ussd/import_ussd_data.py

@@ -0,0 +1,70 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# external imports
+import celery
+from confini import Config
+
+# local imports
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+if __name__ == '__main__':
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # handle ussd_data json object
+            if y[:15] == '_ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    ussd_data = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+
+                s_set_ussd_data = celery.signature(
+                    'import_task.set_ussd_data',
+                    [db_configs, ussd_data]
+                )
+                s_set_ussd_data.apply_async(queue='cic-import-ussd')

apps/data-seeding/cic_ussd/import_util.py

@@ -0,0 +1,72 @@
+# standard imports
+import logging
+
+# external imports
+from eth_contract_registry import Registry
+from eth_token_index import TokenUniqueSymbolIndex
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainlib.eth.erc20 import ERC20
+from chainlib.eth.tx import (
+        count,
+        TxFormat,
+        )
+
+logg = logging.getLogger().getChild(__name__)
+
+
+class BalanceProcessor:
+
+    def __init__(self, conn, chain_spec, registry_address, signer_address, signer):
+
+        self.chain_spec = chain_spec
+        self.conn = conn
+        #self.signer_address = signer_address
+        self.registry_address = registry_address
+
+        self.token_index_address = None
+        self.token_address = None
+        self.signer_address = signer_address
+        self.signer = signer
+
+        o = count(signer_address)
+        c = self.conn.do(o)
+        self.nonce_offset = int(c, 16)
+        self.gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
+
+        self.value_multiplier = 1
+    
+
+    def init(self):
+        # Get Token registry address
+        registry = Registry(self.chain_spec)
+        o = registry.address_of(self.registry_address, 'TokenRegistry')
+        r = self.conn.do(o)
+        self.token_index_address = registry.parse_address_of(r)
+        logg.info('found token index address {}'.format(self.token_index_address))
+
+        token_registry = TokenUniqueSymbolIndex(self.chain_spec)
+        o = token_registry.address_of(self.token_index_address, 'SRF')
+        r = self.conn.do(o)
+        self.token_address = token_registry.parse_address_of(r)
+        logg.info('found SRF token address {}'.format(self.token_address))
+
+        tx_factory = ERC20(self.chain_spec)
+        o = tx_factory.decimals(self.token_address)
+        r = self.conn.do(o)
+        n = tx_factory.parse_decimals(r)
+        self.value_multiplier = 10 ** n
+
+
+    def get_rpc_tx(self, recipient, value, i):
+        logg.debug('initiating nonce offset {} for recipient {}'.format(self.nonce_offset + i, recipient))
+        nonce_oracle = OverrideNonceOracle(self.signer_address, self.nonce_offset + i)
+        tx_factory = ERC20(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=self.gas_oracle)
+        return tx_factory.transfer(self.token_address, self.signer_address, recipient, value, tx_format=TxFormat.RLP_SIGNED)
+        #(tx_hash_hex, o) = tx_factory.transfer(self.token_address, self.signer_address, recipient, value)
+        #self.conn.do(o)
+        #return tx_hash_hex
+
+
+    def get_decimal_amount(self, value):
+        return value * self.value_multiplier