Updates verification steps.

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1,2 +1 @@
 from .erc20 import *
-from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -1,73 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from erc20_faucet import Faucet
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.status import Status
-from hexathon import strip_0x
-
-# local imports
-import cic_cache.db as cic_cache_db
-from .base import TagSyncFilter
-
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
-
-
-class FaucetFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
-        super(FaucetFilter, self).__init__('give_to', domain='faucet')
-        self.chain_spec = chain_spec
-        self.sender_address = sender_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        try:
-            data = strip_0x(tx.payload)
-        except ValueError:
-            return False
-        logg.debug('data {}'.format(data))
-        if Faucet.method_for(data[:8]) == None:
-            return False
-
-        token_sender = tx.inputs[0]
-        token_recipient = data[64+8-40:]
-        logg.debug('token recipient {}'.format(token_recipient))
- 
-        f = Faucet(self.chain_spec)
-        o = f.token(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token = f.parse_token(r)
-
-        f = Faucet(self.chain_spec)
-        o = f.token_amount(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token_value = f.parse_token_amount(r)
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token,
-                token,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -41,7 +41,6 @@ from cic_cache.db import (
         )
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
-        FaucetFilter,
         )
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
@@ -72,7 +71,6 @@ def register_filter_tags(filters, session):
             session.commit()
             logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
         except sqlalchemy.exc.IntegrityError:
-            session.rollback()
             logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
 
 
@@ -114,11 +112,9 @@ def main():
         logg.info('using trusted address {}'.format(address))
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
-    faucet_filter = FaucetFilter(chain_spec)
 
     filters = [
         erc20_transfer_filter,
-        faucet_filter,
             ]
 
     session = SessionBase.create_session()

apps/cic-cache/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \

apps/cic-cache/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.2b9
+cic-base~=0.1.2b8
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1

apps/cic-cache/tests/filters/test_erc20.py

@@ -22,7 +22,7 @@ from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
 logg = logging.getLogger()
 
 
-def test_erc20_filter(
+def test_cache(
         eth_rpc,
         foo_token,
         init_database,

apps/cic-cache/tests/filters/test_faucet.py

@@ -1,71 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.chain import ChainSpec
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_by_hash,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        unpack,
-        transaction,
-        Tx,
-        )
-from hexathon import strip_0x
-from erc20_faucet.faucet import SingleShotFaucet
-from sqlalchemy import text
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
-
-logg = logging.getLogger()
-
-
-def test_filter_faucet(
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        faucet_noregistry,
-        init_database,
-        list_defaults,
-        contract_roles,
-        agent_roles,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
-    c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
-    r = eth_rpc.do(o)
-
-    tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
-
-    o = receipt(r)
-    r = eth_rpc.do(o)
-    rcpt = Tx.src_normalize(r)
-
-    assert r['status'] == 1
-
-    o = block_by_hash(r['block_hash'])
-    r = eth_rpc.do(o)
-    block_object = Block(r)
-
-    tx = Tx(tx_src, block_object)
-    tx.apply_receipt(rcpt)
-
-    r = fltr.filter(eth_rpc, block_object, tx, init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash

apps/cic-eth/cic_eth/eth/account.py

@@ -20,8 +20,7 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index.registry import AccountRegistry
-from eth_accounts_index import AccountsIndex 
+from eth_accounts_index.registry import AccountRegistry # TODO, use interface module instead (needs gas limit method)
 from sarafu_faucet import MinterFaucet
 from chainqueue.db.models.tx import TxCache
 
@@ -128,12 +127,12 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     if writer_address == ZERO_ADDRESS:
         session.close()
         raise RoleMissingError('call address for resgistering {}'.format(account_address))
-    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
+    account_registry_address = registry.by_name('AccountsIndex', sender_address=call_address)
    
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    gas_oracle = self.create_gas_oracle(rpc, AccountsIndex.gas)
     account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -8,11 +8,11 @@ from chainlib.status import Status as TxStatus
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.erc20 import ERC20
 from hexathon import (
         strip_0x,
         add_0x,
         )
-from eth_erc20 import ERC20
 from erc20_faucet import Faucet
 
 # local imports

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -14,7 +14,7 @@ from .base import SyncFilter
 
 logg = logging.getLogger().getChild(__name__)
 
-account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
+account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd'
 
 
 class RegistrationFilter(SyncFilter):

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -30,7 +30,7 @@ class TxFilter(SyncFilter):
         if otx == None:
             logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
             return None
-        logg.debug('otx filter match on {}'.format(otx.tx_hash))
+        logg.info('tx filter match on {}'.format(otx.tx_hash))
         db_session.flush()
         SessionBase.release_session(db_session)
         s_final_state = celery.signature(

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,10 +12,7 @@ import confini
 import celery
 
 # local imports
-from cic_eth.api import (
-        Api,
-        AdminApi,
-        )
+from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -56,13 +53,8 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 queue = args.q
 
 api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
-admin_api = AdminApi(None)
 
 def main():
-    t = admin_api.registry()
-    registry = t.get()
-    print('Registry address: {}'.format(registry))
-
     t = api.default_token()
     token_info = t.get()
     print('Default token symbol: {}'.format(token_info['symbol']))

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.13',
+        'beta.12',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -19,7 +19,7 @@ RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # Copy shared requirements from top of mono-repo
-RUN echo "copying root req file:  ${root_requirement_file}"
+RUN echo "copying root req file ${root_requirement_file}"
 #COPY $root_requirement_file .
 #RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
 RUN /usr/local/bin/python -m pip install --upgrade pip
@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b8
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base==0.1.2b9
+cic-base==0.1.2b8
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
@@ -18,7 +18,7 @@ chainlib~=0.0.3a1
 hexathon~=0.0.1a7
 chainsyncer[sql]~=0.0.2a4
 chainqueue~=0.0.2a2
-sarafu-faucet==0.0.3a3
-erc20-faucet==0.2.1a4
+sarafu-faucet==0.0.3a1
 coincurve==15.0.0
-potaahto~=0.0.1a2
+sarafu-faucet==0.0.3a2
+potaahto~=0.0.1a1

apps/cic-ussd/cic_ussd/account/__init__.py

@@ -1,62 +0,0 @@
-"""This package handles account operations."""
-
-# external imports
-from cic_eth.api import Api
-
-# local imports
-from cic_ussd.operations import (add_tasks_to_tracker,
-                                 cache_account_creation_task_id,
-                                 create_or_update_session,
-                                 define_multilingual_responses)
-from cic_ussd.menu.ussd_menu import UssdMenu
-
-
-def create(chain_str: str,
-           external_session_id: str,
-           phone_number: str,
-           service_code: str,
-           user_input: str) -> str:
-    """This function issues a task to create a blockchain account on cic-eth. It then creates a record of the ussd
-    session corresponding to the creation of the account and returns a response denoting that the user's account is
-    being created.
-    :param chain_str: The chain name and network id.
-    :type chain_str: str
-    :param external_session_id: A unique ID from africastalking.
-    :type external_session_id: str
-    :param phone_number: The phone number for the account to be created.
-    :type phone_number: str
-    :param service_code: The service code dialed.
-    :type service_code: str
-    :param user_input: The input entered by the user.
-    :type user_input: str
-    :return: A response denoting that the account is being created.
-    :rtype: str
-    """
-    # attempt to create a user
-    cic_eth_api = Api(callback_task='cic_ussd.tasks.callback_handler.process_account_creation_callback',
-                      callback_queue='cic-ussd',
-                      callback_param='',
-                      chain_str=chain_str)
-    creation_task_id = cic_eth_api.create_account().id
-
-    # record task initiation time
-    add_tasks_to_tracker(task_uuid=creation_task_id)
-
-    # cache account creation data
-    cache_account_creation_task_id(phone_number=phone_number, task_id=creation_task_id)
-
-    # find menu to notify user account is being created
-    current_menu = UssdMenu.find_by_name(name='account_creation_prompt')
-
-    # create a ussd session session
-    create_or_update_session(
-        external_session_id=external_session_id,
-        phone=phone_number,
-        service_code=service_code,
-        current_menu=current_menu.get('name'),
-        user_input=user_input)
-
-    # define response to relay to user
-    response = define_multilingual_responses(
-        key='ussd.kenya.account_creation_prompt', locales=['en', 'sw'], prefix='END')
-    return response

apps/cic-ussd/cic_ussd/account/data.py

@@ -1,50 +0,0 @@
-"""This module defines functions required to query external components of the cic platform for data relevant to
-accounts on the cic-ussd component.
-"""
-
-# external imports
-import celery
-from cic_eth.api import Api
-
-# local imports
-from cic_ussd.chain import Chain
-
-
-def person_metadata(blockchain_address: str):
-    """This function asynchronously queries the metadata server for metadata associated with the person data type and
-    a given blockchain address.
-    :param blockchain_address: Ethereum address of account whose metadata is being queried.
-    :type blockchain_address: str, 0x-hex
-    """
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
-
-def default_token_data() -> dict:
-    """This function queries for the default token's data from the cic_eth tasks exposed over its Api class.
-    :return: A dict containing the default token address and it's corresponding symbol.
-    :rtype: dict
-    """
-    chain_str = Chain.spec.__str__()
-    cic_eth_api = Api(chain_str=chain_str)
-    default_token_request_task = cic_eth_api.default_token()
-    return default_token_request_task.get()
-
-
-def transactions_statement(blockchain_address: str):
-    """This function asynchronously queries the cic-eth server to retrieve a chronologically reversed list of
-    transactions for an account.
-    :param blockchain_address: Ethereum address of account whose transactions is being queried.
-    :type blockchain_address:
-    """
-    chain_str = Chain.spec.__str__()
-    cic_eth_api = Api(
-        chain_str=chain_str,
-        callback_queue='cic-ussd',
-        callback_task='cic_ussd.tasks.callback_handler.process_statement_callback',
-        callback_param=blockchain_address
-    )
-    cic_eth_api.list(address=blockchain_address, limit=9)

apps/cic-ussd/cic_ussd/balance.py

@@ -33,12 +33,12 @@ class BalanceManager:
     def get_balances(self, asynchronous: bool = False) -> Union[celery.Task, dict]:
         """
         This function queries cic-eth for an account's balances, It provides a means to receive the balance either
-        asynchronously or synchronously depending on the provided asynchronous value from the function's caller. It
-        returns a dictionary containing network, outgoing and incoming balances when synchronously called.
+        asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a
+        dictionary containing network, outgoing and incoming balances.
         :param asynchronous: Boolean value checking whether to return balances asynchronously
         :type asynchronous: bool
-        :return: dict containing network, outgoing and incoming balances | an async result object from celery.
-        :rtype: dict|celery.Task
+        :return:
+        :rtype:
         """
         if asynchronous:
             cic_eth_api = Api(
@@ -47,7 +47,7 @@ class BalanceManager:
                 callback_task='cic_ussd.tasks.callback_handler.process_balances_callback',
                 callback_param=''
             )
-            return cic_eth_api.balance(address=self.address, token_symbol=self.token_symbol)
+            cic_eth_api.balance(address=self.address, token_symbol=self.token_symbol)
         else:
             cic_eth_api = Api(chain_str=self.chain_str)
             balance_request_task = cic_eth_api.balance(
@@ -56,30 +56,27 @@ class BalanceManager:
             return balance_request_task.get()[0]
 
 
-def operational_balance(balances: dict) -> float:
-    """This function computes the operational balance at an instance in the system by providing the difference of the
-    outgoing balance from the sum of incoming and network balances.
-    :param balances: A dictionary containing incoming, outgoing and network balances.
-    :type balances: dict
-    :return: The operational balance of the account at the instance of querying.
-    :rtype: float
+def compute_operational_balance(balances: dict) -> float:
+    """This function calculates the right balance given incoming and outgoing
+    :param balances:
+    :type balances:
+    :return:
+    :rtype:
     """
     incoming_balance = balances.get('balance_incoming')
     outgoing_balance = balances.get('balance_outgoing')
     network_balance = balances.get('balance_network')
 
-    balance = (network_balance + incoming_balance) - outgoing_balance
-    return from_wei(value=balance)
+    operational_balance = (network_balance + incoming_balance) - outgoing_balance
+    return from_wei(value=operational_balance)
 
 
-def cached_operational_balance(blockchain_address: str) -> float:
-    """This function retrieves the cached balances data from redis cache and computes the operational balance from
-    the cached data.
-    :param blockchain_address: Ethereum address of the account whose balance data is being retrieved.
-    :type blockchain_address: str, 0x-hex
-    :return: The operational balance of the account as per cached balance data.
-    :rtype: float
-    :raises CachedDataNotFoundError
+def get_cached_operational_balance(blockchain_address: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
     """
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
@@ -87,7 +84,7 @@ def cached_operational_balance(blockchain_address: str) -> float:
     )
     cached_balance = get_cached_data(key=key)
     if cached_balance:
-        balance = operational_balance(balances=json.loads(cached_balance))
-        return balance
+        operational_balance = compute_operational_balance(balances=json.loads(cached_balance))
+        return operational_balance
     else:
         raise CachedDataNotFoundError('Cached operational balance not found.')

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -120,7 +120,7 @@ class MetadataRequestsHandler(Metadata):
         data = json.loads(response_data.decode('utf-8'))
         if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/operations.py

@@ -325,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -371,13 +371,6 @@ def process_start_menu(display_key: str, user: Account):
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 

apps/contract-migration/scripts/README.md

@@ -104,8 +104,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-ussd-tasker
-* cic-ussd-server
+* cic-user-tasker
+* cic-user-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,6 +169,24 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
+Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
+
+In one terminal run:
+
+`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
+
+This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the current sempo system.
+This csv file will be stored in the pins export dir defined as the positional argument.
+
+Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
+
+- To import the pins:
+
+`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
+
+- To import ussd data:
+`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
+
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -27,6 +27,7 @@ from chainlib.eth.block import (
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
@@ -36,7 +37,6 @@ from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
-from eth_erc20 import ERC20
 
 
 logging.basicConfig(level=logging.WARNING)

apps/contract-migration/scripts/cic_ussd/import_pins.py

@@ -0,0 +1,70 @@
+# standard import
+import argparse
+import csv
+import logging
+import os
+
+# third-party imports
+import celery
+import confini
+
+# local imports
+from import_task import *
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = './config'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit transaction tasks to')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('pins_dir', default='out', type=str, help='user export directory')
+args = arg_parser.parse_args()
+
+# set log levels
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+# process configs
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+
+
+def main():
+    with open(f'{args.pins_dir}/pins.csv') as pins_file:
+        phone_to_pins = [tuple(row) for row in csv.reader(pins_file)]
+
+    s_import_pins = celery.signature(
+        'import_task.set_pins',
+        (db_configs, phone_to_pins),
+        queue=args.q
+    )
+    s_import_pins.apply_async()
+
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/cic_ussd/import_task.py

@@ -8,6 +8,8 @@ import json
 
 # external imports
 import celery
+import psycopg2
+from psycopg2 import extras
 from hexathon import (
         strip_0x,
         add_0x,
@@ -53,7 +55,7 @@ class MetadataTask(ImportTask):
     def meta_url(self):
         scheme = 'http'
         if self.meta_ssl:
-            scheme += s
+            scheme += 's'
         url = urllib.parse.urlparse('{}://{}:{}/{}'.format(scheme, self.meta_host, self.meta_port, self.meta_path))
         return urllib.parse.urlunparse(url)
 
@@ -91,7 +93,6 @@ def resolve_phone(self, phone):
 def generate_metadata(self, address, phone):
     old_address = old_address_from_phone(self.import_dir, phone)
 
-    logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
     logg.debug('address {}'.format(address))
     old_address_upper = strip_0x(old_address).upper()
     metadata_path = '{}/old/{}/{}/{}.json'.format(
@@ -216,3 +217,60 @@ def send_txs(self, nonce):
 
 
     return nonce
+
+
+@celery_app.task
+def set_pins(config: dict, phone_to_pins: list):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # update db
+    for element in phone_to_pins:
+        sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
+        db_cursor.execute(sql, (element[1], element[0]))
+        logg.debug(f'Updating: {element[0]} with: {element[1]}')
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+
+
+@celery_app.task
+def set_ussd_data(config: dict, ussd_data: dict):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # process ussd_data
+    account_status = 1
+    if ussd_data['is_activated'] == 1:
+        account_status = 2
+    preferred_language = ussd_data['preferred_language']
+    phone_number = ussd_data['phone']
+
+    sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s'
+    db_cursor.execute(sql, (account_status, preferred_language, phone_number))
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+

apps/contract-migration/scripts/cic_ussd/import_users.py

@@ -87,6 +87,13 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
   
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
@@ -135,57 +142,60 @@ if __name__ == '__main__':
         for y in x[2]:
             if y[len(y)-5:] != '.json':
                 continue
-            filepath = os.path.join(x[0], y)
-            f = open(filepath, 'r')
-            try:
-                o = json.load(f)
-            except json.decoder.JSONDecodeError as e:
+            # handle json containing person object
+            filepath = None
+            if y != 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
                 f.close()
-                logg.error('load error for {}: {}'.format(y, e))
-                continue
-            f.close()
-            u = Person.deserialize(o)
+                u = Person.deserialize(o)
 
-            new_address = register_ussd(i, u)
+                new_address = register_ussd(i, u)
 
-            phone_object = phonenumbers.parse(u.tel)
-            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-            s_phone = celery.signature(
-                    'import_task.resolve_phone',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_phone = celery.signature(
+                        'import_task.resolve_phone',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta = celery.signature(
-                    'import_task.generate_metadata',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_meta = celery.signature(
+                        'import_task.generate_metadata',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_balance = celery.signature(
-                    'import_task.opening_balance_tx',
-                    [
-                        phone,
-                        i,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_balance = celery.signature(
+                        'import_task.opening_balance_tx',
+                        [
+                            phone,
+                            i,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta.link(s_balance)
-            s_phone.link(s_meta)
-            s_phone.apply_async(countdown=7) # block time plus a bit of time for ussd processing
+                s_meta.link(s_balance)
+                s_phone.link(s_meta)
+                # block time plus a bit of time for ussd processing
+                s_phone.apply_async(countdown=7)
 
-            i += 1
-            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
-        
-            j += 1
-            if j == batch_size:
-                time.sleep(batch_delay)
-                j = 0
+                i += 1
+                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+
+                j += 1
+                if j == batch_size:
+                    time.sleep(batch_delay)
+                    j = 0
 
-    #fi.close()

apps/contract-migration/scripts/cic_ussd/import_ussd_data.py

@@ -0,0 +1,70 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# external imports
+import celery
+from confini import Config
+
+# local imports
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+if __name__ == '__main__':
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # handle ussd_data json object
+            if y == 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    ussd_data = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+
+                s_set_ussd_data = celery.signature(
+                    'import_task.set_ussd_data',
+                    [db_configs, ussd_data]
+                )
+                s_set_ussd_data.apply_async(queue='cic-import-ussd')

apps/contract-migration/scripts/create_import_pins.py

@@ -0,0 +1,90 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# third-party imports
+import bcrypt
+import celery
+import confini
+import phonenumbers
+import random
+from cic_types.models.person import Person
+from cryptography.fernet import Fernet
+
+# local imports
+
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+default_config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='Config dir')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('--userdir', type=str, help='path to users export dir tree')
+arg_parser.add_argument('pins_dir', type=str, help='path to pin export dir tree')
+
+
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+logg.info('loaded config\n{}'.format(config))
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+user_dir = args.userdir
+pins_dir = args.pins_dir
+
+
+def generate_password_hash():
+    key = Fernet.generate_key()
+    fnt = Fernet(key)
+    pin = str(random.randint(1000, 9999))
+    return fnt.encrypt(bcrypt.hashpw(pin.encode('utf-8'), bcrypt.gensalt())).decode()
+
+
+user_old_dir = os.path.join(user_dir, 'old')
+logg.debug(f'reading user data from: {user_old_dir}')
+
+pins_file = open(f'{pins_dir}/pins.csv', 'w')
+
+if __name__ == '__main__':
+
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            # skip non-json files
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # define file path for
+            filepath = None
+            if y != 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+                u = Person.deserialize(o)
+
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                password_hash = generate_password_hash()
+                pins_file.write(f'{phone},{password_hash}\n')
+                logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
+
+    pins_file.close()

apps/contract-migration/scripts/create_import_users.py

@@ -130,6 +130,7 @@ def genCats():
 def genAmount():
     return random.randint(0, gift_max) * gift_factor
 
+
 def genDob():
     dob_src = fake.date_of_birth(minimum_age=15)
     dob = {}
@@ -168,8 +169,9 @@ def gen():
             }
     p.location['area_name'] = city
     if random.randint(0, 1):
-        p.identities['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
-        p.identities['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+        p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
+        p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+
     
     return (old_blockchain_checksum_address, phone, p)
 
@@ -215,6 +217,20 @@ if __name__ == '__main__':
         json.dump(o.serialize(), f)
         f.close()
 
+        # create ussd data
+        ussd_data_dir = os.path.join(d, 'ussd_data')
+        os.makedirs(ussd_data_dir)
+        f = open('{}/{}/{}'.format(d, 'ussd_data', 'ussd_data.json'), 'w')
+        ussd_data = {
+            'phone': phone,
+            'is_activated': 1,
+            'preferred_language': random.sample(['en', 'sw'], 1)[0],
+            'is_disabled': False
+        }
+        json.dump(ussd_data, f)
+        f.close()
+
+
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/contract-migration/scripts/eth/import_balance.py

@@ -27,6 +27,7 @@ from chainlib.eth.block import (
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
@@ -36,7 +37,6 @@ from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
-from eth_erc20 import ERC20
 
 
 logging.basicConfig(level=logging.WARNING)

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b9
-sarafu-faucet==0.0.3a3
-cic-eth==0.11.0b13
-cic-types==0.1.0a11
+cic-base[full_graph]==0.1.2b8
+sarafu-faucet==0.0.3a2
+cic-eth==0.11.0b12
+cic-types==0.1.0a10
 crypto-dev-signer==0.4.14b3

apps/contract-migration/scripts/verify.py

@@ -1,52 +1,43 @@
 # standard imports
+import argparse
+import copy
+import csv
+import hashlib
+import json
+import logging
 import os
 import sys
-import logging
-import time
-import argparse
-import sys
-import re
-import hashlib
-import csv
-import json
 import urllib
-import copy
-import uuid
 import urllib.request
+import uuid
+from collections import Counter
 
 # external imports
 import celery
-import eth_abi
 import confini
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+import eth_abi
+import psycopg2
 from chainlib.chain import ChainSpec
+from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainlib.hash import keccak256_string_to_hex
-from chainlib.eth.address import to_checksum_address
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
-        OverrideGasOracle,
-        balance,
-        )
+    OverrideGasOracle,
+    balance,
+)
 from chainlib.eth.tx import TxFactory
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.jsonrpc import jsonrpc_template
-from chainlib.eth.error import EthException
 from cic_types.models.person import (
-        Person,
-        generate_metadata_pointer,
-        )
-from erc20_faucet import Faucet
-from eth_erc20 import ERC20
+    Person,
+    generate_metadata_pointer,
+)
+from erc20_single_shot_faucet import SingleShotFaucet
+from hexathon import (
+    strip_0x,
+    add_0x,
+)
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -72,6 +63,7 @@ eth_tests = [
 
 phone_tests = [
         'ussd',
+        'ussd_pins'
         ]
 
 all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
@@ -171,6 +163,39 @@ if logg.isEnabledFor(logging.DEBUG):
     outfunc = logg.debug
 
 
+def send_ussd_request(address, data_dir):
+    upper_address = strip_0x(address).upper()
+    f = open(os.path.join(
+        data_dir,
+        'new',
+        upper_address[:2],
+        upper_address[2:4],
+        upper_address + '.json',
+    ), 'r'
+    )
+    o = json.load(f)
+    f.close()
+
+    p = Person.deserialize(o)
+    phone = p.tel
+
+    session = uuid.uuid4().hex
+    data = {
+        'sessionId': session,
+        'serviceCode': config.get('APP_SERVICE_CODE'),
+        'phoneNumber': phone,
+        'text': '',
+    }
+
+    req = urllib.request.Request(config.get('_USSD_PROVIDER'))
+    data_str = json.dumps(data)
+    data_bytes = data_str.encode('utf-8')
+    req.add_header('Content-Type', 'application/json')
+    req.data = data_bytes
+    response = urllib.request.urlopen(req)
+    return response.read().decode('utf-8')
+
+
 class VerifierState:
 
     def __init__(self, item_keys, active_tests=None):
@@ -224,7 +249,7 @@ class Verifier:
         self.api = cic_eth_api
         self.data_dir = data_dir
         self.exit_on_error = exit_on_error
-        self.faucet_tx_factory = Faucet(chain_spec, gas_oracle=gas_oracle)
+        self.faucet_tx_factory = SingleShotFaucet(chain_spec, gas_oracle=gas_oracle)
 
         verifymethods = []
         for k in dir(self):
@@ -354,42 +379,18 @@ class Verifier:
 
 
     def verify_ussd(self, address, balance=None):
-        upper_address = strip_0x(address).upper()
-        f = open(os.path.join(
-            self.data_dir,
-            'new',
-            upper_address[:2],
-            upper_address[2:4],
-            upper_address + '.json',
-            ), 'r'
-            )
-        o = json.load(f)
-        f.close()
-
-        p = Person.deserialize(o)
-        phone = p.tel
-
-        session = uuid.uuid4().hex
-        data = {
-                'sessionId': session,
-                'serviceCode': config.get('APP_SERVICE_CODE'),
-                'phoneNumber': phone,
-                'text': config.get('APP_SERVICE_CODE'),
-            }
-
-        req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-        data_str = json.dumps(data)
-        data_bytes = data_str.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
-        req.data = data_bytes
-        response = urllib.request.urlopen(req)
-        response_data = response.read().decode('utf-8')
+        response_data = send_ussd_request(address, self.data_dir)
         state = response_data[:3]
         out = response_data[4:]
         m = '{} {}'.format(state, out[:7])
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
+    def verify_ussd_pins(self, address, balance):
+        response_data = send_ussd_request(address, self.data_dir)
+        if response_data[:11] != 'CON Balance':
+            raise VerifierError(response_data, 'pins')
+
 
     def verify(self, address, balance, debug_stem=None):