Correct token registry used as account registry, add token symbol to verify

imports: Fix stale dep in sovereign import users script

See merge request grassrootseconomics/cic-internal-integration!149

apps/cic-cache/cic_cache/cache.py

@@ -1,22 +1,28 @@
 # standard imports
 import logging
+import datetime
 
-# third-party imports
+# external imports
 import moolb
 
 # local imports
-from cic_cache.db import list_transactions_mined
-from cic_cache.db import list_transactions_account_mined
+from cic_cache.db.list import (
+        list_transactions_mined,
+        list_transactions_account_mined,
+        list_transactions_mined_with_data,
+        )
 
 logg = logging.getLogger()
 
 
-class BloomCache:
+class Cache:
 
     def __init__(self, session):
         self.session = session
 
 
+class BloomCache(Cache):
+
     @staticmethod
     def __get_filter_size(n):
         n = 8192 * 8
@@ -87,3 +93,43 @@ class BloomCache:
             f_blocktx.add(block + tx)
             logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
+
+
+class DataCache(Cache):
+
+    def load_transactions_with_data(self, offset, end):
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
+        tx_cache = []
+        highest_block = -1;
+        lowest_block = -1;
+        date_is_str = None # stick this in startup
+        for r in rows:
+            if highest_block == -1:
+                highest_block = r['block_number']
+            lowest_block = r['block_number']
+            tx_type = 'unknown'
+
+            if r['value'] != None:
+                tx_type = '{}.{}'.format(r['domain'], r['value'])
+
+            if date_is_str == None:
+                date_is_str = type(r['date_block']).__name__ == 'str'
+
+            o = {
+                'block_number': r['block_number'],
+                'tx_hash': r['tx_hash'],
+                'date_block': r['date_block'],
+                'sender': r['sender'],
+                'recipient': r['recipient'],
+                'from_value': int(r['from_value']),
+                'to_value': int(r['to_value']),
+                'source_token': r['source_token'],
+                'destination_token': r['destination_token'],
+                'tx_type': tx_type,
+            }
+
+            if date_is_str:
+                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
+
+            tx_cache.append(o)
+        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/db/list.py

@@ -28,6 +28,26 @@ def list_transactions_mined(
     return r
 
 
+def list_transactions_mined_with_data(
+        session,
+        offset,
+        end,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param offset: Offset in data set to return transactions from
+    :type offset: int
+    :param limit: Max number of transactions to retrieve
+    :type limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+
+    r = session.execute(s)
+    return r
+
+
 def list_transactions_account_mined(
         session,
         address,

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -0,0 +1,110 @@
+# standard imports
+import logging
+import json
+import re
+import base64
+
+# local imports
+from cic_cache.cache import (
+        BloomCache,
+        DataCache,
+    )
+
+logg = logging.getLogger(__name__)
+
+re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+
+DEFAULT_LIMIT = 100
+
+
+def process_transactions_account_bloom(session, env):
+    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    address = r[1]
+    if r[2] == None:
+        address = '0x' + address
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 2:
+        offset = r[3]
+    limit = 0
+    if r.lastindex > 3:
+        limit = r[4]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_bloom(session, env):
+    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 0:
+        offset = r[1]
+    limit = 0
+    if r.lastindex > 1:
+        limit = r[2]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_data(session, env):
+    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+        return None
+
+    offset = r[1]
+    end = r[2]
+    if r[2] < r[1]:
+        raise ValueError('cart before the horse, dude')
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,18 +1,20 @@
 # standard imports
 import os
-import re
 import logging
 import argparse
-import json
 import base64
 
-# third-party imports
+# external imports
 import confini
 
 # local imports
-from cic_cache import BloomCache
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
+from cic_cache.runnable.daemons.query import (
+        process_transactions_account_bloom,
+        process_transactions_all_bloom,
+        process_transactions_all_data,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -44,72 +46,6 @@ logg.debug('config:\n{}'.format(config))
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-
-DEFAULT_LIMIT = 100
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    address = r[1]
-    if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
 
 # uwsgi application
 def application(env, start_response):
@@ -119,10 +55,16 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
+            process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
             ]:
-        r = handler(session, env)
+        r = None
+        try:
+            r = handler(session, env)
+        except ValueError as e:
+            start_response('400 {}'.format(str(e)))
+            return []
         if r != None:
             (mime_type, content) = r
             break

apps/cic-cache/config/test/syncer.ini

@@ -0,0 +1,2 @@
+[syncer]
+loop_interval = 1

apps/cic-cache/tests/conftest.py

@@ -88,3 +88,16 @@ def txs(
             tx_hash_first,
             tx_hash_second,
             ]
+
+
+@pytest.fixture(scope='function')
+def tag_txs(
+        init_database,
+        txs,
+        ):
+
+    db.add_tag(init_database, 'taag', domain='test')
+    init_database.commit()
+
+    db.tag_transaction(init_database, txs[1], 'taag', domain='test')
+

apps/cic-cache/tests/test_api.py

@@ -0,0 +1,31 @@
+# standard imports
+import json
+
+# external imports
+import pytest
+
+# local imports
+from cic_cache.runnable.daemons.query import process_transactions_all_data
+
+
+def test_api_all_data(
+        init_database,
+        txs,
+        ):
+
+    env = {
+        'PATH_INFO': '/txa/410000/420000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    j = process_transactions_all_data(init_database, env)
+    o = json.loads(j[1])
+    
+    assert len(o['data']) == 2
+
+    env = {
+        'PATH_INFO': '/txa/420000/410000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+   
+    with pytest.raises(ValueError):
+        j = process_transactions_all_data(init_database, env)

apps/cic-cache/tests/test_cache.py

@@ -9,6 +9,7 @@ import pytest
 
 # local imports
 from cic_cache import BloomCache
+from cic_cache.cache import DataCache
 
 logg = logging.getLogger()
 
@@ -33,3 +34,23 @@ def test_cache(
 
     assert b[0] == list_defaults['block'] - 1
 
+
+def test_cache_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        tag_txs,
+        ):
+
+    session = init_database
+
+    c = DataCache(session)
+    b = c.load_transactions_with_data(410000, 420000)
+
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == txs[1]
+    assert b[2][1]['tx_type'] == 'unknown'
+    assert b[2][0]['tx_type'] == 'test.taag'
+    

apps/cic-eth/cic_eth/admin/token.py

@@ -16,4 +16,6 @@ def default_token(self):
     return {
             'symbol': self.default_token_symbol,
             'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
             }

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -22,6 +22,7 @@ from chainlib.eth.connection import (
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
+from cic_eth_registry.erc20 import ERC20Token
 import liveness.linux
 
 
@@ -207,6 +208,11 @@ def main():
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
+    default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
+    default_token.load(rpc)
+    BaseTask.default_token_decimals = default_token.decimals
+    BaseTask.default_token_name = default_token.name
+
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    

apps/cic-eth/cic_eth/runnable/info.py

@@ -67,6 +67,8 @@ def main():
     token_info = t.get()
     print('Default token symbol: {}'.format(token_info['symbol']))
     print('Default token address: {}'.format(token_info['address']))
+    logg.debug('Default token name: {}'.format(token_info['name']))
+    logg.debug('Default token decimals: {}'.format(token_info['decimals']))
 
     
 if __name__ == '__main__':

apps/cic-eth/cic_eth/task.py

@@ -33,6 +33,8 @@ class BaseTask(celery.Task):
     create_gas_oracle = RPCGasOracle
     default_token_address = None
     default_token_symbol = None
+    default_token_name = None
+    default_token_decimals = None
     run_dir = '/run'
 
     def create_session(self):

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.13',
+        'beta.14',
         )
 
 version_object = semver.VersionInfo(

apps/cic-notify/.config/africastalking.ini

@@ -1,4 +1,4 @@
 [AFRICASTALKING]
-api_username = foo
-api_key = bar
-api_sender_id = baz
+api_username =
+api_key =
+api_sender_id =

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -120,7 +120,7 @@ class MetadataRequestsHandler(Metadata):
         data = json.loads(response_data.decode('utf-8'))
         if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/operations.py

@@ -325,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -371,13 +371,6 @@ def process_start_menu(display_key: str, user: Account):
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -1,7 +1,7 @@
 en:
   account_successfully_created: |-
-    Hello, you have been registered on Sarafu Network! Your balance is %{balance} %{token_symbol}. To use dial *483*46#. For help 0757628885.
+    You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
     Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
   terms: |-
-    By using the service, you agree to the terms and conditions at https://www.grassrootseconomics.org/terms-and-conditions.
+    By using the service, you agree to the terms and conditions at http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -1,7 +1,7 @@
 sw:
   account_successfully_created: |-
-    Habari, umesajiliwa kwa huduma ya sarafu! Salio lako ni %{token_symbol} %{balance}. Kutumia bonyeza *483*46#. Kwa Usaidizi 0757628885.
+    Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
     Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
   terms: |-
-    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo https://www.grassrootseconomics.org/terms-and-conditions.
+    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -1,29 +1,30 @@
 en:
   kenya:
     initial_language_selection: |-
-      CON Welcome to Sarafu
+      CON Welcome to Sarafu Network
       1. English
       2. Kiswahili
       3. Help
     initial_pin_entry: |-
-      CON Please enter a PIN to manage your account.
+      CON Please enter a new four number PIN for your account. 
       0. Back
     initial_pin_confirmation: |-
-      CON Enter your PIN again
+      CON Enter your four number PIN again
       0. Back
     enter_given_name: |-
       CON Enter first name
       0. Back
     enter_family_name: |-
-      CON Enter last name
+      CON Enter family name
       0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
       2. Female
+      3. Other
       0. Back
     enter_location: |-
-      CON Enter location
+      CON Enter your location
       0. Back
     enter_products: |-
       CON Please enter a product or service you offer
@@ -83,34 +84,34 @@ en:
         Please enter your PIN to confirm.
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     display_metadata_pin_authorization:
       first: |-
-        CON Please enter your PIN.
+        CON Please enter your PIN
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances_pin_authorization:
       first: |-
-        CON Please enter your PIN to view balances.
+        CON Please enter your PIN to view balances
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_statement_pin_authorization:
       first: |-
-        CON Please enter your PIN to view statement.
+        CON Please enter your PIN to view statement
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances: |-
       CON Your balances are as follows:
         balance: %{operational_balance} %{token_symbol}
-        taxes: %{tax} %{token_symbol}
-        bonsuses: %{bonus} %{token_symbol}
+        fees: %{tax} %{token_symbol}
+        rewards: %{bonus} %{token_symbol}
       0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
@@ -140,9 +141,9 @@ en:
     exit_pin_blocked: |-
       END Your PIN has been blocked. For help, please call %{support_phone}.
     exit_invalid_pin: |-
-      END The PIN you have entered is Invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
     exit_invalid_new_pin: |-
-      END The PIN you have entered is Invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
     exit_pin_mismatch: |-
       END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
     exit_invalid_recipient: |-
@@ -169,4 +170,4 @@ en:
       00. Back
       99. Exit
     account_creation_prompt: |-
-      Your account is being created. You will receive an SMS when your account is ready.
+      Your account is being created. You will receive an SMS when your account is ready.

apps/contract-migration/scripts/README.md

@@ -89,7 +89,12 @@ After this step is run, you can find top-level ethereum addresses (like the cic
 
 
 #### Custodial provisions
-
+response_data = send_ussd_request(address, self.data_dir)
+        state = response_data[:3]
+        out = response_data[4:]
+        m = '{} {}'.format(state, out[:7])
+        if m != 'CON Welcome':
+            raise VerifierError(response_data, 'ussd')
 This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
 
 `RUN_MASK=2 docker-compose up contract-migration`
@@ -104,8 +109,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-ussd-tasker
-* cic-ussd-server
+* cic-user-tasker
+* cic-user-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,6 +174,26 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
+
+##### Importing pins and ussd data (optional)
+Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
+
+In one terminal run:
+
+`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
+
+This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the old system.
+This csv file will be stored in the pins export dir defined as the positional argument.
+
+Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
+
+- To import the pins:
+
+`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
+
+- To import ussd data:
+`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
+
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/contract-migration/scripts/cic_eth/import_users.py

@@ -171,6 +171,7 @@ if __name__ == '__main__':
                 logg.error('load error for {}: {}'.format(y, e))
                 continue
             f.close()
+            logg.debug('deserializing {} {}'.format(filepath, o))
             u = Person.deserialize(o)
 
             new_address = register_eth(i, u)

apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py

@@ -0,0 +1,76 @@
+# external imports
+from chainlib.jsonrpc import JSONRPCException
+from eth_erc20 import ERC20
+from eth_accounts_index import AccountsIndex
+from eth_token_index import TokenUniqueSymbolIndex
+
+class ERC20Token:
+
+    def __init__(self, chain_spec, address, conn):
+        self.__address = address
+
+        c = ERC20(chain_spec)
+        o = c.symbol(address)
+        r = conn.do(o)
+        self.__symbol = c.parse_symbol(r)
+
+        o = c.decimals(address)
+        r = conn.do(o)
+        self.__decimals = c.parse_decimals(r)
+
+
+    def symbol(self):
+        return self.__symbol
+
+
+    def decimals(self):
+        return self.__decimals
+
+
+class IndexCache:
+
+    def __init__(self, chain_spec, address):
+        self.address = address
+        self.chain_spec = chain_spec
+
+    
+    def parse(self, r):
+        return r
+
+
+    def get(self, conn):
+        entries = []
+        i = 0
+        while True:
+            o = self.o.entry(self.address, i)
+            try:
+                r = conn.do(o)
+                entries.append(self.parse(r, conn))
+            except JSONRPCException:
+                return entries
+            i += 1
+
+
+class AccountRegistryCache(IndexCache):
+    
+    def __init__(self, chain_spec, address):
+        super(AccountRegistryCache, self).__init__(chain_spec, address)
+        self.o = AccountsIndex(chain_spec)
+        self.get_accounts = self.get
+
+
+    def parse(self, r, conn):
+        return self.o.parse_account(r)
+
+    
+class TokenRegistryCache(IndexCache):
+
+    def __init__(self, chain_spec, address):
+        super(TokenRegistryCache, self).__init__(chain_spec, address)
+        self.o = TokenUniqueSymbolIndex(chain_spec)
+        self.get_tokens = self.get
+
+
+    def parse(self, r, conn):
+        token_address = self.o.parse_entry(r)
+        return ERC20Token(self.chain_spec, token_address, conn)

apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py

@@ -163,9 +163,9 @@ class TrafficProvisioner:
     """Aux parameter template to be passed to the traffic generator module"""
 
 
-    def __init__(self):
-        self.tokens = self.oracles['token'].get_tokens()
-        self.accounts = self.oracles['account'].get_accounts()
+    def __init__(self, conn):
+        self.tokens = self.oracles['token'].get_tokens(conn)
+        self.accounts = self.oracles['account'].get_accounts(conn)
         self.aux = copy.copy(self.default_aux)
         self.__balances = {}
         for a in self.accounts:
@@ -277,13 +277,14 @@ class TrafficSyncHandler:
     :type traffic_router: TrafficRouter
     :raises Exception: Any Exception redis may raise on connection attempt.
     """
-    def __init__(self, config, traffic_router):
+    def __init__(self, config, traffic_router, conn):
         self.traffic_router = traffic_router
         self.redis_channel = str(uuid.uuid4())
         self.pubsub = self.__connect_redis(self.redis_channel, config)
         self.traffic_items = {}
         self.config = config
         self.init = False
+        self.conn = conn
 
 
     # connects to redis
@@ -307,7 +308,7 @@ class TrafficSyncHandler:
         :param tx_index: Syncer block transaction index at time of call.
         :type tx_index: number
         """
-        traffic_provisioner = TrafficProvisioner()
+        traffic_provisioner = TrafficProvisioner(self.conn)
         traffic_provisioner.add_aux('redis_channel', self.redis_channel)
 
         refresh_accounts = None
@@ -343,7 +344,7 @@ class TrafficSyncHandler:
             sender = traffic_provisioner.accounts[sender_index]
             #balance_full = balances[sender][token_pair[0].symbol()]
             if len(sender_indices) == 1:
-                sender_indices[m] = sender_sender_indices[len(senders)-1]
+                sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
             sender_indices = sender_indices[:len(sender_indices)-1]
 
             balance_full = traffic_provisioner.balance(sender, token_pair[0])
@@ -351,7 +352,14 @@ class TrafficSyncHandler:
             recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
             recipient = traffic_provisioner.accounts[recipient_index]
           
-            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
+            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
+                traffic_item,
+                token_pair,
+                sender,
+                recipient,
+                balance_full,
+                )
+                )
             (e, t, balance_result,) = traffic_item.method(
                     token_pair,
                     sender,
@@ -359,7 +367,6 @@ class TrafficSyncHandler:
                     balance_full,
                     traffic_provisioner.aux,
                     block_number,
-                    tx_index,
                     )
             traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
             sender_indices.append(recipient_index)

apps/contract-migration/scripts/cic_eth/traffic/common/registry.py

@@ -3,7 +3,7 @@ import logging
 import copy
 
 # external imports
-from cic_registry import CICRegistry
+from cic_registry.registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
 from eth_accounts_index import AccountRegistry
 from chainlib.chain import ChainSpec

apps/contract-migration/scripts/cic_eth/traffic/common/signer.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports 
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore import DictKeystore
+from crypto_dev_signer.keystore.dict import DictKeystore
 
 logg = logging.getLogger(__name__)
 

apps/contract-migration/scripts/cic_eth/traffic/local/account.py

@@ -11,7 +11,7 @@ queue = 'cic-eth'
 name = 'account'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Triggers creation and registration of new account through the custodial cic-eth component.
 
     It expects the following aux parameters to exist:

apps/contract-migration/scripts/cic_eth/traffic/local/noop.py

@@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
 
     If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
     :type aux: dict
     :param block_number: Syncer block number position at time of method call
     :type block_number: number
-    :param tx_index: Syncer block transaction index position at time of method call
-    :type tx_index: number
     :raises KeyError: Missing required aux element
     :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
     :rtype: tuple
     """
-    logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
+    logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
 
     return (None, None, sender_balance, )

apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py

@@ -12,7 +12,7 @@ queue = 'cic-eth'
 name = 'erc20_transfer'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
 
     It expects the following aux parameters to exist:
@@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
     balance_units = int(sender_balance_value / decimals)
 
     if balance_units <= 0:
-        return (AttributeError('sender {} has zero balance'), None, 0,)
+        return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
 
     spend_units = random.randint(1, balance_units)
     spend_value = spend_units * decimals

apps/contract-migration/scripts/cic_eth/traffic/traffic.py

@@ -8,16 +8,25 @@ import json
 # external imports
 import redis
 import celery
-from chainsyncer.backend import MemBackend
+from cic_eth_registry.registry import CICRegistry
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
-from chainlib.eth.connection import HTTPConnection
-from chainlib.eth.gas import DefaultGasOracle
-from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.connection import EthHTTPConnection
+from chainlib.chain import ChainSpec
+from chainlib.eth.gas import RPCGasOracle
+from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.block import block_latest
 from hexathon import strip_0x
+from cic_base import (
+        argparse,
+        config,
+        log,
+        rpc,
+        signer as signer_funcs,
+        )
 
 # local imports
-import common
+#import common
 from cmd.traffic import (
         TrafficItem,
         TrafficRouter,
@@ -25,15 +34,19 @@ from cmd.traffic import (
         TrafficSyncHandler,
         )
 from cmd.traffic import add_args as add_traffic_args
+from cmd.cache import (
+        AccountRegistryCache,
+        TokenRegistryCache,
+        )
 
 
 # common basics
 script_dir = os.path.realpath(os.path.dirname(__file__))
-logg = common.log.create()
-argparser = common.argparse.create(script_dir, common.argparse.full_template)
-argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
-args = common.argparse.parse(argparser, logg)
-config = common.config.create(args.c, args, args.env_prefix)
+logg = log.create()
+argparser = argparse.create(script_dir, argparse.full_template)
+argparser = argparse.add(argparser, add_traffic_args, 'traffic')
+args = argparse.parse(argparser, logg)
+config = config.create(args.c, args, args.env_prefix)
 
 # map custom args to local config entries
 batchsize = args.batch_size
@@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True)
 
 config.add(args.q, '_CELERY_QUEUE', True)
 
-common.config.log(config)
+logg.debug(config)
 
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 def main():
     # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
+    (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
 
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # set up registry
-    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
-    registry = common.registry.init_legacy(config, w3)
+    rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    #registry = registry.init_legacy(config, w3)
+    CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
+    registry = CICRegistry(chain_spec, conn)
 
     # Connect to blockchain with chainlib
-    conn = HTTPConnection(config.get('ETH_PROVIDER'))
-    gas_oracle = DefaultGasOracle(conn)
-    nonce_oracle = DefaultNonceOracle(signer_address, conn)
+    gas_oracle = RPCGasOracle(conn)
+    nonce_oracle = RPCNonceOracle(signer_address, conn)
 
     # Set up magic traffic handler
     traffic_router = TrafficRouter()
     traffic_router.apply_import_dict(config.all(), config)
-    handler = TrafficSyncHandler(config, traffic_router)
+    handler = TrafficSyncHandler(config, traffic_router, conn)
 
     # Set up syncer
     syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
@@ -80,9 +96,21 @@ def main():
     block_offset = int(strip_0x(r), 16) + 1
     syncer_backend.set(block_offset, 0)
 
+    # get relevant registry entries
+    token_registry = registry.lookup('TokenRegistry')
+    logg.info('using token registry {}'.format(token_registry))
+    token_cache = TokenRegistryCache(chain_spec, token_registry)
+
+    account_registry = registry.lookup('AccountRegistry')
+    logg.info('using account registry {}'.format(account_registry))
+    account_cache = AccountRegistryCache(chain_spec, account_registry)
+   
     # Set up provisioner for common task input data
-    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.oracles['token'] = token_cache
+    TrafficProvisioner.oracles['account'] = account_cache
+    
     TrafficProvisioner.default_aux = {
             'chain_spec': config.get('CIC_CHAIN_SPEC'),
             'registry': registry,
@@ -92,7 +120,7 @@ def main():
             'api_queue': config.get('_CELERY_QUEUE'),
             }
 
-    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
+    syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
 

apps/contract-migration/scripts/cic_ussd/import_pins.py

@@ -0,0 +1,70 @@
+# standard import
+import argparse
+import csv
+import logging
+import os
+
+# third-party imports
+import celery
+import confini
+
+# local imports
+from import_task import *
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = './config'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit transaction tasks to')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('pins_dir', default='out', type=str, help='user export directory')
+args = arg_parser.parse_args()
+
+# set log levels
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+# process configs
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+
+
+def main():
+    with open(f'{args.pins_dir}/pins.csv') as pins_file:
+        phone_to_pins = [tuple(row) for row in csv.reader(pins_file)]
+
+    s_import_pins = celery.signature(
+        'import_task.set_pins',
+        (db_configs, phone_to_pins),
+        queue=args.q
+    )
+    s_import_pins.apply_async()
+
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/cic_ussd/import_task.py

@@ -8,6 +8,8 @@ import json
 
 # external imports
 import celery
+import psycopg2
+from psycopg2 import extras
 from hexathon import (
         strip_0x,
         add_0x,
@@ -53,7 +55,7 @@ class MetadataTask(ImportTask):
     def meta_url(self):
         scheme = 'http'
         if self.meta_ssl:
-            scheme += s
+            scheme += 's'
         url = urllib.parse.urlparse('{}://{}:{}/{}'.format(scheme, self.meta_host, self.meta_port, self.meta_path))
         return urllib.parse.urlunparse(url)
 
@@ -91,7 +93,6 @@ def resolve_phone(self, phone):
 def generate_metadata(self, address, phone):
     old_address = old_address_from_phone(self.import_dir, phone)
 
-    logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
     logg.debug('address {}'.format(address))
     old_address_upper = strip_0x(old_address).upper()
     metadata_path = '{}/old/{}/{}/{}.json'.format(
@@ -216,3 +217,60 @@ def send_txs(self, nonce):
 
 
     return nonce
+
+
+@celery_app.task
+def set_pins(config: dict, phone_to_pins: list):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # update db
+    for element in phone_to_pins:
+        sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
+        db_cursor.execute(sql, (element[1], element[0]))
+        logg.debug(f'Updating: {element[0]} with: {element[1]}')
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+
+
+@celery_app.task
+def set_ussd_data(config: dict, ussd_data: dict):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # process ussd_data
+    account_status = 1
+    if ussd_data['is_activated'] == 1:
+        account_status = 2
+    preferred_language = ussd_data['preferred_language']
+    phone_number = ussd_data['phone']
+
+    sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s'
+    db_cursor.execute(sql, (account_status, preferred_language, phone_number))
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+

apps/contract-migration/scripts/cic_ussd/import_users.py

@@ -87,6 +87,13 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
   
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
@@ -135,57 +142,60 @@ if __name__ == '__main__':
         for y in x[2]:
             if y[len(y)-5:] != '.json':
                 continue
-            filepath = os.path.join(x[0], y)
-            f = open(filepath, 'r')
-            try:
-                o = json.load(f)
-            except json.decoder.JSONDecodeError as e:
+            # handle json containing person object
+            filepath = None
+            if y[:15] != '_ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
                 f.close()
-                logg.error('load error for {}: {}'.format(y, e))
-                continue
-            f.close()
-            u = Person.deserialize(o)
+                u = Person.deserialize(o)
 
-            new_address = register_ussd(i, u)
+                new_address = register_ussd(i, u)
 
-            phone_object = phonenumbers.parse(u.tel)
-            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-            s_phone = celery.signature(
-                    'import_task.resolve_phone',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_phone = celery.signature(
+                        'import_task.resolve_phone',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta = celery.signature(
-                    'import_task.generate_metadata',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_meta = celery.signature(
+                        'import_task.generate_metadata',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_balance = celery.signature(
-                    'import_task.opening_balance_tx',
-                    [
-                        phone,
-                        i,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_balance = celery.signature(
+                        'import_task.opening_balance_tx',
+                        [
+                            phone,
+                            i,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta.link(s_balance)
-            s_phone.link(s_meta)
-            s_phone.apply_async(countdown=7) # block time plus a bit of time for ussd processing
+                s_meta.link(s_balance)
+                s_phone.link(s_meta)
+                # block time plus a bit of time for ussd processing
+                s_phone.apply_async(countdown=7)
 
-            i += 1
-            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
-        
-            j += 1
-            if j == batch_size:
-                time.sleep(batch_delay)
-                j = 0
+                i += 1
+                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+
+                j += 1
+                if j == batch_size:
+                    time.sleep(batch_delay)
+                    j = 0
 
-    #fi.close()

apps/contract-migration/scripts/cic_ussd/import_ussd_data.py

@@ -0,0 +1,70 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# external imports
+import celery
+from confini import Config
+
+# local imports
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+if __name__ == '__main__':
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # handle ussd_data json object
+            if y[:15] == '_ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    ussd_data = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+
+                s_set_ussd_data = celery.signature(
+                    'import_task.set_ussd_data',
+                    [db_configs, ussd_data]
+                )
+                s_set_ussd_data.apply_async(queue='cic-import-ussd')

apps/contract-migration/scripts/config/traffic.ini

@@ -1,4 +1,4 @@
 [traffic]
 #local.noop_traffic = 2
-local.account = 2
+#local.account = 2
 local.transfer = 2

apps/contract-migration/scripts/create_import_pins.py

@@ -0,0 +1,90 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# third-party imports
+import bcrypt
+import celery
+import confini
+import phonenumbers
+import random
+from cic_types.models.person import Person
+from cryptography.fernet import Fernet
+
+# local imports
+
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+default_config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='Config dir')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('--userdir', type=str, help='path to users export dir tree')
+arg_parser.add_argument('pins_dir', type=str, help='path to pin export dir tree')
+
+
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+logg.info('loaded config\n{}'.format(config))
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+user_dir = args.userdir
+pins_dir = args.pins_dir
+
+
+def generate_password_hash():
+    key = Fernet.generate_key()
+    fnt = Fernet(key)
+    pin = str(random.randint(1000, 9999))
+    return fnt.encrypt(bcrypt.hashpw(pin.encode('utf-8'), bcrypt.gensalt())).decode()
+
+
+user_old_dir = os.path.join(user_dir, 'old')
+logg.debug(f'reading user data from: {user_old_dir}')
+
+pins_file = open(f'{pins_dir}/pins.csv', 'w')
+
+if __name__ == '__main__':
+
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            # skip non-json files
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # define file path for
+            filepath = None
+            if y[:15] != '_ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+                u = Person.deserialize(o)
+
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                password_hash = generate_password_hash()
+                pins_file.write(f'{phone},{password_hash}\n')
+                logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
+
+    pins_file.close()

apps/contract-migration/scripts/create_import_users.py

@@ -105,7 +105,7 @@ def genId(addr, typ):
 def genDate():
 
     ts = random.randint(ts_then, ts_now)
-    return datetime.datetime.fromtimestamp(ts).timestamp()
+    return int(datetime.datetime.fromtimestamp(ts).timestamp())
 
 
 def genPhone():
@@ -130,6 +130,7 @@ def genCats():
 def genAmount():
     return random.randint(0, gift_max) * gift_factor
 
+
 def genDob():
     dob_src = fake.date_of_birth(minimum_age=15)
     dob = {}
@@ -168,8 +169,9 @@ def gen():
             }
     p.location['area_name'] = city
     if random.randint(0, 1):
-        p.identities['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
-        p.identities['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+        p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
+        p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+
     
     return (old_blockchain_checksum_address, phone, p)
 
@@ -191,6 +193,7 @@ def prepareLocalFilePath(datadir, address):
 if __name__ == '__main__':
 
     base_dir = os.path.join(user_dir, 'old')
+    ussd_dir = os.path.join(user_dir, 'ussd')
     os.makedirs(base_dir, exist_ok=True)
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
@@ -210,11 +213,23 @@ if __name__ == '__main__':
 
         print(o)
 
+        ussd_data = {
+            'phone': phone,
+            'is_activated': 1,
+            'preferred_language': random.sample(['en', 'sw'], 1)[0],
+            'is_disabled': False
+        }
+
         d = prepareLocalFilePath(base_dir, uid)
         f = open('{}/{}'.format(d, uid + '.json'), 'w')
         json.dump(o.serialize(), f)
         f.close()
 
+        d = prepareLocalFilePath(ussd_dir, uid)
+        x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w')
+        json.dump(ussd_data, x)
+        x.close()
+
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/contract-migration/scripts/eth/import_users.py

@@ -23,7 +23,7 @@ from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.gas import RPCGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from cic_types.processor import generate_metadata_pointer
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
 from eth_contract_registry import Registry
 from crypto_dev_signer.keystore.dict import DictKeystore
 from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b9
+cic-base[full_graph]==0.1.2b11
 sarafu-faucet==0.0.3a3
-cic-eth==0.11.0b13
+cic-eth==0.11.0b14
 cic-types==0.1.0a11
 crypto-dev-signer==0.4.14b3

apps/contract-migration/scripts/verify.py

@@ -1,52 +1,37 @@
 # standard imports
+import argparse
+import copy
+import hashlib
+import json
+import logging
 import os
 import sys
-import logging
-import time
-import argparse
-import sys
-import re
-import hashlib
-import csv
-import json
 import urllib
-import copy
-import uuid
 import urllib.request
+import uuid
 
 # external imports
 import celery
-import eth_abi
 import confini
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+import eth_abi
 from chainlib.chain import ChainSpec
+from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainlib.hash import keccak256_string_to_hex
-from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import (
-        OverrideGasOracle,
-        balance,
-        )
+    OverrideGasOracle,
+    balance,
+)
 from chainlib.eth.tx import TxFactory
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.jsonrpc import jsonrpc_template
-from chainlib.eth.error import EthException
 from cic_types.models.person import (
-        Person,
-        generate_metadata_pointer,
-        )
+    Person,
+    generate_metadata_pointer,
+)
 from erc20_faucet import Faucet
 from eth_erc20 import ERC20
+from hexathon.parse import strip_0x, add_0x
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -72,6 +57,7 @@ eth_tests = [
 
 phone_tests = [
         'ussd',
+        'ussd_pins'
         ]
 
 all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
@@ -86,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@@ -115,6 +102,8 @@ config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
 
+token_symbol = args.token_symbol
+
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
@@ -171,6 +160,39 @@ if logg.isEnabledFor(logging.DEBUG):
     outfunc = logg.debug
 
 
+def send_ussd_request(address, data_dir):
+    upper_address = strip_0x(address).upper()
+    f = open(os.path.join(
+        data_dir,
+        'new',
+        upper_address[:2],
+        upper_address[2:4],
+        upper_address + '.json',
+    ), 'r'
+    )
+    o = json.load(f)
+    f.close()
+
+    p = Person.deserialize(o)
+    phone = p.tel
+
+    session = uuid.uuid4().hex
+    data = {
+        'sessionId': session,
+        'serviceCode': config.get('APP_SERVICE_CODE'),
+        'phoneNumber': phone,
+        'text': '',
+    }
+
+    req = urllib.request.Request(config.get('_USSD_PROVIDER'))
+    data_str = json.dumps(data)
+    data_bytes = data_str.encode('utf-8')
+    req.add_header('Content-Type', 'application/json')
+    req.data = data_bytes
+    response = urllib.request.urlopen(req)
+    return response.read().decode('utf-8')
+
+
 class VerifierState:
 
     def __init__(self, item_keys, active_tests=None):
@@ -254,7 +276,10 @@ class Verifier:
     def verify_balance(self, address, balance):
         o = self.erc20_tx_factory.balance(self.token_address, address)
         r = self.conn.do(o)
-        actual_balance = int(strip_0x(r), 16)
+        try:
+            actual_balance = int(strip_0x(r), 16)
+        except ValueError:
+            actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
@@ -354,42 +379,18 @@ class Verifier:
 
 
     def verify_ussd(self, address, balance=None):
-        upper_address = strip_0x(address).upper()
-        f = open(os.path.join(
-            self.data_dir,
-            'new',
-            upper_address[:2],
-            upper_address[2:4],
-            upper_address + '.json',
-            ), 'r'
-            )
-        o = json.load(f)
-        f.close()
-
-        p = Person.deserialize(o)
-        phone = p.tel
-
-        session = uuid.uuid4().hex
-        data = {
-                'sessionId': session,
-                'serviceCode': config.get('APP_SERVICE_CODE'),
-                'phoneNumber': phone,
-                'text': config.get('APP_SERVICE_CODE'),
-            }
-
-        req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-        data_str = json.dumps(data)
-        data_bytes = data_str.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
-        req.data = data_bytes
-        response = urllib.request.urlopen(req)
-        response_data = response.read().decode('utf-8')
+        response_data = send_ussd_request(address, self.data_dir)
         state = response_data[:3]
         out = response_data[4:]
         m = '{} {}'.format(state, out[:7])
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
+    def verify_ussd_pins(self, address, balance):
+        response_data = send_ussd_request(address, self.data_dir)
+        if response_data[:11] != 'CON Balance':
+            raise VerifierError(response_data, 'pins')
+
 
     def verify(self, address, balance, debug_stem=None):
   
@@ -466,7 +467,7 @@ def main():
     tx = txf.template(ZERO_ADDRESS, token_index_address)
     data = add_0x(registry_addressof_method)
     h = hashlib.new('sha256')
-    h.update(b'SRF')
+    h.update(token_symbol.encode('utf-8'))
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)