Bump cic-eth in cic-ussd

Adapt deployment to Bloxberg

See merge request grassrootseconomics/cic-internal-integration!99

.gitignore

@@ -1,2 +1,6 @@
 service-configs/*
 !service-configs/.gitkeep
+**/node_modules/
+__pycache__
+*.pyc
+*.o

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -67,6 +67,7 @@ class ERC20TransferFilter(SyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        db_session.flush()
+        #db_session.flush()
+        db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -26,9 +26,10 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend import SyncerBackend
+from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver import (
         HeadSyncer,
+        HistorySyncer,
         )
 from chainsyncer.db.models.base import SessionBase
 
@@ -70,19 +71,21 @@ def main():
 
     syncers = []
 
-    #if SyncerBackend.first(chain_spec):
-    #    backend = SyncerBackend.initial(chain_spec, block_offset)
-    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
+    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
+    for syncer_backend in syncer_backends:
+        syncers.append(HistorySyncer(syncer_backend))
 
+    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
     syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')

apps/cic-cache/docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a58
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \
@@ -47,6 +47,9 @@ RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
 	mkdir -p /usr/local/share/cic/solidity && \
 	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
 
+COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
+COPY cic-cache/docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server

apps/cic-cache/docker/db.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+>&2 echo executing database migration
+python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
+set +e

apps/cic-cache/docker/start_tracker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. ./db.sh
+
+/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/requirements.txt

@@ -1,13 +1,12 @@
-cic-base~=0.1.2a58
+cic-base~=0.1.2a77
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.4a10
+cic-eth-registry~=0.5.4a16
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainlib~=0.0.2a2
-chainsyncer~=0.0.1a21
+chainsyncer[sql]~=0.0.2a2

apps/cic-eth/cic_eth/db/models/base.py

@@ -10,6 +10,7 @@ from sqlalchemy.pool import (
         StaticPool,
         QueuePool,
         AssertionPool,
+        NullPool,
         )
 
 logg = logging.getLogger()
@@ -64,6 +65,7 @@ class SessionBase(Model):
         if SessionBase.poolable:
             poolclass = QueuePool
             if pool_size > 1:
+                logg.info('db using queue pool')
                 e = create_engine(
                         dsn,
                         max_overflow=pool_size*3,
@@ -74,17 +76,22 @@ class SessionBase(Model):
                         echo=debug,
                     )
             else:
-                if debug:
+                if pool_size == 0:
+                    logg.info('db using nullpool')
+                    poolclass = NullPool
+                elif debug:
+                    logg.info('db using assertion pool')
                     poolclass = AssertionPool
                 else:
+                    logg.info('db using static pool')
                     poolclass = StaticPool
-
                 e = create_engine(
                         dsn,
                         poolclass=poolclass,
                         echo=debug,
                     )
         else:
+            logg.info('db not poolable')
             e = create_engine(
                     dsn,
                     echo=debug,

apps/cic-eth/cic_eth/eth/meta.py

@@ -1,6 +1,10 @@
-# extended imports
+# external imports
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
+from cic_eth_registry.erc20 import ERC20Token
+
+# local imports
+from cic_eth.ext.address import translate_address
 
 
 class ExtendedTx:
@@ -27,12 +31,12 @@ class ExtendedTx:
         self.status_code = TxStatus.PENDING.value
 
 
-    def set_actors(self, sender, recipient, trusted_declarator_addresses=None):
+    def set_actors(self, sender, recipient, trusted_declarator_addresses=None, caller_address=ZERO_ADDRESS):
         self.sender = sender
         self.recipient = recipient
         if trusted_declarator_addresses != None:
-            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain_spec)
-            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain_spec)
+            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain_spec, sender_address=caller_address)
+            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain_spec, sender_address=caller_address)
 
 
     def set_tokens(self, source, source_value, destination=None, destination_value=None):
@@ -40,8 +44,8 @@ class ExtendedTx:
             destination = source
         if destination_value == None:
             destination_value = source_value
-        st = ERC20Token(self.rpc, source)
-        dt = ERC20Token(self.rpc, destination)
+        st = ERC20Token(self.chain_spec, self.rpc, source)
+        dt = ERC20Token(self.chain_spec, self.rpc, destination)
         self.source_token = source
         self.source_token_symbol = st.symbol
         self.source_token_name = st.name
@@ -62,10 +66,10 @@ class ExtendedTx:
         self.status_code = n
 
 
-    def to_dict(self):
+    def asdict(self):
         o = {}
         for attr in dir(self):
-            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'to_dict']:
+            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'asdict', 'rpc']:
                 continue
             o[attr] = getattr(self, attr)
         return o 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -15,7 +15,6 @@ from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
 from chainlib.connection import RPCConnection
-from chainsyncer.error import SyncDone
 from hexathon import strip_0x
 from chainqueue.db.enum import (
     StatusEnum,
@@ -153,10 +152,7 @@ class DispatchSyncer:
 def main(): 
     syncer = DispatchSyncer(chain_spec)
     conn = RPCConnection.connect(chain_spec, 'default')
-    try:
-        syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
-    except SyncDone as e:
-        sys.stderr.write("dispatcher done at block {}\n".format(e))
+    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -1,7 +1,7 @@
 # standard imports
 import logging
 
-# third-party imports
+# external imports
 import celery
 from cic_eth_registry.error import UnknownContractError
 from chainlib.status import Status as TxStatus
@@ -9,7 +9,13 @@ from chainlib.eth.address import to_checksum_address
 from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.erc20 import ERC20
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+# TODO: use sarafu_Faucet for both when inheritance has been implemented
+from erc20_single_shot_faucet import SingleShotFaucet
+from sarafu_faucet import MinterFaucet as Faucet
 
 # local imports
 from .base import SyncFilter
@@ -18,65 +24,73 @@ from cic_eth.eth.meta import ExtendedTx
 logg = logging.getLogger().getChild(__name__)
 
 
-def parse_transfer(tx):
-    r = ERC20.parse_transfer_request(tx.payload)
-    transfer_data = {}
-    transfer_data['to'] = r[0]
-    transfer_data['value'] = r[1]
-    transfer_data['from'] = tx['from']
-    transfer_data['token_address'] = tx['to']
-    return ('transfer', transfer_data)
-
-
-def parse_transferfrom(tx):
-    r = ERC20.parse_transfer_request(tx.payload)
-    transfer_data = unpack_transferfrom(tx.payload)
-    transfer_data['from'] = r[0]
-    transfer_data['to'] = r[1]
-    transfer_data['value'] = r[2]
-    transfer_data['token_address'] = tx['to']
-    return ('transferfrom', transfer_data)
-
-
-def parse_giftto(tx):
-    # TODO: broken
-    logg.error('broken')
-    return
-    transfer_data = unpack_gift(tx.payload)
-    transfer_data['from'] = tx.inputs[0]
-    transfer_data['value'] = 0
-    transfer_data['token_address'] = ZERO_ADDRESS
-    # TODO: would be better to query the gift amount from the block state
-    for l in tx.logs:
-        topics = l['topics']
-        logg.debug('topixx {}'.format(topics))
-        if strip_0x(topics[0]) == '45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
-            #transfer_data['value'] = web3.Web3.toInt(hexstr=strip_0x(l['data']))
-            transfer_data['value'] = int.from_bytes(bytes.fromhex(strip_0x(l_data)))
-            #token_address_bytes = topics[2][32-20:]
-            token_address = strip_0x(topics[2])[64-40:]
-            transfer_data['token_address'] = to_checksum_address(token_address)
-    return ('tokengift', transfer_data)
-
 
 class CallbackFilter(SyncFilter):
 
     trusted_addresses = []
 
-    def __init__(self, chain_spec, method, queue):
+    def __init__(self, chain_spec, method, queue, caller_address=ZERO_ADDRESS):
         self.queue = queue
         self.method = method
         self.chain_spec = chain_spec
+        self.caller_address = caller_address
+
+
+    def parse_transfer(self, tx, conn):
+        if not tx.payload:
+            return (None, None)
+        r = ERC20.parse_transfer_request(tx.payload)
+        transfer_data = {}
+        transfer_data['to'] = r[0]
+        transfer_data['value'] = r[1]
+        transfer_data['from'] = tx.outputs[0]
+        transfer_data['token_address'] = tx.inputs[0]
+        return ('transfer', transfer_data)
+
+
+    def parse_transferfrom(self, tx, conn):
+        if not tx.payload:
+            return (None, None)
+        r = ERC20.parse_transfer_from_request(tx.payload)
+        transfer_data = {}
+        transfer_data['from'] = r[0]
+        transfer_data['to'] = r[1]
+        transfer_data['value'] = r[2]
+        transfer_data['token_address'] = tx.inputs[0]
+        return ('transferfrom', transfer_data)
+
+
+    def parse_giftto(self, tx, conn):
+        if not tx.payload:
+            return (None, None)
+        r = Faucet.parse_give_to_request(tx.payload)
+        transfer_data = {}
+        transfer_data['to'] = r[0]
+        transfer_data['value'] = tx.value
+        transfer_data['from'] = tx.outputs[0]
+        #transfer_data['token_address'] = tx.inputs[0]
+        faucet_contract = tx.inputs[0]
+
+        c = SingleShotFaucet(self.chain_spec)
+        o = c.token(faucet_contract, sender_address=self.caller_address)
+        r = conn.do(o)
+        transfer_data['token_address'] = add_0x(c.parse_token(r))
+
+        o = c.amount(faucet_contract, sender_address=self.caller_address)
+        r = conn.do(o)
+        transfer_data['value'] = c.parse_amount(r)
+
+        return ('tokengift', transfer_data)
 
 
     def call_back(self, transfer_type, result):
-        logg.debug('result {}'.format(result))
+        result['chain_spec'] = result['chain_spec'].asdict()
         s = celery.signature(
             self.method,
             [
                 result,
                 transfer_type,
-                int(result['status_code'] == 0),
+                int(result['status_code'] != 0),
             ],
             queue=self.queue,
             )
@@ -92,26 +106,29 @@ class CallbackFilter(SyncFilter):
 #        s_translate.link(s)
 #        s_translate.apply_async()
         t = s.apply_async()
-        return s
+        return t
 
 
-    def parse_data(self, tx):
+    def parse_data(self, tx, conn):
         transfer_type = None
         transfer_data = None
         # TODO: what's with the mix of attributes and dict keys
         logg.debug('have payload {}'.format(tx.payload))
-        method_signature = tx.payload[:8]
 
         logg.debug('tx status {}'.format(tx.status))
 
         for parser in [
-                parse_transfer,
-                parse_transferfrom,
-                parse_giftto,
+                self.parse_transfer,
+                self.parse_transferfrom,
+                self.parse_giftto,
                 ]:
             try:
-                (transfer_type, transfer_data) = parser(tx)
-                break
+                if tx:
+                    (transfer_type, transfer_data) = parser(tx, conn)
+                    if transfer_type == None:
+                        continue
+                else:
+                    pass
             except RequestMismatchException:
                 continue
 
@@ -128,7 +145,7 @@ class CallbackFilter(SyncFilter):
         transfer_data = None
         transfer_type = None
         try:
-            (transfer_type, transfer_data) = self.parse_data(tx)
+            (transfer_type, transfer_data) = self.parse_data(tx, conn)
         except TypeError:
             logg.debug('invalid method data length for tx {}'.format(tx.hash))
             return
@@ -144,16 +161,17 @@ class CallbackFilter(SyncFilter):
             result = None
             try:
                 tokentx = ExtendedTx(conn, tx.hash, self.chain_spec)
-                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses)
+                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses, caller_address=self.caller_address)
                 tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
                 if transfer_data['status'] == 0:
                     tokentx.set_status(1)
                 else:
                     tokentx.set_status(0)
-                t = self.call_back(transfer_type, tokentx.to_dict())
-                logg.info('callback success task id {} tx {}'.format(t, tx.hash))
+                result = tokentx.asdict()
+                t = self.call_back(transfer_type, result)
+                logg.info('callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue))
             except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc.queue, tc.method, transfer_data['to'], tx.hash))
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tx.queue, tx.method, transfer_data['to'], tx.hash))
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/retry.py

@@ -20,6 +20,7 @@ from cic_eth.admin.ctrl import lock_send
 from cic_eth.db.enum import LockEnum
 from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
 from cic_eth.sync.retry import RetrySyncer
+from cic_eth.stat import init_chain_stat
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -71,57 +72,21 @@ RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='def
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
-
-## TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here
-#def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
-#    tx_dict = get_tx(tx_hash)
-#    tx = unpack(tx_dict['signed_tx'], chain_spec)
-#    logg.debug('submitting tx {} for retry'.format(tx_hash))
-#    s_check = celery.signature(
-#            'cic_eth.admin.ctrl.check_lock',
-#            [
-#                tx_hash,
-#                chain_str,
-#                LockEnum.QUEUE,
-#                tx['from'],
-#                ],
-#            queue=queue,
-#            )
-##    s_resume = celery.signature(
-##            'cic_eth.eth.tx.resume_tx',
-##            [
-##                chain_str,
-##                ],
-##            queue=queue,
-##            )
-#    
-##    s_retry_status = celery.signature(
-##            'cic_eth.queue.state.set_ready',
-##            [],
-##            queue=queue,
-##    )
-#    s_resend = celery.signature(
-#            'cic_eth.eth.gas.resend_with_higher_gas',
-#            [
-#                chain_str,
-#                ],
-#            queue=queue,
-#            )
-#
-#    #s_resume.link(s_retry_status)
-#    #s_check.link(s_resume)
-#    s_check.link(s_resend)
-#    s_check.apply_async()
-
 
 def main(): 
     conn = RPCConnection.connect(chain_spec, 'default')
+
+    straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
+    loop_interval = config.get('SYNCER_LOOP_INTERVAL')
+    if loop_interval == None:
+        stat = init_chain_stat(conn)
+        loop_interval = stat.block_average()
+
     syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE'))
     syncer.backend.set(0, 0)
     fltr = StragglerFilter(chain_spec, queue=queue)
     syncer.add_filter(fltr)
-    syncer.loop(float(straggler_delay), conn)
+    syncer.loop(int(loop_interval), conn)
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -39,6 +39,7 @@ from cic_eth.queue import (
 from cic_eth.callbacks import (
         Callback,
         http,
+        noop,
         #tcp,
         redis,
         )
@@ -91,7 +92,7 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=50, debug=config.true('DATABASE_DEBUG'))
+SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 
 # verify database connection with minimal sanity query
 session = SessionBase.create_session()

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -7,7 +7,7 @@ import argparse
 import sys
 import re
 
-# third-party imports
+# external imports
 import confini
 import celery
 import rlp
@@ -15,7 +15,6 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
-from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,7 +25,7 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend import SyncerBackend
+from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver import (
         HeadSyncer,
         HistorySyncer,
@@ -42,6 +41,13 @@ from cic_eth.runnable.daemons.filters import (
         RegistrationFilter,
         TransferAuthFilter,
         )
+from cic_eth.stat import init_chain_stat
+from cic_eth.registry import (
+        connect as connect_registry,
+        connect_declarator,
+        connect_token_registry,
+        )
+
 
 script_dir = os.path.realpath(os.path.dirname(__file__))
 
@@ -78,24 +84,27 @@ def main():
     block_current = int(r, 16)
     block_offset = block_current + 1
 
-    stat = init_chain_stat(rpc, block_current)
+    loop_interval = config.get('SYNCER_LOOP_INTERVAL')
+    if loop_interval == None:
+        stat = init_chain_stat(rpc, block_start=block_current)
+        loop_interval = stat.block_average()
 
     logg.debug('starting at block {}'.format(block_offset))
 
     syncers = []
 
-    #if SyncerBackend.first(chain_spec):
-    #    backend = SyncerBackend.initial(chain_spec, block_offset)
-    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
+    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
+    syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1))
 
     for syncer_backend in syncer_backends:
         try:
@@ -105,6 +114,8 @@ def main():
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
             syncers.append(HeadSyncer(syncer_backend))
 
+    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -112,6 +123,8 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
+    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_token_registry(rpc, chain_spec)
     CallbackFilter.trusted_addresses = trusted_addresses
 
     callback_filters = []
@@ -142,7 +155,8 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
+        #r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
+        r = syncer.loop(int(loop_interval), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 
         i += 1

apps/cic-eth/cic_eth/stat.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.stat import ChainStat
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+
+logg = logging.getLogger().getChild(__name__)
+
+BLOCK_SAMPLES = 10
+
+
+def init_chain_stat(rpc, block_start=0):
+    stat = ChainStat()
+
+    if block_start == 0:
+        o = block_latest()
+        r = rpc.do(o)
+        block_start = int(r, 16)
+
+    for i in range(BLOCK_SAMPLES):
+        o = block_by_number(block_start-10+i)
+        block_src = rpc.do(o)
+        logg.debug('block {}'.format(block_src))
+        block = Block(block_src)
+        stat.block_apply(block)
+
+    logg.debug('calculated block time {} from {} block samples'.format(stat.block_average(), BLOCK_SAMPLES))
+    return stat

apps/cic-eth/cic_eth/sync/retry.py

@@ -4,7 +4,7 @@ import datetime
 
 # external imports
 from chainsyncer.driver import HeadSyncer
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.error import NoBlockForYou
 from chainlib.eth.block import (
         block_by_number,

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.1',
+        'beta.7',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/database.ini

@@ -6,4 +6,5 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
+POOL_SIZE=50
 DEBUG=0

apps/cic-eth/config/docker/cic.ini

@@ -2,3 +2,4 @@
 registry_address = 
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+tx_retry_delay = 20

apps/cic-eth/config/docker/database.ini

@@ -6,4 +6,5 @@ HOST=localhost
 PORT=63432
 ENGINE=postgresql
 DRIVER=psycopg2
+POOL_SIZE=50
 DEBUG=0

apps/cic-eth/config/docker/syncer.ini

@@ -1,2 +1,2 @@
 [SYNCER]
-loop_interval = 1
+loop_interval = 

apps/cic-eth/config/syncer.ini

@@ -1,2 +1,2 @@
 [SYNCER]
-loop_interval = 1
+loop_interval = 

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a61
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base~=0.1.2a60
+cic-base~=0.1.2a76
 celery==4.4.7
-crypto-dev-signer~=0.4.14a17
+crypto-dev-signer~=0.4.14b2
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a12
+cic-eth-registry~=0.5.4a16
 #cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a7
-erc20-transfer-authorization~=0.3.1a3
-#simple-rlp==0.1.2
+eth_accounts_index~=0.0.11a9
+erc20-transfer-authorization~=0.3.1a5
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a7
-chainlib~=0.0.2a4
+eth-address-index~=0.1.1a9
+chainlib~=0.0.2a13
 hexathon~=0.0.1a7
-chainsyncer~=0.0.1a21
+chainsyncer[sql]~=0.0.2a2
 chainqueue~=0.0.1a7
 pysha3==1.0.2
 coincurve==15.0.0
-sarafu-faucet~=0.0.2a19
+sarafu-faucet==0.0.2a28
+potaahto~=0.0.1a1

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,4 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a4
+giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -0,0 +1,223 @@
+# standard import
+import logging
+import datetime
+import os
+
+# external imports
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        transaction,
+        Tx,
+        )
+from chainlib.eth.block import Block
+from chainlib.eth.erc20 import ERC20
+from sarafu_faucet import MinterFaucet
+from eth_accounts_index import AccountRegistry
+from potaahto.symbols import snake_and_camel
+from hexathon import add_0x
+
+# local imports
+from cic_eth.runnable.daemons.filters.callback import CallbackFilter
+
+logg = logging.getLogger()
+
+
+@pytest.mark.skip()
+def test_transfer_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        token_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    logg.debug(r)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = fltr.parse_transfer(tx, eth_rpc)
+
+    assert transfer_type == 'transfer'
+
+
+@pytest.mark.skip()
+def test_transfer_from_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        token_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+
+    (tx_hash_hex, o) = txf.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer_from(foo_token, agent_roles['ALICE'], token_roles['FOO_TOKEN_OWNER'], agent_roles['BOB'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = fltr.parse_transferfrom(tx, eth_rpc)
+
+    assert transfer_type == 'transferfrom'
+
+
+def test_faucet_gift_to_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        contract_roles,
+        faucet,
+        account_registry,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=800000)
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], rpc)
+    txf = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+   
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
+    txf = MinterFaucet(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.give_to(faucet, agent_roles['ALICE'], agent_roles['ALICE'])
+    r = rpc.do(o)
+
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = fltr.parse_giftto(tx, eth_rpc)
+
+    assert transfer_type == 'tokengift'
+    assert transfer_data['token_address'] == foo_token
+
+
+def test_callback_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        token_roles,
+        agent_roles,
+        contract_roles,
+        register_lookups,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    logg.debug(r)
+
+    mockblock_src = {
+        'hash': add_0x(os.urandom(32).hex()),
+        'number': '0x2a',
+        'transactions': [tx_hash_hex],
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+            }
+    mockblock = Block(mockblock_src)
+
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src, block=mockblock)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    class CallbackMock:
+
+        def __init__(self):
+            self.results = {}
+
+        def call_back(self, transfer_type, result):
+            self.results[transfer_type] = result
+
+    mock = CallbackMock()
+    fltr.call_back = mock.call_back
+
+    fltr.filter(eth_rpc, mockblock, tx, init_database)
+
+    assert mock.results.get('transfer') != None
+    assert mock.results['transfer']['destination_token'] == foo_token

apps/cic-meta/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.3",
+	"version": "0.0.7-alpha.6",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
@@ -40,6 +40,6 @@
 	],
 	"license": "GPL-3.0-or-later",
 	"engines": {
-		"node": "~15.3.0"
+		"node": "~14.16.1"
 	}
 }

apps/cic-meta/scripts/server/server.ts

@@ -114,6 +114,7 @@ async function processRequest(req, res) {
 		return;
 	}
 
+
 	if (!['PUT', 'GET', 'POST'].includes(req.method)) {
 		res.writeHead(405, {"Content-Type": "text/plain"});
 		res.end();
@@ -123,6 +124,7 @@ async function processRequest(req, res) {
 	try {
 		digest = parseDigest(req.url);
 	} catch(e) {
+		console.error('digest error: ' + e)
 		res.writeHead(400, {"Content-Type": "text/plain"});
 		res.end();
 		return;

apps/cic-meta/src/assets/phone.ts

@@ -1,12 +1,12 @@
 import { ArgPair, Syncable } from '../sync';
-import { Addressable, addressToBytes, bytesToHex, toKey } from '../digest';
+import { Addressable, mergeKey } from '../digest';
 
 class Phone extends Syncable implements Addressable {
 
 	address:	string
 	value:		number
 
-	constructor(address:string, v:number) {
+	constructor(address:string, v:string) {
 		const o = {
 			msisdn: v,
 		}
@@ -17,8 +17,8 @@ class Phone extends Syncable implements Addressable {
 		});
 	}
 
-	public static async toKey(msisdn:number) {
-		return await toKey(msisdn.toString(), ':cic.msisdn');
+	public static async toKey(msisdn:string) {
+		return await mergeKey(Buffer.from(msisdn), Buffer.from(':cic.phone'));
 	}
 
 	public key(): string {

apps/cic-meta/src/digest.ts

@@ -61,6 +61,7 @@ function addressToBytes(s:string) {
 export {
 	toKey,
 	toAddressKey,
+	mergeKey,
 	bytesToHex,
 	addressToBytes,
 	Addressable,

apps/cic-notify/cic_notify/api.py

@@ -3,6 +3,7 @@ import logging
 import re
 
 # third-party imports
+from celery.app.control import Inspect
 import celery
 
 # local imports
@@ -15,6 +16,29 @@ logg = logging.getLogger()
 sms_tasks_matcher = r"^(cic_notify.tasks.sms)(\.\w+)?"
 
 
+re_q = r'^cic-notify'
+def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
+    host_queues = []
+
+    i = Inspect(app=app)
+    qs = i.active_queues()
+    for host in qs.keys():
+        for q in qs[host]:
+            if re.match(re_q, q['name']):
+                host_queues.append((host, q['name'],))
+   
+    task_prefix_len = len(task_prefix)
+    queue_tasks = []
+    for (host, queue) in host_queues:
+        i = Inspect(app=app, destination=[host])
+        for tasks in i.registered_tasks().values():
+            for task in tasks:
+                if len(task) >= task_prefix_len and task[:task_prefix_len] == task_prefix:
+                    queue_tasks.append((queue, task,))
+    
+    return queue_tasks
+
+
 class Api:
     # TODO: Implement callback strategy
     def __init__(self, queue='cic-notify'):
@@ -22,17 +46,9 @@ class Api:
         :param queue: The queue on which to execute notification tasks
         :type queue: str
         """
-        registered_tasks = app.tasks
-        self.sms_tasks = []
+        self.sms_tasks = get_sms_queue_tasks(app)
+        logg.debug('sms tasks {}'.format(self.sms_tasks))
 
-        for task in registered_tasks.keys():
-            logg.debug(f'Found: {task} {registered_tasks[task]}')
-            match = re.match(sms_tasks_matcher, task)
-            if match:
-                self.sms_tasks.append(task)
-
-        self.queue = queue
-        logg.info(f'api using queue: {self.queue}')
 
     def sms(self, message, recipient):
         """This function chains all sms tasks in order to send a message, log and persist said data to disk
@@ -44,12 +60,17 @@ class Api:
         :rtype: Celery.Task
         """
         signatures = []
-        for task in self.sms_tasks:
-            signature = celery.signature(task)
+        for q in self.sms_tasks:
+            signature = celery.signature(
+                    q[1],
+                    [
+                        message,
+                        recipient,
+                        ],
+                    queue=q[0],
+                    )
             signatures.append(signature)
-            signature_group = celery.group(signatures)
-            result = signature_group.apply_async(
-                args=[message, recipient],
-                queue=self.queue
-            )
-            return result
+
+        t = celery.group(signatures)()
+
+        return t

apps/cic-notify/cic_notify/runnable/send.py

@@ -0,0 +1,76 @@
+# standard imports
+import sys
+import os
+import logging
+import argparse
+import tempfile
+
+# external imports
+import celery
+import confini
+
+# local imports
+from cic_notify.api import Api
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_dir = os.path.join('/usr/local/etc/cic-notify')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+argparser.add_argument('recipient', type=str, help='notification recipient')
+argparser.add_argument('message', type=str, help='message text')
+args = argparser.parse_args()
+
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.add(args.recipient, '_RECIPIENT', True)
+config.add(args.message, '_MESSAGE', True)
+
+# set up celery
+app = celery.Celery(__name__)
+
+broker = config.get('CELERY_BROKER_URL')
+if broker[:4] == 'file':
+    bq = tempfile.mkdtemp()
+    bp = tempfile.mkdtemp()
+    app.conf.update({
+            'broker_url': broker,
+            'broker_transport_options': {
+                'data_folder_in': bq,
+                'data_folder_out': bq,
+                'data_folder_processed': bp,
+            },
+            },
+            )
+    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
+else:
+    app.conf.update({
+        'broker_url': broker,
+        })
+
+result = config.get('CELERY_RESULT_URL')
+if result[:4] == 'file':
+    rq = tempfile.mkdtemp()
+    app.conf.update({
+        'result_backend': 'file://{}'.format(rq),
+        })
+    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
+else:
+    app.conf.update({
+        'result_backend': result,
+        })
+
+if __name__ == '__main__':
+    a = Api()
+    t = a.sms(config.get('_RECIPIENT'), config.get('_MESSAGE'))

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -33,7 +33,9 @@ elif args.v:
 
 config = confini.Config(args.c, args.env_prefix)
 config.process()
+config.add(args.q, '_CELERY_QUEUE', True)
 config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)

apps/cic-notify/cic_notify/version.py

@@ -6,11 +6,10 @@ import time
 import semver
 
 # local imports
-from cic_notify.error import PleaseCommitFirstError
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.3')
+version = (0, 4, 0, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],
@@ -18,27 +17,4 @@ version_object = semver.VersionInfo(
         patch=version[2],
         prerelease=version[3],
         )
-
 version_string = str(version_object)
-
-
-def git_hash():
-    import subprocess
-
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-        time_string_pair[0],
-        int(time_string_pair[1]),
-    )
-logg.info(f'Final version string will be {version_string}')
-
-__version_string__ = version_string

apps/cic-notify/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.6
+FROM python:3.8.6-slim-buster
 
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
@@ -6,7 +6,7 @@ RUN apt-get update && \
 WORKDIR /usr/src/cic-notify
 
 ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
 
 COPY cic-notify/setup.cfg \
 	 cic-notify/setup.py \

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]~=0.1.2a46
+cic_base[full_graph]~=0.1.2a61

apps/cic-notify/setup.cfg

@@ -1,6 +1,5 @@
 [metadata]
 name = cic-notify
-version= 0.4.0a2
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no
@@ -45,3 +44,4 @@ testing =
 [options.entry_points]
 console_scripts =
 	cic-notify-tasker = cic_notify.runnable.tasker:main
+	cic-notify-send = cic_notify.runnable.send:main

apps/cic-notify/setup.py

@@ -1,9 +1,31 @@
 # standard imports
+import logging
+import subprocess
+import time
 from setuptools import setup
 
-# third-party imports
-
 # local imports
+from cic_notify.version import version_string
+
+logg = logging.getLogger()
+
+
+def git_hash():
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+            time_string_pair[0],
+            int(time_string_pair[1]),
+            )
+logg.info(f'Final version string will be {version_string}')
 
 
 requirements = []
@@ -25,6 +47,6 @@ while True:
 test_requirements_file.close()
 
 setup(
+    version=version_string,
     install_requires=requirements,
-    tests_require=test_requirements,
-)
+    tests_require=test_requirements)

apps/cic-ussd/.config/app.ini

@@ -1,14 +1,24 @@
 [app]
-ALLOWED_IP=127.0.0.1
+ALLOWED_IP=0.0.0.0/0
 LOCALE_FALLBACK=en
-LOCALE_PATH=var/lib/locale/
+LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#
 
+[phone_number]
+REGION=KE
+
 [ussd]
 MENU_FILE=/usr/src/data/ussd_menu.json
+user = 
+pass = 
 
 [statemachine]
 STATES=/usr/src/cic-ussd/states/
 TRANSITIONS=/usr/src/cic-ussd/transitions/
+
+[client]
+host =
+port = 
+ssl =

apps/cic-ussd/.config/database.ini

@@ -6,3 +6,5 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
+DEBUG=0
+POOL_SIZE=1

apps/cic-ussd/.config/redis.ini

@@ -3,7 +3,7 @@ BROKER_URL=redis://
 RESULT_URL=redis://
 
 [redis]
-HOSTNAME=localhost
+HOSTNAME=redis
 PASSWORD=
 PORT=6379
 DATABASE=0

apps/cic-ussd/cic_ussd/account.py

@@ -8,12 +8,12 @@ from cic_types.processor import generate_metadata_pointer
 
 # local imports
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.redis import get_cached_data
 
 
-def define_account_tx_metadata(user: User):
+def define_account_tx_metadata(user: Account):
     # get sender metadata
     identifier = blockchain_address_to_metadata_pointer(
         blockchain_address=user.blockchain_address
@@ -27,7 +27,7 @@ def define_account_tx_metadata(user: User):
     if account_metadata:
         account_metadata = json.loads(account_metadata)
         person = Person()
-        deserialized_person = person.deserialize(metadata=account_metadata)
+        deserialized_person = person.deserialize(person_data=account_metadata)
         given_name = deserialized_person.given_name
         family_name = deserialized_person.family_name
         phone_number = deserialized_person.tel
@@ -46,4 +46,4 @@ def retrieve_account_statement(blockchain_address: str):
         callback_task='cic_ussd.tasks.callback_handler.process_statement_callback',
         callback_param=blockchain_address
     )
-    result = cic_eth_api.list(address=blockchain_address, limit=9)
+    cic_eth_api.list(address=blockchain_address, limit=9)

apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py

@@ -1,4 +1,4 @@
-"""Create user table
+"""Create account table
 
 Revision ID: f289e8510444
 Revises: 
@@ -17,7 +17,7 @@ depends_on = None
 
 
 def upgrade():
-    op.create_table('user',
+    op.create_table('account',
                     sa.Column('id', sa.Integer(), nullable=False),
                     sa.Column('blockchain_address', sa.String(), nullable=False),
                     sa.Column('phone_number', sa.String(), nullable=False),
@@ -29,11 +29,11 @@ def upgrade():
                     sa.Column('updated', sa.DateTime(), nullable=False),
                     sa.PrimaryKeyConstraint('id')
                     )
-    op.create_index(op.f('ix_user_phone_number'), 'user', ['phone_number'], unique=True)
-    op.create_index(op.f('ix_user_blockchain_address'), 'user', ['blockchain_address'], unique=True)
+    op.create_index(op.f('ix_account_phone_number'), 'account', ['phone_number'], unique=True)
+    op.create_index(op.f('ix_account_blockchain_address'), 'account', ['blockchain_address'], unique=True)
 
 
 def downgrade():
-    op.drop_index(op.f('ix_user_blockchain_address'), table_name='user')
-    op.drop_index(op.f('ix_user_phone_number'), table_name='user')
-    op.drop_table('user')
+    op.drop_index(op.f('ix_account_blockchain_address'), table_name='account')
+    op.drop_index(op.f('ix_account_phone_number'), table_name='account')
+    op.drop_table('account')

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -16,12 +16,12 @@ class AccountStatus(IntEnum):
     RESET = 4
 
 
-class User(SessionBase):
+class Account(SessionBase):
     """
     This class defines a user record along with functions responsible for hashing the user's corresponding password and
      subsequently verifying a password's validity given an input to compare against the persisted hash.
     """
-    __tablename__ = 'user'
+    __tablename__ = 'account'
 
     blockchain_address = Column(String)
     phone_number = Column(String)
@@ -38,7 +38,7 @@ class User(SessionBase):
         self.account_status = AccountStatus.PENDING.value
 
     def __repr__(self):
-        return f'<User: {self.blockchain_address}>'
+        return f'<Account: {self.blockchain_address}>'
 
     def create_password(self, password):
         """This method takes a password value and hashes the value before assigning it to the corresponding

apps/cic-ussd/cic_ussd/db/models/base.py

@@ -1,47 +1,129 @@
-# standard imports
+# stanard imports
+import logging
 import datetime
 
-# third-party imports
+# external imports
 from sqlalchemy import Column, Integer, DateTime
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import (
+        StaticPool,
+        QueuePool,
+        AssertionPool,
+        NullPool,
+        )
+
+logg = logging.getLogger().getChild(__name__)
 
 Model = declarative_base(name='Model')
 
 
 class SessionBase(Model):
+    """The base object for all SQLAlchemy enabled models. All other models must extend this.
+    """
     __abstract__ = True
 
-    id = Column(Integer, primary_key=True)
     created = Column(DateTime, default=datetime.datetime.utcnow)
     updated = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
 
+    id = Column(Integer, primary_key=True)
+
     engine = None
-    session = None
-    query = None
+    """Database connection engine of the running aplication"""
+    sessionmaker = None
+    """Factory object responsible for creating sessions from the  connection pool"""
+    transactional = True
+    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
+    poolable = True
+    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
+    procedural = True
+    """Whether the database backend supports stored procedures"""
+    localsessions = {}
+    """Contains dictionary of sessions initiated by db model components"""
+
 
     @staticmethod
     def create_session():
-        session = sessionmaker(bind=SessionBase.engine)
-        return session()
+        """Creates a new database session.
+        """
+        return SessionBase.sessionmaker()
+
 
     @staticmethod
     def _set_engine(engine):
+        """Sets the database engine static property
+        """
         SessionBase.engine = engine
+        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
+
 
     @staticmethod
-    def build():
-        Model.metadata.create_all(bind=SessionBase.engine)
+    def connect(dsn, pool_size=16, debug=False):
+        """Create new database connection engine and connect to database backend.
+
+        :param dsn: DSN string defining connection.
+        :type dsn: str
+        """
+        e = None
+        if SessionBase.poolable:
+            poolclass = QueuePool
+            if pool_size > 1:
+                logg.info('db using queue pool')
+                e = create_engine(
+                        dsn,
+                        max_overflow=pool_size*3,
+                        pool_pre_ping=True,
+                        pool_size=pool_size,
+                        pool_recycle=60,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+            else:
+                if pool_size == 0:
+                    poolclass = NullPool
+                elif debug:
+                    poolclass = AssertionPool
+                else:
+                    poolclass = StaticPool
+                e = create_engine(
+                        dsn,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+        else:
+            logg.info('db connection not poolable')
+            e = create_engine(
+                    dsn,
+                    echo=debug,
+                    )
+
+        SessionBase._set_engine(e)
 
-    @staticmethod
-    # https://docs.sqlalchemy.org/en/13/core/pooling.html#pool-disconnects
-    def connect(data_source_name):
-        engine = create_engine(data_source_name, pool_pre_ping=True)
-        SessionBase._set_engine(engine)
 
     @staticmethod
     def disconnect():
+        """Disconnect from database and free resources.
+        """
         SessionBase.engine.dispose()
         SessionBase.engine = None
 
+
+    @staticmethod
+    def bind_session(session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+            localsession_key = str(id(localsession))
+            logg.debug('creating new session {}'.format(localsession_key))
+            SessionBase.localsessions[localsession_key] = localsession
+        return localsession
+
+
+    @staticmethod
+    def release_session(session=None):
+        session_key = str(id(session))
+        if SessionBase.localsessions.get(session_key) != None:
+            logg.debug('commit and destroy session {}'.format(session_key))
+            session.commit()
+            session.close()

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -128,8 +128,8 @@
         },
         "22": {
             "description": "Pin entry menu.",
-            "display_key": "ussd.kenya.standard_pin_authorization",
-            "name": "standard_pin_authorization",
+            "display_key": "ussd.kenya.display_metadata_pin_authorization",
+            "name": "display_metadata_pin_authorization",
             "parent": "start"
         },
         "23": {
@@ -230,9 +230,22 @@
         },
         "39": {
             "description": "Menu to instruct users to call the office.",
-            "display_key": "ussd.key.help",
+            "display_key": "ussd.kenya.help",
             "name": "help",
             "parent": null
+        },
+        "40": {
+            "description": "Menu to display a user's entire profile",
+            "display_key": "ussd.kenya.display_user_metadata",
+            "name": "display_user_metadata",
+            "parent": "account_management"
+        },
+        "41": {
+            "description": "The recipient is not in the system",
+            "display_key": "ussd.kenya.exit_invalid_recipient",
+            "name": "exit_invalid_recipient",
+            "parent": null
         }
+
     }
 }

apps/cic-ussd/cic_ussd/error.py

@@ -18,7 +18,7 @@ class ActionDataNotFoundError(OSError):
     pass
 
 
-class UserMetadataNotFoundError(OSError):
+class MetadataNotFoundError(OSError):
     """Raised when metadata is expected but not available in cache."""
     pass
 
@@ -31,3 +31,10 @@ class UnsupportedMethodError(OSError):
 class CachedDataNotFoundError(OSError):
     """Raised when the method passed to the make request function is unsupported."""
     pass
+
+
+class MetadataStoreError(Exception):
+    """Raised when metadata storage fails"""
+    pass
+
+

apps/cic-ussd/cic_ussd/metadata/__init__.py

@@ -3,7 +3,10 @@
 # third-party imports
 import requests
 from chainlib.eth.address import to_checksum
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from cic_ussd.error import UnsupportedMethodError
@@ -40,4 +43,4 @@ def blockchain_address_to_metadata_pointer(blockchain_address: str):
     :return:
     :rtype:
     """
-    return bytes.fromhex(blockchain_address[2:])
+    return bytes.fromhex(strip_0x(blockchain_address))

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -0,0 +1,126 @@
+# standard imports
+import json
+import logging
+import os
+from typing import Dict, Union
+
+# third-part imports
+import requests
+from cic_types.models.person import generate_metadata_pointer, Person
+
+# local imports
+from cic_ussd.metadata import make_request
+from cic_ussd.metadata.signer import Signer
+from cic_ussd.redis import cache_data
+from cic_ussd.error import MetadataStoreError
+
+
+logg = logging.getLogger().getChild(__name__)
+
+
+class Metadata:
+    """
+    :cvar base_url: The base url or the metadata server.
+    :type base_url: str
+    """
+
+    base_url = None
+
+
+def metadata_http_error_handler(result: requests.Response):
+    """ This function handles and appropriately raises errors from http requests interacting with the metadata server.
+    :param result: The response object from a http request.
+    :type result: requests.Response
+    """
+    status_code = result.status_code
+
+    if 100 <= status_code < 200:
+        raise MetadataStoreError(f'Informational errors: {status_code}, reason: {result.reason}')
+
+    elif 300 <= status_code < 400:
+        raise MetadataStoreError(f'Redirect Issues: {status_code}, reason: {result.reason}')
+
+    elif 400 <= status_code < 500:
+        raise MetadataStoreError(f'Client Error: {status_code}, reason: {result.reason}')
+
+    elif 500 <= status_code < 600:
+        raise MetadataStoreError(f'Server Error: {status_code}, reason: {result.reason}')
+
+
+class MetadataRequestsHandler(Metadata):
+
+    def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
+        """
+        :param cic_type: The salt value with which to hash a specific metadata identifier.
+        :type cic_type: str
+        :param engine: Encryption used for sending data to the metadata server.
+        :type engine: str
+        :param identifier: A unique element of data in bytes necessary for creating a metadata pointer.
+        :type identifier: bytes
+        """
+        self.cic_type = cic_type
+        self.engine = engine
+        self.headers = {
+            'X-CIC-AUTOMERGE': 'server',
+            'Content-Type': 'application/json'
+        }
+        self.identifier = identifier
+        self.metadata_pointer = generate_metadata_pointer(
+            identifier=self.identifier,
+            cic_type=self.cic_type
+        )
+        if self.base_url:
+            self.url = os.path.join(self.base_url, self.metadata_pointer)
+
+    def create(self, data: Union[Dict, str]):
+        """ This function is responsible for posting data to the metadata server with a corresponding metadata pointer
+        for storage.
+        :param data: The data to be stored in the metadata server.
+        :type data: dict|str
+        """
+        data = json.dumps(data).encode('utf-8')
+        result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
+        metadata_http_error_handler(result=result)
+        metadata = result.content
+        self.edit(data=metadata)
+
+    def edit(self, data: bytes):
+        """ This function is responsible for editing data in the metadata server corresponding to a unique pointer.
+        :param data: The data to be edited in the metadata server.
+        :type data: bytes
+        """
+        cic_meta_signer = Signer()
+        signature = cic_meta_signer.sign_digest(data=data)
+        algorithm = cic_meta_signer.get_operational_key().get('algo')
+        decoded_data = data.decode('utf-8')
+        formatted_data = {
+            'm': data.decode('utf-8'),
+            's': {
+                'engine': self.engine,
+                'algo': algorithm,
+                'data': signature,
+                'digest': json.loads(data).get('digest'),
+            }
+        }
+        formatted_data = json.dumps(formatted_data).encode('utf-8')
+        result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
+        logg.info(f'signed metadata submission status: {result.status_code}.')
+        metadata_http_error_handler(result=result)
+        try:
+            decoded_identifier = self.identifier.decode("utf-8")
+        except UnicodeDecodeError:
+            decoded_identifier = self.identifier.hex()
+        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {decoded_data}.')
+
+    def query(self):
+        """This function is responsible for querying the metadata server for data corresponding to a unique pointer."""
+        result = make_request(method='GET', url=self.url)
+        metadata_http_error_handler(result=result)
+        response_data = result.content
+        data = json.loads(response_data.decode('utf-8'))
+        if result.status_code == 200 and self.cic_type == 'cic.person':
+            person = Person()
+            deserialized_person = person.deserialize(person_data=json.loads(data))
+            data = json.dumps(deserialized_person.serialize())
+            cache_data(self.metadata_pointer, data=data)
+            logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# third-party imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PersonMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type='cic.person', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -0,0 +1,13 @@
+# standard imports
+import logging
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PhonePointerMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type='cic.msisdn', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/signer.py

@@ -44,7 +44,7 @@ class Signer:
         gpg_keys = self.gpg.list_keys()
         key_algorithm = gpg_keys[0].get('algo')
         key_id = gpg_keys[0].get("keyid")
-        logg.info(f'using signing key: {key_id}, algorithm: {key_algorithm}')
+        logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
         return gpg_keys[0]
 
     def sign_digest(self, data: bytes):

apps/cic-ussd/cic_ussd/metadata/user.py

@@ -1,102 +0,0 @@
-# standard imports
-import json
-import logging
-import os
-
-# third-party imports
-import requests
-from cic_types.models.person import generate_metadata_pointer, Person
-
-# local imports
-from cic_ussd.chain import Chain
-from cic_ussd.metadata import make_request
-from cic_ussd.metadata.signer import Signer
-from cic_ussd.redis import cache_data
-
-logg = logging.getLogger()
-
-
-class UserMetadata:
-    """
-    :cvar base_url:
-    :type base_url:
-    """
-    base_url = None
-
-    def __init__(self, identifier: bytes):
-        """
-        :param identifier:
-        :type identifier:
-        """
-        self. headers = {
-            'X-CIC-AUTOMERGE': 'server',
-            'Content-Type': 'application/json'
-        }
-        self.identifier = identifier
-        self.metadata_pointer = generate_metadata_pointer(
-                identifier=self.identifier,
-                cic_type='cic.person'
-        )
-        if self.base_url:
-            self.url = os.path.join(self.base_url, self.metadata_pointer)
-
-    def create(self, data: dict):
-        try:
-            data = json.dumps(data).encode('utf-8')
-            result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
-            metadata = result.content
-            self.edit(data=metadata, engine='pgp')
-            logg.info(f'Get sign material response status: {result.status_code}')
-            result.raise_for_status()
-        except requests.exceptions.HTTPError as error:
-            raise RuntimeError(error)
-
-    def edit(self, data: bytes, engine: str):
-        """
-        :param data:
-        :type data:
-        :param engine:
-        :type engine:
-        :return:
-        :rtype:
-        """
-        cic_meta_signer = Signer()
-        signature = cic_meta_signer.sign_digest(data=data)
-        algorithm = cic_meta_signer.get_operational_key().get('algo')
-        formatted_data = {
-            'm': data.decode('utf-8'),
-            's': {
-                'engine': engine,
-                'algo': algorithm,
-                'data': signature,
-                'digest': json.loads(data).get('digest'),
-            }
-        }
-        formatted_data = json.dumps(formatted_data).encode('utf-8')
-
-        try:
-            result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
-            logg.info(f'Signed content submission status: {result.status_code}.')
-            result.raise_for_status()
-        except requests.exceptions.HTTPError as error:
-            raise RuntimeError(error)
-
-    def query(self):
-        result = make_request(method='GET', url=self.url)
-        status = result.status_code
-        logg.info(f'Get latest data status: {status}')
-        try:
-            if status == 200:
-                response_data = result.content
-                data = json.loads(response_data.decode())
-
-                # validate data
-                person = Person()
-                deserialized_person = person.deserialize(metadata=json.loads(data))
-
-                cache_data(key=self.metadata_pointer, data=json.dumps(deserialized_person.serialize()))
-            elif status == 404:
-                logg.info('The data is not available and might need to be added.')
-            result.raise_for_status()
-        except requests.exceptions.HTTPError as error:
-            raise RuntimeError(error)

apps/cic-ussd/cic_ussd/operations.py

@@ -10,7 +10,7 @@ from tinydb.table import Document
 from typing import Optional
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
@@ -143,10 +143,10 @@ def get_account_status(phone_number) -> str:
     :return: The user account status.
     :rtype: str
     """
-    user = User.session.query(User).filter_by(phone_number=phone_number).first()
+    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
     status = user.get_account_status()
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
     return status
 
@@ -269,12 +269,12 @@ def cache_account_creation_task_id(phone_number: str, task_id: str):
     redis_cache.persist(name=task_id)
 
 
-def process_current_menu(ussd_session: Optional[dict], user: User, user_input: str) -> Document:
+def process_current_menu(ussd_session: Optional[dict], user: Account, user_input: str) -> Document:
     """This function checks user input and returns a corresponding ussd menu
     :param ussd_session: An in db ussd session object.
     :type ussd_session: UssdSession
     :param user: A user object.
-    :type user: User
+    :type user: Account
     :param user_input: The user's input.
     :type user_input: str
     :return: An in memory ussd menu object.
@@ -324,7 +324,7 @@ def process_menu_interaction_requests(chain_str: str,
 
     else:
         # get user
-        user = User.session.query(User).filter_by(phone_number=phone_number).first()
+        user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
@@ -390,10 +390,10 @@ def reset_pin(phone_number: str) -> str:
     :return: The status of the pin reset.
     :rtype: str
     """
-    user = User.session.query(User).filter_by(phone_number=phone_number).first()
+    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
     user.reset_account_pin()
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
     response = f'Pin reset for user {phone_number} is successful!'
     return response

apps/cic-ussd/cic_ussd/phone_number.py

@@ -5,7 +5,7 @@ from typing import Optional
 import phonenumbers
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 
 def process_phone_number(phone_number: str, region: str):
@@ -30,14 +30,14 @@ def process_phone_number(phone_number: str, region: str):
     return parsed_phone_number
 
 
-def get_user_by_phone_number(phone_number: str) -> Optional[User]:
+def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
     """This function queries the database for a user based on the provided phone number.
     :param phone_number: A valid phone number.
     :type phone_number: str
     :return: A user object matching a given phone number
-    :rtype: User|None
+    :rtype: Account|None
     """
     # consider adding region to user's metadata
     phone_number = process_phone_number(phone_number=phone_number, region='KE')
-    user = User.session.query(User).filter_by(phone_number=phone_number).first()
+    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
     return user

apps/cic-ussd/cic_ussd/processor.py

@@ -13,8 +13,9 @@ from tinydb.table import Document
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
 from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.user import AccountStatus, User
+from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.db.models.ussd_session import UssdSession
+from cic_ussd.error import MetadataNotFoundError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.phone_number import get_user_by_phone_number
@@ -22,17 +23,18 @@ from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.conversions import to_wei, from_wei
 from cic_ussd.translation import translation_for
+from cic_types.models.person import generate_metadata_pointer, get_contact_data_from_vcard
 
 logg = logging.getLogger(__name__)
 
 
-def process_pin_authorization(display_key: str, user: User, **kwargs) -> str:
+def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
     """
     This method provides translation for all ussd menu entries that follow the pin authorization pattern.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user in a running USSD session.
-    :type user: User
+    :type user: Account
     :param kwargs: Any additional information required by the text values in the internationalization files.
     :type kwargs
     :return: A string value corresponding the ussd menu's text value.
@@ -53,13 +55,13 @@ def process_pin_authorization(display_key: str, user: User, **kwargs) -> str:
         )
 
 
-def process_exit_insufficient_balance(display_key: str, user: User, ussd_session: dict):
+def process_exit_insufficient_balance(display_key: str, user: Account, ussd_session: dict):
     """This function processes the exit menu letting users their account balance is insufficient to perform a specific
     transaction.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -88,12 +90,12 @@ def process_exit_insufficient_balance(display_key: str, user: User, ussd_session
     )
 
 
-def process_exit_successful_transaction(display_key: str, user: User, ussd_session: dict):
+def process_exit_successful_transaction(display_key: str, user: Account, ussd_session: dict):
     """This function processes the exit menu after a successful initiation for a transfer of tokens.
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -116,11 +118,11 @@ def process_exit_successful_transaction(display_key: str, user: User, ussd_sessi
     )
 
 
-def process_transaction_pin_authorization(user: User, display_key: str, ussd_session: dict):
+def process_transaction_pin_authorization(user: Account, display_key: str, ussd_session: dict):
     """This function processes pin authorization where making a transaction is concerned. It constructs a
     pre-transaction response menu that shows the details of the transaction.
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's
@@ -136,7 +138,7 @@ def process_transaction_pin_authorization(user: User, display_key: str, ussd_ses
     tx_sender_information = define_account_tx_metadata(user=user)
 
     token_symbol = 'SRF'
-    user_input = ussd_session.get('user_input').split('*')[-1]
+    user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
     return process_pin_authorization(
@@ -149,7 +151,7 @@ def process_transaction_pin_authorization(user: User, display_key: str, ussd_ses
     )
 
 
-def process_account_balances(user: User, display_key: str, ussd_session: dict):
+def process_account_balances(user: Account, display_key: str, ussd_session: dict):
     """
     :param user:
     :type user:
@@ -187,22 +189,56 @@ def format_transactions(transactions: list, preferred_language: str):
             value = transaction.get('to_value')
             timestamp = transaction.get('timestamp')
             action_tag = transaction.get('action_tag')
+            direction = transaction.get('direction')
             token_symbol = 'SRF'
 
             if action_tag == 'SENT' or action_tag == 'ULITUMA':
-                formatted_transactions += f'{action_tag} {value} {token_symbol} {recipient_phone_number} {timestamp}.\n'
+                formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {recipient_phone_number} {timestamp}.\n'
             else:
-                formatted_transactions += f'{action_tag} {value} {token_symbol} {sender_phone_number} {timestamp}. \n'
+                formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {sender_phone_number} {timestamp}. \n'
         return formatted_transactions
     else:
         if preferred_language == 'en':
-            formatted_transactions = 'Empty'
+            formatted_transactions = 'NO TRANSACTION HISTORY'
         else:
-            formatted_transactions = 'Hamna historia'
+            formatted_transactions = 'HAMNA RIPOTI YA MATUMIZI'
         return formatted_transactions
 
 
-def process_account_statement(user: User, display_key: str, ussd_session: dict):
+def process_display_user_metadata(user: Account, display_key: str):
+    """
+    :param user:
+    :type user:
+    :param display_key:
+    :type display_key:
+    """
+    key = generate_metadata_pointer(
+        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+        cic_type='cic.person'
+    )
+    user_metadata = get_cached_data(key)
+    if user_metadata:
+        user_metadata = json.loads(user_metadata)
+        contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard'))
+        logg.debug(f'{contact_data}')
+        full_name = f'{contact_data.get("given")} {contact_data.get("family")}'
+        gender = user_metadata.get('gender')
+        products = ', '.join(user_metadata.get('products'))
+        location = user_metadata.get('location').get('area_name')
+
+        return translation_for(
+            key=display_key,
+            preferred_language=user.preferred_language,
+            full_name=full_name,
+            gender=gender,
+            location=location,
+            products=products
+        )
+    else:
+        raise MetadataNotFoundError(f'Expected person metadata but found none in cache for key: {key}')
+
+
+def process_account_statement(user: Account, display_key: str, ussd_session: dict):
     """
     :param user:
     :type user:
@@ -229,7 +265,7 @@ def process_account_statement(user: User, display_key: str, ussd_session: dict):
         middle_transaction_set += transactions[3:][:3]
         first_transaction_set += transactions[:3]
     # there are probably much cleaner and operational inexpensive ways to do this so find them
-    elif 4 < len(transactions) < 7:
+    elif 3 < len(transactions) < 7:
         middle_transaction_set += transactions[3:]
         first_transaction_set += transactions[:3]
     else:
@@ -265,12 +301,12 @@ def process_account_statement(user: User, display_key: str, ussd_session: dict):
         )
 
 
-def process_start_menu(display_key: str, user: User):
+def process_start_menu(display_key: str, user: Account):
     """This function gets data on an account's balance and token in order to append it to the start of the start menu's
     title. It passes said arguments to the translation function and returns the appropriate corresponding text from the
     translation files.
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :return: Corresponding translation text response
@@ -295,11 +331,11 @@ def process_start_menu(display_key: str, user: User):
     operational_balance = compute_operational_balance(balances=balances_data)
 
     # retrieve and cache account's metadata
-    s_query_user_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_user_metadata',
+    s_query_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_person_metadata',
         [blockchain_address]
     )
-    s_query_user_metadata.apply_async(queue='cic-ussd')
+    s_query_person_metadata.apply_async(queue='cic-ussd')
 
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
@@ -325,13 +361,13 @@ def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
     return last_ussd_session
 
 
-def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document:
+def process_request(user_input: str, user: Account, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
     provided  session id exists in the database or whether the creation of a new ussd session object is warranted.
     It then returns the appropriate ussd menu text values.
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :param ussd_session: A JSON serialized in-memory ussd session object
@@ -349,18 +385,24 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] =
         if user.has_valid_pin():
             last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
 
-            key = create_cached_data_key(
+            key = generate_metadata_pointer(
                 identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-                salt='cic.person'
+                cic_type='cic.person'
             )
-            user_metadata = get_cached_data(key=key)
+            person_metadata = get_cached_data(key=key)
 
             if last_ussd_session:
                 # get last state
                 last_state = last_ussd_session.state
-                logg.debug(f'LAST USSD SESSION STATE: {last_state}')
                 # if last state is account_creation_prompt and metadata exists, show start menu
-                if last_state == 'account_creation_prompt' and user_metadata is not None:
+                if last_state in [
+                    'account_creation_prompt',
+                    'exit',
+                    'exit_invalid_pin',
+                    'exit_invalid_new_pin',
+                    'exit_pin_mismatch',
+                    'exit_invalid_request'
+                ] and person_metadata is not None:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)
@@ -373,14 +415,14 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] =
                 return UssdMenu.find_by_name(name='initial_pin_entry')
 
 
-def next_state(ussd_session: dict, user: User, user_input: str) -> str:
+def next_state(ussd_session: dict, user: Account, user_input: str) -> str:
     """This function navigates the state machine based on the ussd session object and user inputs it receives.
     It checks the user input and provides the successive state in the state machine. It then updates the session's
     state attribute with the new state.
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :param user: The user requesting access to the ussd menu.
-    :type user: User
+    :type user: Account
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :return: A string value corresponding the successive give a specific state in the state machine.
@@ -396,7 +438,7 @@ def custom_display_text(
         display_key: str,
         menu_name: str,
         ussd_session: dict,
-        user: User) -> str:
+        user: Account) -> str:
     """This function extracts the appropriate session data based on the current menu name. It then inserts them as
     keywords in the i18n function.
     :param display_key: The path in the translation files defining an appropriate ussd response
@@ -404,7 +446,7 @@ def custom_display_text(
     :param menu_name: The name by which a specific menu can be identified.
     :type menu_name: str
     :param user: The user in a running USSD session.
-    :type user: User
+    :type user: Account
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: A string value corresponding the ussd menu's text value.
@@ -420,9 +462,13 @@ def custom_display_text(
         return process_start_menu(display_key=display_key, user=user)
     elif 'pin_authorization' in menu_name:
         return process_pin_authorization(display_key=display_key, user=user)
+    elif 'enter_current_pin' in menu_name:
+        return process_pin_authorization(display_key=display_key, user=user)
     elif menu_name == 'account_balances':
         return process_account_balances(display_key=display_key, user=user, ussd_session=ussd_session)
     elif 'transaction_set' in menu_name:
         return process_account_statement(display_key=display_key, user=user, ussd_session=ussd_session)
+    elif menu_name == 'display_user_metadata':
+        return process_display_user_metadata(display_key=display_key, user=user)
     else:
         return translation_for(key=display_key, preferred_language=user.preferred_language)

apps/cic-ussd/cic_ussd/requests.py

@@ -10,7 +10,7 @@ from urllib.parse import urlparse, parse_qs
 from sqlalchemy import desc
 
 # local imports
-from cic_ussd.db.models.user import AccountStatus, User
+from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import get_account_status, reset_pin
 from cic_ussd.validator import check_known_user
 
@@ -123,9 +123,9 @@ def process_locked_accounts_requests(env: dict) -> tuple:
             else:
                 limit = r[1]
 
-        locked_accounts = User.session.query(User.blockchain_address).filter(
-            User.account_status == AccountStatus.LOCKED.value,
-            User.failed_pin_attempts >= 3).order_by(desc(User.updated)).offset(offset).limit(limit).all()
+        locked_accounts = Account.session.query(Account.blockchain_address).filter(
+            Account.account_status == AccountStatus.LOCKED.value,
+            Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all()
 
         # convert lists to scalar blockchain addresses
         locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts]

apps/cic-ussd/cic_ussd/runnable/server.py

@@ -23,10 +23,11 @@ from cic_ussd.encoder import PasswordEncoder
 from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata.signer import Signer
-from cic_ussd.metadata.user import UserMetadata
+from cic_ussd.metadata.base import Metadata
 from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
+from cic_ussd.phone_number import process_phone_number
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
                                get_request_method,
@@ -35,7 +36,8 @@ from cic_ussd.requests import (get_request_endpoint,
                                process_pin_reset_requests)
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
-from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number
+from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
+    validate_presence
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -55,20 +57,17 @@ arg_parser.add_argument('--env-prefix',
                         help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
+# define log levels
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
 # parse config
 config = Config(config_dir=args.c, env_prefix=args.env_prefix)
 config.process()
 config.censor('PASSWORD', 'DATABASE')
-
-# define log levels
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-    logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-# log config vars
-logg.debug(config)
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # initialize elements
 # set up translations
@@ -85,7 +84,7 @@ UssdMenu.ussd_menu_db = ussd_menu_db
 
 # set up db
 data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name=data_source_name)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 # create session for the life time of http request
 SessionBase.session = SessionBase.create_session()
 
@@ -98,12 +97,18 @@ InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'),
 InMemoryUssdSession.redis_cache = InMemoryStore.cache
 
 # define metadata URL
-UserMetadata.base_url = config.get('CIC_META_URL')
+Metadata.base_url = config.get('CIC_META_URL')
 
 # define signer values
-Signer.gpg_path = config.get('PGP_EXPORT_DIR')
+export_dir = config.get('PGP_EXPORT_DIR')
+if export_dir:
+    validate_presence(path=export_dir)
+Signer.gpg_path = export_dir
 Signer.gpg_passphrase = config.get('PGP_PASSPHRASE')
-Signer.key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
+key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
+if key_file_path:
+    validate_presence(path=key_file_path)
+Signer.key_file_path = key_file_path
 
 # initialize celery app
 celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@@ -144,6 +149,10 @@ def application(env, start_response):
         external_session_id = post_data.get('sessionId')
         user_input = post_data.get('text')
 
+        # add validation for phone number
+        if phone_number:
+            phone_number = process_phone_number(phone_number=phone_number, region=config.get('PHONE_NUMBER_REGION'))
+
         # validate ip address
         if not check_ip(config=config, env=env):
             start_response('403 Sneaky, sneaky', errors_headers)
@@ -167,8 +176,10 @@ def application(env, start_response):
 
         # validate phone number
         if not validate_phone_number(phone_number):
+            logg.error('invalid phone number {}'.format(phone_number))
             start_response('400 Invalid phone number format', errors_headers)
             return []
+        logg.debug('session {} started for {}'.format(external_session_id, phone_number))
 
         # handle menu interaction requests
         chain_str = chain_spec.__str__()

apps/cic-ussd/cic_ussd/runnable/tasker.py

@@ -6,6 +6,7 @@ import tempfile
 
 # third party imports
 import celery
+import i18n
 import redis
 from confini import Config
 
@@ -13,12 +14,14 @@ from confini import Config
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.metadata.signer import Signer
-from cic_ussd.metadata.user import UserMetadata
+from cic_ussd.metadata.base import Metadata
 from cic_ussd.redis import InMemoryStore
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
+from cic_ussd.validator import validate_presence
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
+logging.getLogger('gnupg').setLevel(logging.WARNING)
 
 config_directory = '/usr/local/etc/cic-ussd/'
 
@@ -31,22 +34,22 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 arg_parser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
-# parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-
 # define log levels
 if args.vv:
     logging.getLogger().setLevel(logging.DEBUG)
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-logg.debug(config)
+# parse config
+config = Config(args.c, args.env_prefix)
+config.process()
+config.add(args.q, '_CELERY_QUEUE', True)
+config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name=data_source_name)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 
 # verify database connection with minimal sanity query
 session = SessionBase.create_session()
@@ -62,12 +65,22 @@ InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'),
 InMemoryUssdSession.redis_cache = InMemoryStore.cache
 
 # define metadata URL
-UserMetadata.base_url = config.get('CIC_META_URL')
+Metadata.base_url = config.get('CIC_META_URL')
 
 # define signer values
-Signer.gpg_path = config.get('PGP_EXPORT_DIR')
+export_dir = config.get('PGP_EXPORT_DIR')
+if export_dir:
+    validate_presence(path=export_dir)
+Signer.gpg_path = export_dir
 Signer.gpg_passphrase = config.get('PGP_PASSPHRASE')
-Signer.key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
+key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
+if key_file_path:
+    validate_presence(path=key_file_path)
+Signer.key_file_path = key_file_path
+
+# set up translations
+i18n.load_path.append(config.get('APP_LOCALE_PATH'))
+i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
 
 # set up celery
 current_app = celery.Celery(__name__)

apps/cic-ussd/cic_ussd/state_machine/logic/balance.py

@@ -5,12 +5,12 @@ from typing import Tuple
 # third-party imports
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 logg = logging.getLogger(__file__)
 
 
-def process_mini_statement_request(state_machine_data: Tuple[str, dict, User]):
+def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]):
     """This function compiles a brief statement of a user's last three inbound and outbound transactions and send the
     same as a message on their selected avenue for notification.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -6,10 +6,10 @@ ussd menu facilitating the return of appropriate menu responses based on said us
 from typing import Tuple
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 
-def menu_one_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_one_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that user input matches a string with value '1'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
@@ -20,7 +20,7 @@ def menu_one_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '1'
 
 
-def menu_two_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_two_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that user input matches a string with value '2'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -31,7 +31,7 @@ def menu_two_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '2'
 
 
-def menu_three_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_three_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that user input matches a string with value '3'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -42,7 +42,7 @@ def menu_three_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '3'
 
 
-def menu_four_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_four_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '4'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -54,7 +54,7 @@ def menu_four_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '4'
 
 
-def menu_five_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '5'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -66,7 +66,7 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '5'
 
 
-def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '00'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -78,7 +78,7 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user_input == '00'
 
 
-def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, User]) -> bool:
+def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '99'
     :param state_machine_data: A tuple containing user input, a ussd session and user object.

apps/cic-ussd/cic_ussd/state_machine/logic/pin.py

@@ -12,7 +12,7 @@ from typing import Tuple
 import bcrypt
 
 # local imports
-from cic_ussd.db.models.user import AccountStatus, User
+from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.encoder import PasswordEncoder, create_password_hash
 from cic_ussd.operations import persist_session_to_db_task, create_or_update_session
 from cic_ussd.redis import InMemoryStore
@@ -21,7 +21,7 @@ from cic_ussd.redis import InMemoryStore
 logg = logging.getLogger(__file__)
 
 
-def is_valid_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks a pin's validity by ensuring it has a length of for characters and the characters are
     numeric.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -37,7 +37,7 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
     return pin_is_valid
 
 
-def is_authorized_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_authorized_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -48,7 +48,7 @@ def is_authorized_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user.verify_password(password=user_input)
 
 
-def is_locked_account(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_locked_account(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks whether a user's account is locked due to too many failed attempts.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -59,7 +59,7 @@ def is_locked_account(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user.get_account_status() == AccountStatus.LOCKED.name
 
 
-def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, User]):
+def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account]):
     """This function hashes a pin and stores it in session data.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -94,7 +94,7 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, User])
     persist_session_to_db_task(external_session_id=external_session_id, queue='cic-ussd')
 
 
-def pins_match(state_machine_data: Tuple[str, dict, User]) -> bool:
+def pins_match(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -108,7 +108,7 @@ def pins_match(state_machine_data: Tuple[str, dict, User]) -> bool:
     return bcrypt.checkpw(user_input.encode(), initial_pin)
 
 
-def complete_pin_change(state_machine_data: Tuple[str, dict, User]):
+def complete_pin_change(state_machine_data: Tuple[str, dict, Account]):
     """This function persists the user's pin to the database
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -116,11 +116,11 @@ def complete_pin_change(state_machine_data: Tuple[str, dict, User]):
     user_input, ussd_session, user = state_machine_data
     password_hash = ussd_session.get('session_data').get('initial_pin')
     user.password_hash = password_hash
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
 
-def is_blocked_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_blocked_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -131,7 +131,7 @@ def is_blocked_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
     return user.get_account_status() == AccountStatus.LOCKED.name
 
 
-def is_valid_new_pin(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks whether the user's new pin is a valid pin and that it isn't the same as the old one.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple

apps/cic-ussd/cic_ussd/state_machine/logic/sms.py

@@ -3,21 +3,21 @@ import logging
 from typing import Tuple
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 logg = logging.getLogger()
 
 
-def send_terms_to_user_if_required(state_machine_data: Tuple[str, dict, User]):
+def send_terms_to_user_if_required(state_machine_data: Tuple[str, dict, Account]):
     user_input, ussd_session, user = state_machine_data
     logg.debug('Requires integration to cic-notify.')
 
 
-def process_mini_statement_request(state_machine_data: Tuple[str, dict, User]):
+def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]):
     user_input, ussd_session, user = state_machine_data
     logg.debug('Requires integration to cic-notify.')
 
 
-def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, User]):
+def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account]):
     user_input, ussd_session, user = state_machine_data
     logg.debug('Requires integration to cic-notify.')

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -9,7 +9,7 @@ import celery
 # local imports
 from cic_ussd.balance import BalanceManager, compute_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.user import AccountStatus, User
+from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
 from cic_ussd.phone_number import get_user_by_phone_number
 from cic_ussd.redis import create_cached_data_key, get_cached_data
@@ -19,7 +19,7 @@ from cic_ussd.transactions import OutgoingTransactionProcessor
 logg = logging.getLogger(__file__)
 
 
-def is_valid_recipient(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that a user exists, is not the initiator of the transaction, has an active account status
     and is authorized to perform  standard transactions.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -34,7 +34,7 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, User]) -> bool:
     return is_not_initiator and has_active_account_status and recipient is not None
 
 
-def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, User]) -> bool:
+def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that the transaction amount provided is valid as per the criteria for the transaction
     being attempted.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -49,7 +49,7 @@ def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, User]) -> b
         return False
 
 
-def has_sufficient_balance(state_machine_data: Tuple[str, dict, User]) -> bool:
+def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """This function checks that the transaction amount provided is valid as per the criteria for the transaction
     being attempted.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -72,19 +72,20 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, User]) -> bool:
     return int(user_input) <= operational_balance
 
 
-def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, User]):
+def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
     user_input, ussd_session, user = state_machine_data
-    session_data = {
-        'recipient_phone_number': user_input
-    }
+
+    session_data = ussd_session.get('session_data') or {}
+    session_data['recipient_phone_number'] = user_input
+
     save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
 
 
-def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, User]):
+def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account]):
     """
     :param state_machine_data:
     :type state_machine_data:
@@ -96,26 +97,27 @@ def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, User]):
     recipient = get_user_by_phone_number(phone_number=user_input)
     blockchain_address = recipient.blockchain_address
     # retrieve and cache account's metadata
-    s_query_user_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_user_metadata',
+    s_query_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_person_metadata',
         [blockchain_address]
     )
-    s_query_user_metadata.apply_async(queue='cic-ussd')
+    s_query_person_metadata.apply_async(queue='cic-ussd')
 
 
-def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, User]):
+def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
     user_input, ussd_session, user = state_machine_data
-    session_data = {
-        'transaction_amount': user_input
-    }
+
+    session_data = ussd_session.get('session_data') or {}
+    session_data['transaction_amount'] = user_input
+
     save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
 
 
-def process_transaction_request(state_machine_data: Tuple[str, dict, User]):
+def process_transaction_request(state_machine_data: Tuple[str, dict, Account]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -10,8 +10,8 @@ from cic_types.models.person import generate_vcard_from_contact_data, manage_ide
 
 # local imports
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.user import User
-from cic_ussd.error import UserMetadataNotFoundError
+from cic_ussd.db.models.account import Account
+from cic_ussd.error import MetadataNotFoundError
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
 from cic_ussd.redis import get_cached_data
@@ -19,40 +19,40 @@ from cic_ussd.redis import get_cached_data
 logg = logging.getLogger(__file__)
 
 
-def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, User]):
+def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account]):
     """This function changes the user's preferred language to english.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
     user_input, ussd_session, user = state_machine_data
     user.preferred_language = 'en'
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
 
-def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, User]):
+def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account]):
     """This function changes the user's preferred language to swahili.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
     user_input, ussd_session, user = state_machine_data
     user.preferred_language = 'sw'
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
 
-def update_account_status_to_active(state_machine_data: Tuple[str, dict, User]):
+def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account]):
     """This function sets user's account to active.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
     user_input, ussd_session, user = state_machine_data
     user.activate_account()
-    User.session.add(user)
-    User.session.commit()
+    Account.session.add(user)
+    Account.session.commit()
 
 
-def process_gender_user_input(user: User, user_input: str):
+def process_gender_user_input(user: Account, user_input: str):
     """
     :param user:
     :type user:
@@ -74,7 +74,7 @@ def process_gender_user_input(user: User, user_input: str):
     return gender
 
 
-def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, User]):
+def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account]):
     """This function saves first name data to the ussd session in the redis cache.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -109,7 +109,7 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict,
     save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
 
 
-def format_user_metadata(metadata: dict, user: User):
+def format_user_metadata(metadata: dict, user: Account):
     """
     :param metadata:
     :type metadata:
@@ -150,7 +150,7 @@ def format_user_metadata(metadata: dict, user: User):
     }
 
 
-def save_complete_user_metadata(state_machine_data: Tuple[str, dict, User]):
+def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     """This function persists elements of the user metadata stored in session data
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
@@ -164,14 +164,14 @@ def save_complete_user_metadata(state_machine_data: Tuple[str, dict, User]):
     user_metadata = format_user_metadata(metadata=metadata, user=user)
 
     blockchain_address = user.blockchain_address
-    s_create_user_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.create_user_metadata',
+    s_create_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.create_person_metadata',
         [blockchain_address, user_metadata]
     )
-    s_create_user_metadata.apply_async(queue='cic-ussd')
+    s_create_person_metadata.apply_async(queue='cic-ussd')
 
 
-def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]):
+def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
     user_input, ussd_session, user = state_machine_data
     blockchain_address = user.blockchain_address
     key = generate_metadata_pointer(
@@ -181,7 +181,7 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]):
     user_metadata = get_cached_data(key=key)
 
     if not user_metadata:
-        raise UserMetadataNotFoundError(f'Expected user metadata but found none in cache for key: {blockchain_address}')
+        raise MetadataNotFoundError(f'Expected user metadata but found none in cache for key: {blockchain_address}')
 
     given_name = ussd_session.get('session_data').get('given_name')
     family_name = ussd_session.get('session_data').get('family_name')
@@ -192,7 +192,7 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]):
     # validate user metadata
     person = Person()
     user_metadata = json.loads(user_metadata)
-    deserialized_person = person.deserialize(metadata=user_metadata)
+    deserialized_person = person.deserialize(person_data=user_metadata)
 
     # edit specific metadata attribute
     if given_name:
@@ -211,18 +211,18 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, User]):
 
     edited_metadata = deserialized_person.serialize()
 
-    s_edit_user_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.edit_user_metadata',
-        [blockchain_address, edited_metadata, 'pgp']
+    s_edit_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.edit_person_metadata',
+        [blockchain_address, edited_metadata]
     )
-    s_edit_user_metadata.apply_async(queue='cic-ussd')
+    s_edit_person_metadata.apply_async(queue='cic-ussd')
 
 
-def get_user_metadata(state_machine_data: Tuple[str, dict, User]):
+def get_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     user_input, ussd_session, user = state_machine_data
     blockchain_address = user.blockchain_address
     s_get_user_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_user_metadata',
+        'cic_ussd.tasks.metadata.query_person_metadata',
         [blockchain_address]
     )
     s_get_user_metadata.apply_async(queue='cic-ussd')

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -7,14 +7,14 @@ from typing import Tuple
 from cic_types.models.person import generate_metadata_pointer
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.redis import get_cached_data
 
 logg = logging.getLogger()
 
 
-def has_cached_user_metadata(state_machine_data: Tuple[str, dict, User]):
+def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     """This function checks whether the attributes of the user's metadata constituting a profile are filled out.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
@@ -29,7 +29,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, User]):
     return user_metadata is not None
 
 
-def is_valid_name(state_machine_data: Tuple[str, dict, User]):
+def is_valid_name(state_machine_data: Tuple[str, dict, Account]):
     """This function checks that a user provided name is valid
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
@@ -43,7 +43,7 @@ def is_valid_name(state_machine_data: Tuple[str, dict, User]):
         return False
 
 
-def is_valid_gender_selection(state_machine_data: Tuple[str, dict, User]):
+def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]):
     """
     :param state_machine_data:
     :type state_machine_data:

apps/cic-ussd/cic_ussd/state_machine/state_machine.py

@@ -13,7 +13,7 @@ class UssdStateMachine(Machine):
     """This class describes a finite state machine responsible for maintaining all the states that describe the ussd
     menu  as well as providing a means for navigating through these states based on different user inputs.
     It defines different helper functions that co-ordinate with the stakeholder components of the ussd menu: i.e  the
-    User, UssdSession, UssdMenu to facilitate user interaction with ussd menu.
+    Account, UssdSession, UssdMenu to facilitate user interaction with ussd menu.
     :cvar states: A list of pre-defined states.
     :type states: list
     :cvar transitions: A list of pre-defined transitions.

apps/cic-ussd/cic_ussd/tasks/base.py

@@ -5,9 +5,24 @@ import celery
 import sqlalchemy
 
 # local imports
+from cic_ussd.error import MetadataStoreError
+from cic_ussd.db.models.base import SessionBase
 
 
-class CriticalTask(celery.Task):
+class BaseTask(celery.Task):
+
+    session_func = SessionBase.create_session
+
+    def create_session(self):
+        return BaseTask.session_func()
+
+
+    def log_banner(self):
+        logg.debug('task {} root uuid {}'.format(self.__class__.__name__, self.request.root_id))
+        return
+
+    
+class CriticalTask(BaseTask):
     retry_jitter = True
     retry_backoff = True
     retry_backoff_max = 8
@@ -17,4 +32,11 @@ class CriticalSQLAlchemyTask(CriticalTask):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
+        sqlalchemy.exc.ResourceClosedError,
+        ) 
+
+
+class CriticalMetadataTask(CriticalTask):
+    autoretry_for = (
+        MetadataStoreError,
     )

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -9,7 +9,7 @@ import celery
 # local imports
 from cic_ussd.conversions import from_wei
 from cic_ussd.db.models.base import SessionBase
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.account import define_account_tx_metadata
 from cic_ussd.error import ActionDataNotFoundError
 from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key
@@ -49,10 +49,17 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
             phone_number = account_creation_data.get('phone_number')
 
             # create user
-            user = User(blockchain_address=result, phone_number=phone_number)
+            user = Account(blockchain_address=result, phone_number=phone_number)
             session.add(user)
             session.commit()
             session.close()
+    
+            queue = self.request.delivery_info.get('routing_key')
+            s = celery.signature(
+                    'cic_ussd.tasks.metadata.add_phone_pointer',
+                    [result, phone_number]
+                    )
+            s.apply_async(queue=queue)
 
             # expire cache
             cache.expire(task_id, timedelta(seconds=180))
@@ -65,6 +72,8 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
         session.close()
         raise ActionDataNotFoundError(f'Account creation task: {task_id}, returned unexpected response: {status_code}')
 
+    session.close()
+
 
 @celery_app.task
 def process_incoming_transfer_callback(result: dict, param: str, status_code: int):
@@ -74,12 +83,12 @@ def process_incoming_transfer_callback(result: dict, param: str, status_code: in
         # collect result data
         recipient_blockchain_address = result.get('recipient')
         sender_blockchain_address = result.get('sender')
-        token_symbol = result.get('token_symbol')
-        value = result.get('destination_value')
+        token_symbol = result.get('destination_token_symbol')
+        value = result.get('destination_token_value')
 
         # try to find users in system
-        recipient_user = session.query(User).filter_by(blockchain_address=recipient_blockchain_address).first()
-        sender_user = session.query(User).filter_by(blockchain_address=sender_blockchain_address).first()
+        recipient_user = session.query(Account).filter_by(blockchain_address=recipient_blockchain_address).first()
+        sender_user = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first()
 
         # check whether recipient is in the system
         if not recipient_user:
@@ -118,6 +127,7 @@ def process_incoming_transfer_callback(result: dict, param: str, status_code: in
         session.close()
         raise ValueError(f'Unexpected status code: {status_code}.')
 
+    session.close()
 
 @celery_app.task
 def process_balances_callback(result: list, param: str, status_code: int):
@@ -143,21 +153,24 @@ def define_transaction_action_tag(
         # check preferred language
         if preferred_language == 'en':
             action_tag = 'SENT'
+            direction = 'TO'
         else:
             action_tag = 'ULITUMA'
+            direction = 'KWA'
     else:
         if preferred_language == 'en':
             action_tag = 'RECEIVED'
+            direction = 'FROM'
         else:
             action_tag = 'ULIPOKEA'
-    return action_tag
+            direction = 'KUTOKA'
+    return action_tag, direction
 
 
 @celery_app.task
 def process_statement_callback(result, param: str, status_code: int):
     if status_code == 0:
         # create session
-        session = SessionBase.create_session()
         processed_transactions = []
 
         # process transaction data to cache
@@ -170,35 +183,40 @@ def process_statement_callback(result, param: str, status_code: int):
             if '0x0000000000000000000000000000000000000000' in source_token:
                 pass
             else:
+                session = SessionBase.create_session()
                 # describe a processed transaction
                 processed_transaction = {}
 
                 # check if sender is in the system
-                sender: User = session.query(User).filter_by(blockchain_address=sender_blockchain_address).first()
+                sender: Account = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first()
+                owner: Account = session.query(Account).filter_by(blockchain_address=param).first()
                 if sender:
                     processed_transaction['sender_phone_number'] = sender.phone_number
 
-                    action_tag = define_transaction_action_tag(
-                        preferred_language=sender.preferred_language,
+                    action_tag, direction = define_transaction_action_tag(
+                        preferred_language=owner.preferred_language,
                         sender_blockchain_address=sender_blockchain_address,
                         param=param
                     )
                     processed_transaction['action_tag'] = action_tag
+                    processed_transaction['direction'] = direction
 
                 else:
                     processed_transaction['sender_phone_number'] = 'GRASSROOTS ECONOMICS'
 
                 # check if recipient is in the system
-                recipient: User = session.query(User).filter_by(blockchain_address=recipient_address).first()
+                recipient: Account = session.query(Account).filter_by(blockchain_address=recipient_address).first()
                 if recipient:
                     processed_transaction['recipient_phone_number'] = recipient.phone_number
 
                 else:
                     logg.warning(f'Tx with recipient not found in cic-ussd')
 
+                session.close()
+
                 # add transaction values
-                processed_transaction['to_value'] = from_wei(value=transaction.get('to_value'))
-                processed_transaction['from_value'] = from_wei(value=transaction.get('from_value'))
+                processed_transaction['to_value'] = from_wei(value=transaction.get('to_value')).__str__()
+                processed_transaction['from_value'] = from_wei(value=transaction.get('from_value')).__str__()
 
                 raw_timestamp = transaction.get('timestamp')
                 timestamp = datetime.utcfromtimestamp(raw_timestamp).strftime('%d/%m/%y, %H:%M')

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -1,20 +1,22 @@
 # standard imports
-import json
 import logging
 
 # third-party imports
 import celery
+from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
-from cic_ussd.metadata.user import UserMetadata
+from cic_ussd.metadata.person import PersonMetadata
+from cic_ussd.metadata.phone import PhonePointerMetadata
+from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 
 @celery_app.task
-def query_user_metadata(blockchain_address: str):
+def query_person_metadata(blockchain_address: str):
     """
     :param blockchain_address:
     :type blockchain_address:
@@ -22,12 +24,12 @@ def query_user_metadata(blockchain_address: str):
     :rtype:
     """
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
-    user_metadata_client.query()
+    person_metadata_client = PersonMetadata(identifier=identifier)
+    person_metadata_client.query()
 
 
 @celery_app.task
-def create_user_metadata(blockchain_address: str, data: dict):
+def create_person_metadata(blockchain_address: str, data: dict):
     """
     :param blockchain_address:
     :type blockchain_address:
@@ -37,12 +39,20 @@ def create_user_metadata(blockchain_address: str, data: dict):
     :rtype:
     """
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
-    user_metadata_client.create(data=data)
+    person_metadata_client = PersonMetadata(identifier=identifier)
+    person_metadata_client.create(data=data)
 
 
 @celery_app.task
-def edit_user_metadata(blockchain_address: str, data: bytes, engine: str):
+def edit_person_metadata(blockchain_address: str, data: bytes):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
-    user_metadata_client.edit(data=data, engine=engine)
+    person_metadata_client = PersonMetadata(identifier=identifier)
+    person_metadata_client.edit(data=data)
+
+
+@celery_app.task(bind=True, base=CriticalMetadataTask)
+def add_phone_pointer(self, blockchain_address: str, phone_number: str):
+    identifier = phone_number.encode('utf-8')
+    stripped_address = strip_0x(blockchain_address)
+    phone_metadata_client = PhonePointerMetadata(identifier=identifier)
+    phone_metadata_client.create(data=stripped_address)

apps/cic-ussd/cic_ussd/tasks/ussd_session.py

@@ -70,3 +70,4 @@ def persist_session_to_db(external_session_id: str):
         session.close()
         raise SessionNotFoundError('Session does not exist!')
 
+    session.close()

apps/cic-ussd/cic_ussd/transactions.py

@@ -47,7 +47,7 @@ def to_wei(value: int) -> int:
     :return: Wei equivalent of value in SRF
     :rtype: int
     """
-    return int(value * 1e+18)
+    return int(value * 1e+6)
 
 
 class IncomingTransactionProcessor:

apps/cic-ussd/cic_ussd/translation.py

@@ -8,7 +8,7 @@ from typing import Optional
 def translation_for(key: str, preferred_language: Optional[str] = None, **kwargs) -> str:
     """
     Translates text mapped to a specific YAML key into the user's set preferred language.
-    :param preferred_language: User's preferred language in which to view the ussd menu.
+    :param preferred_language: Account's preferred language in which to view the ussd menu.
     :type preferred_language str
     :param key: Key to a specific YAML test entry
     :type key: str

apps/cic-ussd/cic_ussd/validator.py

@@ -1,12 +1,14 @@
 # standard imports
 import logging
+import os
 import re
+import ipaddress
 
 # third-party imports
 from confini import Config
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 logg = logging.getLogger(__file__)
 
@@ -20,7 +22,14 @@ def check_ip(config: Config, env: dict):
     :return: Request IP validity
     :rtype: boolean
     """
-    return env.get('REMOTE_ADDR') == config.get('APP_ALLOWED_IP')
+    # TODO: do once at boot time
+    actual_ip = ipaddress.ip_network(env.get('REMOTE_ADDR') + '/32')
+    for allowed_net_src in config.get('APP_ALLOWED_IP').split(','):
+        allowed_net = ipaddress.ip_network(allowed_net_src)
+        if actual_ip.subnet_of(allowed_net):
+            return True
+
+    return False
 
 
 def check_request_content_length(config: Config, env: dict):
@@ -59,7 +68,7 @@ def check_known_user(phone: str):
     :return: Is known phone number
     :rtype: boolean
     """
-    user = User.session.query(User).filter_by(phone_number=phone).first()
+    user = Account.session.query(Account).filter_by(phone_number=phone).first()
     return user is not None
 
 
@@ -110,7 +119,7 @@ def validate_phone_number(phone: str):
 
 
 def validate_response_type(processor_response: str) -> bool:
-    """1*3443*3443*Philip*Wanga*1*Juja*Software Developer*2*3
+    """
     This function checks the prefix for a corresponding menu's text from the response offered by the Ussd Processor and
     determines whether the response should prompt the end of a ussd session or the
     :param processor_response: A ussd menu's text value.
@@ -126,3 +135,14 @@ def validate_response_type(processor_response: str) -> bool:
         return True
     return False
 
+
+def validate_presence(path: str):
+    """
+
+    """
+    is_present = os.path.exists(path=path)
+
+    if not is_present:
+        raise ValueError(f'Directory/File in path: {path} not found.')
+    else:
+        logg.debug(f'Loading data from: {path}')

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.8')
+version = (0, 3, 0, 'alpha.9')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile

@@ -51,4 +51,4 @@ RUN cd cic-ussd && \
 COPY cic-ussd/.config/ /usr/local/etc/cic-ussd/
 COPY cic-ussd/cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
 
-WORKDIR /root
+WORKDIR /root

apps/cic-ussd/docker/start_tasker.sh

@@ -2,4 +2,4 @@
 
 . /root/db.sh
 
-/usr/local/bin/cic-ussd-tasker -vv "$@"
+/usr/local/bin/cic-ussd-tasker $@

apps/cic-ussd/docker/start_uwsgi.sh

@@ -2,4 +2,6 @@
 
 . /root/db.sh
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :9000 --pyargv "-vv"
+server_port=${SERVER_PORT:-9000}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :$server_port --pyargv "$@"

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2a58
-cic-eth~=0.11.0a4
-cic-notify~=0.4.0a3
+cic_base[full_graph]~=0.1.2a79
+cic-eth~=0.11.0b7
+cic-notify~=0.4.0a4
 cic-types~=0.1.0a10

apps/cic-ussd/states/user_metadata_states.json

@@ -4,5 +4,6 @@
   "enter_gender",
   "enter_age",
   "enter_location",
-  "enter_products"
+  "enter_products",
+  "display_metadata_pin_authorization"
 ]

apps/cic-ussd/tests/cic_ussd/db/models/test_user.py

@@ -4,19 +4,19 @@
 import pytest
 
 # platform imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 
 
 def test_user(init_database, set_fernet_key):
-    user = User(blockchain_address='0x417f5962fc52dc33ff0689659b25848680dec6dcedc6785b03d1df60fc6d5c51',
-                phone_number='+254700000000')
+    user = Account(blockchain_address='0x417f5962fc52dc33ff0689659b25848680dec6dcedc6785b03d1df60fc6d5c51',
+                   phone_number='+254700000000')
     user.create_password('0000')
 
-    session = User.session
+    session = Account.session
     session.add(user)
     session.commit()
 
-    queried_user = session.query(User).get(1)
+    queried_user = session.query(Account).get(1)
     assert queried_user.blockchain_address == '0x417f5962fc52dc33ff0689659b25848680dec6dcedc6785b03d1df60fc6d5c51'
     assert queried_user.phone_number == '+254700000000'
     assert queried_user.failed_pin_attempts == 0
@@ -25,7 +25,7 @@ def test_user(init_database, set_fernet_key):
 
 def test_user_state_transition(create_pending_user):
     user = create_pending_user
-    session = User.session
+    session = Account.session
 
     assert user.get_account_status() == 'PENDING'
     user.activate_account()

apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py

@@ -9,26 +9,26 @@ from cic_types.models.person import generate_metadata_pointer
 # local imports
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.metadata.signer import Signer
-from cic_ussd.metadata.user import UserMetadata
+from cic_ussd.metadata.person import PersonMetadata
 from cic_ussd.redis import get_cached_data
 
 
 def test_user_metadata(create_activated_user, define_metadata_pointer_url, load_config):
-    UserMetadata.base_url = load_config.get('CIC_META_URL')
+    PersonMetadata.base_url = load_config.get('CIC_META_URL')
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
+    person_metadata_client = PersonMetadata(identifier=identifier)
 
-    assert user_metadata_client.url == define_metadata_pointer_url
+    assert person_metadata_client.url == define_metadata_pointer_url
 
 
-def test_create_user_metadata(caplog,
-                              create_activated_user,
-                              define_metadata_pointer_url,
-                              load_config,
-                              mock_meta_post_response,
-                              person_metadata):
+def test_create_person_metadata(caplog,
+                                create_activated_user,
+                                define_metadata_pointer_url,
+                                load_config,
+                                mock_meta_post_response,
+                                person_metadata):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
+    person_metadata_client = PersonMetadata(identifier=identifier)
 
     with requests_mock.Mocker(real_http=False) as request_mocker:
         request_mocker.register_uri(
@@ -38,7 +38,7 @@ def test_create_user_metadata(caplog,
             reason='CREATED',
             content=json.dumps(mock_meta_post_response).encode('utf-8')
         )
-        user_metadata_client.create(data=person_metadata)
+        person_metadata_client.create(data=person_metadata)
         assert 'Get signed material response status: 201' in caplog.text
 
     with pytest.raises(RuntimeError) as error:
@@ -49,19 +49,19 @@ def test_create_user_metadata(caplog,
                 status_code=400,
                 reason='BAD REQUEST'
             )
-            user_metadata_client.create(data=person_metadata)
+            person_metadata_client.create(data=person_metadata)
         assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}'
 
 
-def test_edit_user_metadata(caplog,
-                            create_activated_user,
-                            define_metadata_pointer_url,
-                            load_config,
-                            person_metadata,
-                            setup_metadata_signer):
+def test_edit_person_metadata(caplog,
+                              create_activated_user,
+                              define_metadata_pointer_url,
+                              load_config,
+                              person_metadata,
+                              setup_metadata_signer):
     Signer.gpg_passphrase = load_config.get('KEYS_PASSPHRASE')
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
+    person_metadata_client = PersonMetadata(identifier=identifier)
     with requests_mock.Mocker(real_http=False) as request_mocker:
         request_mocker.register_uri(
             'PUT',
@@ -69,7 +69,7 @@ def test_edit_user_metadata(caplog,
             status_code=200,
             reason='OK'
         )
-        user_metadata_client.edit(data=person_metadata, engine='pgp')
+        person_metadata_client.edit(data=person_metadata)
         assert 'Signed content submission status: 200' in caplog.text
 
     with pytest.raises(RuntimeError) as error:
@@ -80,7 +80,7 @@ def test_edit_user_metadata(caplog,
                 status_code=400,
                 reason='BAD REQUEST'
             )
-            user_metadata_client.edit(data=person_metadata, engine='pgp')
+            person_metadata_client.edit(data=person_metadata)
         assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}'
 
 
@@ -92,7 +92,7 @@ def test_get_user_metadata(caplog,
                            person_metadata,
                            setup_metadata_signer):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address)
-    user_metadata_client = UserMetadata(identifier=identifier)
+    person_metadata_client = PersonMetadata(identifier=identifier)
     with requests_mock.Mocker(real_http=False) as request_mocker:
         request_mocker.register_uri(
             'GET',
@@ -101,7 +101,7 @@ def test_get_user_metadata(caplog,
             content=json.dumps(person_metadata).encode('utf-8'),
             reason='OK'
         )
-        user_metadata_client.query()
+        person_metadata_client.query()
         assert 'Get latest data status: 200' in caplog.text
     key = generate_metadata_pointer(
         identifier=identifier,
@@ -118,6 +118,6 @@ def test_get_user_metadata(caplog,
                 status_code=404,
                 reason='NOT FOUND'
             )
-            user_metadata_client.query()
+            person_metadata_client.query()
             assert 'The data is not available and might need to be added.' in caplog.text
         assert str(error.value) == f'400 Client Error: NOT FOUND for url: {define_metadata_pointer_url}'

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py

@@ -15,7 +15,7 @@ from cic_ussd.state_machine.logic.user import (
     get_user_metadata,
     save_complete_user_metadata,
     process_gender_user_input,
-    save_profile_attribute_to_session_data,
+    save_metadata_attribute_to_session_data,
     update_account_status_to_active)
 
 
@@ -41,14 +41,14 @@ def test_update_account_status_to_active(create_pending_user, create_in_db_ussd_
     ("enter_location", "location", "Kangemi", "Kangemi"),
     ("enter_products", "products", "Mandazi", "Mandazi"),
 ])
-def test_save_save_profile_attribute_to_session_data(current_state,
-                                                     expected_key,
-                                                     expected_result,
-                                                     user_input,
-                                                     celery_session_worker,
-                                                     create_activated_user,
-                                                     create_in_db_ussd_session,
-                                                     create_in_redis_ussd_session):
+def test_save_metadata_attribute_to_session_data(current_state,
+                                                 expected_key,
+                                                 expected_result,
+                                                 user_input,
+                                                 celery_session_worker,
+                                                 create_activated_user,
+                                                 create_in_db_ussd_session,
+                                                 create_in_redis_ussd_session):
     create_in_db_ussd_session.state = current_state
     serialized_in_db_ussd_session = create_in_db_ussd_session.to_json()
     state_machine_data = (user_input, serialized_in_db_ussd_session, create_activated_user)
@@ -56,7 +56,7 @@ def test_save_save_profile_attribute_to_session_data(current_state,
     in_memory_ussd_session = json.loads(in_memory_ussd_session)
     assert in_memory_ussd_session.get('session_data') == {}
     serialized_in_db_ussd_session['state'] = current_state
-    save_profile_attribute_to_session_data(state_machine_data=state_machine_data)
+    save_metadata_attribute_to_session_data(state_machine_data=state_machine_data)
 
     in_memory_ussd_session = InMemoryStore.cache.get('AT974186')
     in_memory_ussd_session = json.loads(in_memory_ussd_session)
@@ -82,23 +82,23 @@ def test_format_user_metadata(create_activated_user,
     from cic_types.models.person import Person
     formatted_user_metadata = format_user_metadata(metadata=complete_user_metadata, user=create_activated_user)
     person = Person()
-    user_metadata = person.deserialize(metadata=formatted_user_metadata)
+    user_metadata = person.deserialize(person_data=formatted_user_metadata)
     assert formatted_user_metadata == user_metadata.serialize()
 
 
 def test_save_complete_user_metadata(celery_session_worker,
-                                        complete_user_metadata,
-                                        create_activated_user,
-                                        create_in_redis_ussd_session,
-                                        mocker,
-                                        setup_chain_spec,
-                                        ussd_session_data):
+                                     complete_user_metadata,
+                                     create_activated_user,
+                                     create_in_redis_ussd_session,
+                                     mocker,
+                                     setup_chain_spec,
+                                     ussd_session_data):
     ussd_session = create_in_redis_ussd_session.get(ussd_session_data.get('external_session_id'))
     ussd_session = json.loads(ussd_session)
     ussd_session['session_data'] = complete_user_metadata
     user_metadata = format_user_metadata(metadata=ussd_session.get('session_data'), user=create_activated_user)
     state_machine_data = ('', ussd_session, create_activated_user)
-    mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_user_metadata.apply_async')
+    mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_person_metadata.apply_async')
     save_complete_user_metadata(state_machine_data=state_machine_data)
     mocked_create_metadata_task.assert_called_with(
         (user_metadata, create_activated_user.blockchain_address),
@@ -127,7 +127,7 @@ def test_edit_user_metadata_attribute(celery_session_worker,
     }
     state_machine_data = ('', ussd_session, create_activated_user)
 
-    mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.edit_user_metadata.apply_async')
+    mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.edit_person_metadata.apply_async')
     edit_user_metadata_attribute(state_machine_data=state_machine_data)
     person_metadata['location']['area_name'] = 'nairobi'
     mocked_edit_metadata.assert_called_with(
@@ -146,7 +146,7 @@ def test_get_user_metadata_attribute(celery_session_worker,
     ussd_session = json.loads(ussd_session)
     state_machine_data = ('', ussd_session, create_activated_user)
 
-    mocked_get_metadata = mocker.patch('cic_ussd.tasks.metadata.query_user_metadata.apply_async')
+    mocked_get_metadata = mocker.patch('cic_ussd.tasks.metadata.query_person_metadata.apply_async')
     get_user_metadata(state_machine_data=state_machine_data)
     mocked_get_metadata.assert_called_with(
         (create_activated_user.blockchain_address,),

apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler_tasks.py

@@ -8,7 +8,7 @@ import celery
 import pytest
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.error import ActionDataNotFoundError
 from cic_ussd.conversions import from_wei
 
@@ -29,7 +29,7 @@ def test_successful_process_account_creation_callback_task(account_creation_acti
     # WARNING: [THE SETTING OF THE ROOT ID IS A HACK AND SHOULD BE REVIEWED OR IMPROVED]
     mocked_task_request.root_id = task_id
 
-    user = init_database.query(User).filter_by(phone_number=phone_number).first()
+    user = init_database.query(Account).filter_by(phone_number=phone_number).first()
     assert user is None
 
     redis_cache = init_redis_cache
@@ -48,7 +48,7 @@ def test_successful_process_account_creation_callback_task(account_creation_acti
     )
     s_process_callback_request.apply_async().get()
 
-    user = init_database.query(User).filter_by(phone_number=phone_number).first()
+    user = init_database.query(Account).filter_by(phone_number=phone_number).first()
     assert user.blockchain_address == result
 
     action_data = redis_cache.get(task_id)

apps/cic-ussd/tests/cic_ussd/test_requests.py

@@ -2,7 +2,7 @@
 import json
 
 # local imports
-from cic_ussd.db.models.user import User
+from cic_ussd.db.models.account import Account
 from cic_ussd.requests import (get_query_parameters,
                                get_request_endpoint,
                                get_request_method,
@@ -58,8 +58,8 @@ def test_process_locked_accounts_requests(create_locked_accounts, valid_locked_a
     assert len(locked_account_addresses) == 10
 
     # check that blockchain addresses are ordered by most recently accessed
-    user_1 = User.session.query(User).filter_by(blockchain_address=locked_account_addresses[2]).first()
-    user_2 = User.session.query(User).filter_by(blockchain_address=locked_account_addresses[7]).first()
+    user_1 = Account.session.query(Account).filter_by(blockchain_address=locked_account_addresses[2]).first()
+    user_2 = Account.session.query(Account).filter_by(blockchain_address=locked_account_addresses[7]).first()
 
     assert user_1.updated > user_2.updated
 

apps/cic-ussd/tests/fixtures/config.py

@@ -18,7 +18,7 @@ from cic_ussd.files.local_files import create_local_file_data_stores, json_file_
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.metadata.signer import Signer
-from cic_ussd.metadata.user import UserMetadata
+from cic_ussd.metadata.person import PersonMetadata
 from cic_ussd.state_machine import UssdStateMachine
 
 
@@ -121,9 +121,9 @@ def setup_metadata_signer(load_config):
 @pytest.fixture(scope='function')
 def define_metadata_pointer_url(load_config, create_activated_user):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address)
-    UserMetadata.base_url = load_config.get('CIC_META_URL')
-    user_metadata_client = UserMetadata(identifier=identifier)
-    return user_metadata_client.url
+    PersonMetadata.base_url = load_config.get('CIC_META_URL')
+    person_metadata_client = PersonMetadata(identifier=identifier)
+    return person_metadata_client.url
 
 
 @pytest.fixture(scope='function')

apps/cic-ussd/tests/fixtures/user.py

@@ -9,7 +9,7 @@ from cic_types.models.person import generate_metadata_pointer
 from faker import Faker
 
 # local imports
-from cic_ussd.db.models.user import AccountStatus, User
+from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.redis import cache_data
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 
@@ -19,7 +19,7 @@ fake = Faker()
 
 @pytest.fixture(scope='function')
 def create_activated_user(init_database, set_fernet_key):
-    user = User(
+    user = Account(
         blockchain_address='0xFD9c5aD15C72C6F60f1a119A608931226674243f',
         phone_number='+25498765432'
     )
@@ -33,7 +33,7 @@ def create_activated_user(init_database, set_fernet_key):
 
 @pytest.fixture(scope='function')
 def create_valid_tx_recipient(init_database, set_fernet_key):
-    user = User(
+    user = Account(
         blockchain_address='0xd6204101012270Bf2558EDcFEd595938d1847bf0',
         phone_number='+25498765432'
     )
@@ -47,7 +47,7 @@ def create_valid_tx_recipient(init_database, set_fernet_key):
 
 @pytest.fixture(scope='function')
 def create_valid_tx_sender(init_database, set_fernet_key):
-    user = User(
+    user = Account(
         blockchain_address='0xd6204101012270Bf2558EDcFEd595938d1847bf1',
         phone_number='+25498765433'
     )
@@ -61,7 +61,7 @@ def create_valid_tx_sender(init_database, set_fernet_key):
 
 @pytest.fixture(scope='function')
 def create_pending_user(init_database, set_fernet_key):
-    user = User(
+    user = Account(
         blockchain_address='0x0ebdea8612c1b05d952c036859266c7f2cfcd6a29842d9c6cce3b9f1ba427588',
         phone_number='+25498765432'
     )
@@ -72,7 +72,7 @@ def create_pending_user(init_database, set_fernet_key):
 
 @pytest.fixture(scope='function')
 def create_pin_blocked_user(init_database, set_fernet_key):
-    user = User(
+    user = Account(
         blockchain_address='0x0ebdea8612c1b05d952c036859266c7f2cfcd6a29842d9c6cce3b9f1ba427588',
         phone_number='+25498765432'
     )
@@ -90,7 +90,7 @@ def create_locked_accounts(init_database, set_fernet_key):
         blockchain_address = str(uuid.uuid4())
         phone_number = fake.phone_number()
         pin = f'{randint(1000, 9999)}'
-        user = User(phone_number=phone_number, blockchain_address=blockchain_address)
+        user = Account(phone_number=phone_number, blockchain_address=blockchain_address)
         user.create_password(password=pin)
         user.failed_pin_attempts = 3
         user.account_status = AccountStatus.LOCKED.value

apps/cic-ussd/transitions/exit_transitions.json

@@ -38,7 +38,7 @@
   {
     "trigger": "scan_data",
     "source": "exit_invalid_recipient",
-    "dest": "send_enter_recipient",
+    "dest": "enter_transaction_recipient",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
   },
   {
@@ -49,13 +49,13 @@
     "after": "cic_ussd.state_machine.logic.sms.upsell_unregistered_recipient"
   },
   {
-    "trigger": "feed_char",
+    "trigger": "scan_data",
     "source": "exit_successful_transaction",
     "dest": "start",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
   },
   {
-    "trigger": "feed_char",
+    "trigger": "scan_data",
     "source": "exit_successful_transaction",
     "dest": "exit",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_ninety_nine_selected"

apps/cic-ussd/transitions/user_metadata_transitions.json

@@ -26,18 +26,18 @@
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "standard_pin_authorization",
+    "dest": "display_metadata_pin_authorization",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected"
   },
   {
     "trigger": "scan_data",
-    "source": "standard_pin_authorization",
+    "source": "display_metadata_pin_authorization",
     "dest": "display_user_metadata",
     "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin"
   },
   {
     "trigger": "scan_data",
-    "source": "standard_pin_authorization",
+    "source": "display_metadata_pin_authorization",
     "dest": "exit_pin_blocked",
     "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
   },

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -55,8 +55,8 @@ en:
       4. Edit products
       5. View my profile
       0. Back
-    display_user_profile_data: |-
-      END Your details are:
+    display_user_metadata: |-
+      CON Your details are:
         Name: %{full_name}
         Gender: %{gender}
         Location: %{location}
@@ -85,7 +85,7 @@ en:
       retry: |-
         CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
         0. Back
-    standard_pin_authorization:
+    display_metadata_pin_authorization:
       first: |-
         CON Please enter your PIN.
         0. Back

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -56,7 +56,7 @@ sw:
       5. Angalia wasifu wako
       0. Nyuma
     display_user_metadata: |-
-      END Wasifu wako una maelezo yafuatayo:
+      CON Wasifu wako una maelezo yafuatayo:
         Jina: %{full_name}
         Jinsia: %{gender}
         Eneo: %{location}

apps/contract-migration/docker/Dockerfile

@@ -57,19 +57,22 @@ WORKDIR /home/grassroots
 USER grassroots
 
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a61
-ARG cic_eth_version=0.11.0b1
-ARG sarafu_faucet_version=0.0.2a19
+ARG cic_base_version=0.1.2a77
+ARG cic_eth_version=0.11.0b6
+ARG sarafu_faucet_version=0.0.2a28
+ARG sarafu_token_version==0.0.1a6
 ARG cic_contracts_version=0.0.2a2
 RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
 	cic-contracts==$cic_contracts_version \
-	sarafu-faucet==$sarafu_faucet_version
+	sarafu-faucet==$sarafu_faucet_version \
+	sarafu-token==$sarafu_token_version
 
 FROM python:3.8.6-slim-buster as runtime-image
 
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends gnupg libpq-dev
+RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
+RUN apt-get install -y --no-install-recommends jq
 
 COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
 COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/

apps/contract-migration/reset.sh

@@ -2,78 +2,112 @@
 
 set -a
 
-DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
+DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
-keystore_file=$(realpath ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c)
+DEV_FAUCET_AMOUNT=${DEV_FAUCET_AMOUNT:-0}
+DEV_ETH_KEYSTORE_FILE=${DEV_ETH_KEYSTORE_FILE:-`realpath ./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c`}
+
+set -e
+
+DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $DEV_ETH_KEYSTORE_FILE | jq -r .address)`
+
+if [ ! -z $DEV_ETH_GAS_PRICE ]; then
+	gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
+	>&2 echo using static gas price $DEV_ETH_GAS_PRICE
+fi
+
+if [[ $DEV_TOKEN_TYPE != 'giftable' && $DEV_TOKEN_TYPE != 'sarafu' ]]; then
+	echo $DEV_TOKEN_TYPE
+	>&2 echo DEV_TOKEN_TYPE must be one of [giftable,sarafu]
+	exit 1
+fi
 
 echo "environment:"
 printenv
 echo \n
 
+echo "using wallet address '$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER' from keystore file $DEV_ETH_KEYSTORE_FILE"
+
 # This is a grassroots team convention for building the Bancor contracts using the bancor protocol repository truffle setup
 # Running this in docker-internal dev container (built from Docker folder in this repo) will write a
 # source-able env file to CIC_DATA_DIR. Services dependent on these contracts can mount this file OR 
 # define these parameters at runtime
 # pushd /usr/src
 
+if [ -z $CIC_DATA_DIR ]; then
+	CIC_DATA_DIR=`mktemp -d`
+fi
+>&2 echo using data dir $CIC_DATA_DIR
+
 init_level_file=${CIC_DATA_DIR}/.init
 if [ ! -f ${CIC_DATA_DIR}/.init ]; then
   echo "Creating .init file..."
   mkdir -p $CIC_DATA_DIR
-  touch /tmp/cic/config/.init
+  touch $CIC_DATA_DIR/.init
 #   touch $init_level_file
 fi
 echo -n 1 > $init_level_file
 
 # Abort on any error (including if wait-for-it fails).
-set -e
 
 # Wait for the backend to be up, if we know where it is.
 if [[ -n "${ETH_PROVIDER}" ]]; then
-	echo "waiting for ${ETH_PROVIDER}..."
-  	./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 
-	DEV_RESERVE_ADDRESS=`giftable-token-deploy -p $ETH_PROVIDER -y $keystore_file -i $CIC_CHAIN_SPEC -v -w --name "Sarafu" --symbol "SRF" --decimals 6`
-	giftable-token-gift -p $ETH_PROVIDER -y $keystore_file -i $CIC_CHAIN_SPEC -v -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+	if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
+		echo "waiting for ${ETH_PROVIDER}..."
+		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
+	fi
 
-	#BANCOR_REGISTRY_ADDRESS=`cic-bancor-deploy --bancor-dir /usr/local/share/cic/bancor -z $DEV_ETH_RESERVE_ADDRESS -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
+	if [ $DEV_TOKEN_TYPE == 'giftable' ]; then
+		>&2 echo "deploying 'giftable token'"
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Giftable Token" --symbol "GFT" --decimals 6 -vv`
+	else
+		>&2 echo "deploying 'sarafu' token'"
+		DEV_RESERVE_ADDRESS=`sarafu-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Sarafu" --decimals 6 -vv SRF $DEV_SARAFU_DEMURRAGE_LEVEL`
+	fi
+	giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+
+	#BANCOR_REGISTRY_ADDRESS=`cic-bancor-deploy $gas_price_arg --bancor-dir /usr/local/share/cic/bancor -z $DEV_ETH_RESERVE_ADDRESS -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
 
 	>&2 echo "deploy account index contract"
-	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $keystore_file -vv -w`
+	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -w`
 	>&2 echo "add deployer address as account index writer"
-	eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
-	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy -i $CIC_CHAIN_SPEC -y $keystore_file --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
-	#cic-registry-set -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -k BancorRegistry -p $ETH_PROVIDER $BANCOR_REGISTRY_ADDRESS -vv
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
+	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
 
 	# Deploy address declarator registry
 	>&2 echo "deploy address declarator contract"
 	declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
-	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v $declarator_description`
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
+	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
 	>&2 echo "deploy address declarator contract"
-	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v`
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
+	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 
 	# Deploy token index contract
 	>&2 echo "deploy token index contract"
-	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v`
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
+	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
 	>&2 echo "add reserve token to token index"
-	eth-token-index-add -w -y $keystore_file  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -a $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
+	eth-token-index-add $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -a $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
 
 	# Sarafu faucet contract
 	>&2 echo "deploy token faucet contract"
-	DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS`
-	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS
+	DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS
 	>&2 echo "set faucet as token minter"
-	giftable-token-minter -w -y $keystore_file -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+	giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+
+	>&2 echo "set token faucet amount"
+	sarafu-faucet-set $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS -vv $DEV_FAUCET_AMOUNT
 
 
 else

apps/contract-migration/scripts/README.md

@@ -2,88 +2,234 @@
 
 This folder contains tools to generate and import test data.
 
-## DATA CREATION
+## OVERVIEW
 
-Does not need the cluster to run.
+Three sets of tools are available, sorted by respective subdirectories.
 
-Vanilla:
+* **eth**: Import using sovereign wallets.
+* **cic_eth**: Import using the `cic_eth` custodial engine.
+* **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`)
+
+Each of the modules include two main scripts:
+
+* **import_users.py**: Registers all created accounts in the network
+* **import_balance.py**: Transfer an opening balance using an external keystore wallet
+
+The balance script will sync with the blockchain, processing transactions and triggering actions when it finds. In its current version it does not keep track of any other state, so it will run indefinitly and needs You the Human to decide when it has done what it needs to do.
+
+
+In addition the following common tools are available:
+
+* **create_import_users.py**: User creation script
+* **verify.py**: Import verification script
+* **cic_meta**: Metadata imports
+
+
+## REQUIREMENTS
+
+A virtual environment for the python scripts is recommended. We know it works with `python 3.8.x`. Let us know if you run it successfully with other minor versions.
+
+```
+python3 -m venv .venv
+source .venv/bin/activate
+```
+
+Install all requirements from the `requirements.txt` file:
+
+`pip install --extra-index-url https://pip.grassrootseconomics.net:8433 -r requirements.txt`
+
+
+If you are importing metadata, also do ye olde:
+
+`npm install`
+
+
+## HOW TO USE
+
+### Step 1 - Data creation
+
+Before running any of the imports, the user data to import has to be generated and saved to disk.
+
+The script does not need any services to run.
+
+Vanilla version:
 
 `python create_import_users.py [--dir <datadir>] <number_of_users>`
 
-If you want to use the `import_balance.py` script to add to the user's balance from an external address, add:
+If you want to use a `import_balance.py` script to add to the user's balance from an external address, use:
 
 `python create_import_users.py --gift-threshold <max_units_to_send> [--dir <datadir>] <number_of_users>`
 
 
-## IMPORT
+### Step 2 - Services
 
-Make sure the following is running in the cluster:
-	* eth
-	* postgres
-	* redis
-	* cic-meta-server
+Unless you know what you are doing, start with a clean slate, and execute (in the repository root):
+
+`docker-compose down -v`
+
+Then go through, in sequence:
+
+#### Base requirements
+
+If you are importing using `eth` and _not_ importing metadata, then the only service you need running in the cluster is:
+* eth
+
+In all other cases you will _also_ need:
+* postgres
+* redis
 
 
-If using the _custodial_ alternative for user imports, also run:
-	* cic-eth-tasker
-	* cic-eth-dispatcher
-	* cic-eth-tracker
+#### EVM provisions
+
+This step is needed in *all* cases.
+
+`RUN_MASK=1 docker-compose up contract-migration`
+
+After this step is run, you can find top-level ethereum addresses (like the cic registry address, which you will need below) in `<repository_root>/service-configs/.env`
 
 
-You will want to run these in sequence:
+#### Custodial provisions
+
+This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
+
+`RUN_MASK=2 docker-compose up contract-migration`
 
 
-## 1. Metadata
+#### Custodial services
 
-`node import_meta.js <datadir> <number_of_users>`
+If importing using `cic_eth` or `cic_ussd` also run:
+* cic-eth-tasker
+* cic-eth-dispatcher
+* cic-eth-tracker
+* cic-eth-retrier
+
+If importing using `cic_ussd` also run:
+* cic-ussd-tasker
+* cic-ussd-server
+* cic-notify-tasker
+
+If metadata is to be imported, also run:
+* cic-meta-server
+
+
+
+### Step 3 - User imports
+
+If you did not change the docker-compose setup, your `eth_provider` the you need for the commands below will be `http://localhost:63545`.
+
+Only run _one_ of the alternatives.
+
+The keystore file used for transferring external opening balances tracker is relative to the directory you found this README in. Of course you can use a different wallet, but then you will have to provide it with tokens yourself (hint: `../reset.sh`)
+
+All external balance transactions are saved in raw wire format in `<datadir>/txs`, with transaction hash as file name.
+
+
+
+#### Alternative 1 - Sovereign wallet import - `eth` 
+
+
+First, make a note of the **block height** before running anything.
+
+To import, run to _completion_:
+
+`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
+
+After the script completes, keystore files for all generated accouts will be found in `<datadir>/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one).
+
+Then run:
+
+`python eth/import_balance.py -v -c config -r <cic_registry_address> -p <eth_provider> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
+
+
+
+#### Alternative 2 - Custodial engine import - `cic_eth`
+
+Run in sequence, in first terminal:
+
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
+
+In another terminal:
+
+`python cic_eth/import_users.py -v -c config --redis-host-callback <redis_hostname_in_docker> out`
+
+The `redis_hostname_in_docker` value is the hostname required to reach the redis server from within the docker cluster, and should be `redis` if you left the docker-compose unchanged. The `import_users` script will receive the address of each newly created custodial account on a redis subscription fed by a callback task in the `cic_eth` account creation task chain.
+
+
+#### Alternative 3 - USSD import - `cic_ussd`
+
+If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`.
+
+`celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker <redis_url>`
+
+Then, in sequence, run in first terminal:
+
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
+
+In second terminal:
+
+`python cic_ussd/import_users.py -v -c config out`
+
+The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
+
+The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
+
+
+### Step 4 - Metadata import (optional)
+
+The metadata import scripts can be run at any time after step 1 has been completed.
+
+
+#### Importing user metadata
+
+To import the main user metadata structs, run:
+
+`node cic_meta/import_meta.js <datadir> <number_of_users>`
 
 Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service.
 
-
-## 2. Balances
-
-(Only if you used the `--gift-threshold` option above)
-
-`python -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> --head -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
-
-This will monitor new mined blocks and send balances to the newly created accounts.
+If _number of users_ is omitted the script will run until manually interrupted.
 
 
-### 3. Users
+#### Importing phone pointer
 
-Only use **one** of the following
+`node cic_meta/import_meta_phone.js <datadir> <number_of_users>`
 
-#### Custodial
+If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
 
-This alternative generates accounts using the `cic-eth` custodial engine
 
-Without any modifications to the cluster and config files:
+### Step 5 - Verify
 
-`python import_users.py -c config --redis-host-callback redis <datadir>`
+`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>` 
 
-** A note on the The callback**:  The script uses a redis callback to retrieve the newly generated custodial address. This is the redis server _from the perspective of the cic-eth component_.
+Included checks:
+* Private key is in cic-eth keystore
+* Address is in accounts index
+* Address has gas balance
+* Address has triggered the token faucet
+* Address has token balance matching the gift threshold 
+* Personal metadata can be retrieved and has exact match
+* Phone pointer metadata can be retrieved and matches address
+* USSD menu response is initial state after registration
 
-#### Sovereign
+Checks can be selectively included and excluded. See `--help` for details.
 
-This alternative generates keystore files, while  registering corresponding addresses in the accounts registry directly
-
-`python import_sovereign_users.py -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
-
-A `keystore` sub-directory in the data path is created, with ethereum keystore files for all generated private keys. Passphrase is set to empty string for all of them.
-
-## VERIFY
-
-`python verify.py -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> <datadir>` 
-
-Checks
-	* Private key is in cic-eth keystore
-	* Address is in accounts index
-	* Address has balance matching the gift threshold 
-	* Metadata can be retrieved and has exact match
+Will output one line for each check, with name of check and number of errors found per check.
 
 Should exit with code 0 if all input data is found in the respective services.
 
 
 ## KNOWN ISSUES
 
-If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount.
+- If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount.
+
+- When the account callback in `cic_eth` fails, the `cic_eth/import_users.py` script will exit with a cryptic complaint concerning a `None` value.
+
+- Sovereign import scripts use the same keystore, and running them simultaneously will mess up the transaction nonce sequence. Better would be to use two different keystore wallets so balance and users scripts can be run simultaneously.
+
+- `pycrypto` and `pycryptodome` _have to be installed in that order_. If you get errors concerning `Crypto.KDF` then uninstall both and re-install in that order. Make sure you use the versions listed in `requirements.txt`. `pycryptodome` is a legacy dependency and will be removed as soon as possible.
+
+- Sovereign import script is very slow because it's scrypt'ing keystore files for the accounts that it creates. An improvement would be optional and/or asynchronous keyfile generation.
+
+- Running the balance script should be _optional_ in all cases, but is currently required in the case of `cic_ussd` because it is needed to generate the metadata. An improvement would be moving the task to `import_users.py`, for a different queue than the balance tx handler.
+
+- `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it.

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -10,14 +10,14 @@ import hashlib
 import csv
 import json
 
-# third-party impotts
+# external imports
 import eth_abi
 import confini
 from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
@@ -25,13 +25,13 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
@@ -42,7 +42,7 @@ from cic_types.models.person import Person
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = '/usr/local/etc/cic-syncer'
+config_dir = './config'
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
@@ -162,6 +162,15 @@ class Handler:
             (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
             logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
             r = conn.do(o)
+
+            tx_path = os.path.join(
+                    user_dir,
+                    'txs',
+                    strip_0x(tx_hash_hex),
+                    )
+            f = open(tx_path, 'w')
+            f.write(strip_0x(o['params'][0]))
+            f.close()
 #        except TypeError as e:
 #            logg.warning('typerror {}'.format(e))
 #            pass
@@ -195,8 +204,8 @@ class Handler:
 #        return b
 
 
-def progress_callback(block_number, tx_index, s):
-    sys.stdout.write(str(s).ljust(200) + "\n")
+def progress_callback(block_number, tx_index):
+    sys.stdout.write(str(block_number).ljust(200) + "\n")
 
 
 
@@ -290,7 +299,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, progress_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/contract-migration/scripts/cic_eth/import_users.py

@@ -7,6 +7,7 @@ import argparse
 import uuid
 import datetime
 import time
+import phonenumbers
 from glob import glob
 
 # third-party imports
@@ -17,7 +18,7 @@ from hexathon import (
         add_0x,
         strip_0x,
         )
-from chainlib.eth.address import to_checksum
+from chainlib.eth.address import to_checksum_address
 from cic_types.models.person import Person
 from cic_eth.api.api_task import Api
 from chainlib.chain import ChainSpec
@@ -75,9 +76,15 @@ os.makedirs(user_new_dir)
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
+phone_dir = os.path.join(args.user_dir, 'phone')
+os.makedirs(os.path.join(phone_dir, 'meta'))
+
 user_old_dir = os.path.join(args.user_dir, 'old')
 os.stat(user_old_dir)
 
+txs_dir = os.path.join(args.user_dir, 'txs')
+os.makedirs(txs_dir)
+
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 
@@ -165,12 +172,32 @@ if __name__ == '__main__':
             f.write(json.dumps(o))
             f.close()
 
-            #old_address = to_checksum(add_0x(y[:len(y)-5]))
             #fi.write('{},{}\n'.format(new_address, old_address))
             meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
             meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
             os.symlink(os.path.realpath(filepath), meta_filepath)
 
+            phone_object = phonenumbers.parse(u.tel)
+            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+            meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
+            meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
+
+            filepath = os.path.join(
+                    phone_dir,
+                    'new',
+                    meta_phone_key[:2].upper(),
+                    meta_phone_key[2:4].upper(),
+                    meta_phone_key.upper(),
+                    )
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+            
+            f = open(filepath, 'w')
+            f.write(to_checksum_address(new_address_clean))
+            f.close()
+
+            os.symlink(os.path.realpath(filepath), meta_phone_filepath)
+
+
             i += 1
             sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")