Updates verification steps.

should have dropped the user flag

See merge request grassrootseconomics/cic-internal-integration!136

apps/cic-cache/.config/database.ini

@@ -6,3 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
+DEBUG=

apps/cic-cache/.config/test/database.ini

@@ -6,3 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
+DEBUG=

apps/cic-cache/cic_cache/db/__init__.py

@@ -2,9 +2,14 @@
 import logging
 
 # local imports
-from .list import list_transactions_mined
-from .list import list_transactions_account_mined
-from .list import add_transaction
+from .list import (
+        list_transactions_mined,
+        list_transactions_account_mined,
+        add_transaction,
+        tag_transaction,
+        add_tag,
+    )
+
 
 logg = logging.getLogger()
 

apps/cic-cache/cic_cache/db/list.py

@@ -2,8 +2,9 @@
 import logging
 import datetime
 
-# third-party imports
+# external imports
 from cic_cache.db.models.base import SessionBase
+from sqlalchemy import text
 
 logg = logging.getLogger()
 
@@ -50,7 +51,8 @@ def list_transactions_account_mined(
 
 
 def add_transaction(
-        session, tx_hash,
+        session,
+        tx_hash,
         block_number,
         tx_index,
         sender,
@@ -62,6 +64,33 @@ def add_transaction(
         success,
         timestamp,
         ):
+    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param tx_hash: Transaction hash
+    :type tx_hash: str, 0x-hex
+    :param block_number: Block number
+    :type block_number: int
+    :param tx_index: Transaction index in block
+    :type tx_index: int
+    :param sender: Ethereum address of effective sender
+    :type sender: str, 0x-hex
+    :param receiver: Ethereum address of effective recipient
+    :type receiver: str, 0x-hex
+    :param source_token: Ethereum address of token used by sender
+    :type source_token: str, 0x-hex
+    :param destination_token: Ethereum address of token received by recipient
+    :type destination_token: str, 0x-hex
+    :param from_value: Source token value spent in transaction
+    :type from_value: int
+    :param to_value: Destination token value received in transaction
+    :type to_value: int
+    :param success: True if code execution on network was successful
+    :type success: bool
+    :param date_block: Block timestamp
+    :type date_block: datetime
+    """
     date_block = datetime.datetime.fromtimestamp(timestamp)
     s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
             tx_hash,
@@ -77,3 +106,74 @@ def add_transaction(
             date_block,
             )
     session.execute(s)
+
+
+
+def tag_transaction(
+        session,
+        tx_hash,
+        name,
+        domain=None,
+        ):
+    """Tag a single transaction with a single tag.
+
+    Tag must already exist in storage.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param tx_hash: Transaction hash
+    :type tx_hash: str, 0x-hex
+    :param name: Tag value
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    :raises ValueError: Unknown tag or transaction hash
+
+    """
+
+    s = text("SELECT id from tx where tx_hash = :a")
+    r = session.execute(s, {'a': tx_hash}).fetchall()
+    tx_id = r[0].values()[0]
+
+    if tx_id == None:
+        raise ValueError('unknown tx hash {}'.format(tx_hash))
+
+    #s = text("SELECT id from tag where value = :a and domain = :b")
+    if domain == None:
+        s = text("SELECT id from tag where value = :a")
+    else:
+        s = text("SELECT id from tag where value = :a and domain = :b")
+    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
+    tag_id = r[0].values()[0]
+
+    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
+
+    if tag_id == None:
+        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
+
+    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
+    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
+
+
+def add_tag(
+        session,
+        name,
+        domain=None,
+        ):
+    """Add a single tag to storage.
+
+    :param session: Persistent storage session object
+    :type session: SQLAlchemy session
+    :param name: Tag value
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    :raises sqlalchemy.exc.IntegrityError: Tag already exists
+    """
+
+    s = None
+    if domain == None: 
+        s = text("INSERT INTO tag (value) VALUES (:b)")
+    else:
+        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
+    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -0,0 +1,38 @@
+"""Transaction tags
+
+Revision ID: aaf2bdce7d6e
+Revises: 6604de4203e2
+Create Date: 2021-05-01 09:20:20.775082
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'aaf2bdce7d6e'
+down_revision = '6604de4203e2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'tag',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('domain', sa.String(), nullable=True),
+            sa.Column('value', sa.String(), nullable=False),
+            )
+    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
+
+    op.create_table(
+            'tag_tx_link',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
+            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
+                )
+
+def downgrade():
+    op.drop_table('tag_tx_link')
+    op.drop_index('idx_tag_domain_value')
+    op.drop_table('tag')

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,2 +1,27 @@
-class SyncFilter:
-    pass
+class TagSyncFilter:
+    """Holds tag name and domain for an implementing filter.
+
+    :param name: Tag value 
+    :type name: str
+    :param domain: Tag domain
+    :type domain: str
+    """
+
+    def __init__(self, name, domain=None):
+        self.tag_name = name
+        self.tag_domain = domain
+
+
+    def tag(self):
+        """Return tag value/domain.
+
+        :rtype: Tuple
+        :returns: tag value/domain.
+        """
+        return (self.tag_name, self.tag_domain)
+
+
+    def __str__(self):
+        if self.tag_domain == None:
+            return self.tag_name
+        return '{}.{}'.format(self.tag_domain, self.tag_name)

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,7 +2,6 @@
 import logging
 
 # external imports
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -13,17 +12,19 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
+from eth_erc20 import ERC20
 
 # local imports
-from .base import SyncFilter
+from .base import TagSyncFilter
 from cic_cache import db as cic_cache_db
 
 logg = logging.getLogger().getChild(__name__)
 
 
-class ERC20TransferFilter(SyncFilter):
+class ERC20TransferFilter(TagSyncFilter):
 
     def __init__(self, chain_spec):
+        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
         self.chain_spec = chain_spec
 
 
@@ -46,6 +47,9 @@ class ERC20TransferFilter(SyncFilter):
         except RequestMismatchException:
             logg.debug('erc20 match but not a transfer, skipping')
             return False
+        except ValueError:
+            logg.debug('erc20 match but bogus data, skipping')
+            return False
 
         token_sender = tx.outputs[0]
         token_recipient = transfer_data[0]
@@ -67,7 +71,13 @@ class ERC20TransferFilter(SyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        #db_session.flush()
+        db_session.flush()
+        cic_cache_db.tag_transaction(
+                db_session,
+                tx.hash,
+                self.tag_name,
+                domain=self.tag_domain,
+                )
         db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -7,9 +7,10 @@ import argparse
 import sys
 import re
 
-# third-party imports
+# external imports
 import confini
 import celery
+import sqlalchemy
 import rlp
 import cic_base.config
 import cic_base.log
@@ -34,7 +35,10 @@ from chainsyncer.driver import (
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
-from cic_cache.db import dsn_from_config
+from cic_cache.db import (
+        dsn_from_config,
+        add_tag,
+        )
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
         )
@@ -59,6 +63,17 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+def register_filter_tags(filters, session):
+    for f in filters:
+        tag = f.tag()
+        try:
+            add_tag(session, tag[0], domain=tag[1])
+            session.commit()
+            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
+        except sqlalchemy.exc.IntegrityError:
+            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
+
+
 def main():
     # Connect to blockchain with chainlib
     rpc = RPCConnection.connect(chain_spec, 'default')
@@ -98,10 +113,19 @@ def main():
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
 
+    filters = [
+        erc20_transfer_filter,
+            ]
+
+    session = SessionBase.create_session()
+    register_filter_tags(filters, session)
+    session.close()
+
     i = 0
     for syncer in syncers:
         logg.debug('running syncer index {}'.format(i))
-        syncer.add_filter(erc20_transfer_filter)
+        for f in filters:
+            syncer.add_filter(f)
         r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-cache/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=
+DEBUG=0

apps/cic-cache/config/test/cic.ini

@@ -1,2 +1,4 @@
 [cic]
 registry_address = 
+chain_spec = 
+trust_address =

apps/cic-cache/config/test/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=
+DEBUG=1

apps/cic-cache/docker/Dockerfile

@@ -43,10 +43,6 @@ COPY cic-cache/config/ /usr/local/etc/cic-cache/
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
 COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
-	mkdir -p /usr/local/share/cic/solidity && \
-	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
-
 COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
 COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2a77
+cic-base~=0.1.2b8
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.4a16
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/test_requirements.txt

@@ -4,3 +4,8 @@ pytest-mock==3.3.1
 pysqlite3==0.4.3
 sqlparse==0.4.1
 pytest-celery==0.0.0a1
+eth_tester==0.5.0b3
+py-evm==0.3.0a20
+web3==5.12.2
+cic-eth-registry~=0.5.5a3
+cic-base[full]==0.1.2b8

apps/cic-cache/tests/conftest.py

@@ -3,7 +3,7 @@ import os
 import sys
 import datetime
 
-# third-party imports
+# external imports
 import pytest
 
 # local imports
@@ -84,3 +84,7 @@ def txs(
     
     session.commit()
 
+    return [
+            tx_hash_first,
+            tx_hash_second,
+            ]

apps/cic-cache/tests/filters/conftest.py

@@ -0,0 +1,3 @@
+from chainlib.eth.pytest import *
+from cic_eth_registry.pytest.fixtures_tokens import *
+

apps/cic-cache/tests/filters/test_erc20.py

@@ -0,0 +1,69 @@
+# standard imports
+import os
+import datetime
+import logging
+import json
+
+# external imports
+import pytest
+from sqlalchemy import text
+from chainlib.eth.tx import Tx
+from chainlib.eth.block import Block
+from chainlib.chain import ChainSpec
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+
+# local imports
+from cic_cache.db import add_tag
+from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
+
+logg = logging.getLogger()
+
+
+def test_cache(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    data = 'a9059cbb'
+    data += strip_0x(list_actors['alice'])
+    data += '1000'.ljust(64, '0')
+
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': foo_token,
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert r
+
+    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
+    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
+    assert r[0] == tx.hash

apps/cic-cache/tests/fixtures_config.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
@@ -13,7 +13,7 @@ logg = logging.getLogger(__file__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, '.config/test')
+    config_dir = os.path.join(root_dir, 'config/test')
     conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))

apps/cic-cache/tests/fixtures_database.py

@@ -3,13 +3,16 @@ import os
 import logging
 import re
 
-# third-party imports
+# external imports
 import pytest
 import sqlparse
+import alembic
+from alembic.config import Config as AlembicConfig
 
 # local imports
 from cic_cache.db.models.base import SessionBase
 from cic_cache.db import dsn_from_config
+from cic_cache.db import add_tag
 
 logg = logging.getLogger(__file__)
 
@@ -26,11 +29,10 @@ def database_engine(
         except FileNotFoundError:
             pass
     dsn = dsn_from_config(load_config)
-    SessionBase.connect(dsn)
+    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 
 
-# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
 @pytest.fixture(scope='function')
 def init_database(
         load_config,
@@ -38,52 +40,23 @@ def init_database(
         ):
 
     rootdir = os.path.dirname(os.path.dirname(__file__))
-    schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
-
-    if load_config.get('DATABASE_ENGINE') == 'sqlite':
-        rconn = SessionBase.engine.raw_connection()
-        f = open(os.path.join(schemadir, 'db.sql'))
-        s = f.read()
-        f.close()
-        rconn.executescript(s)
-
-    else:
-        rconn = SessionBase.engine.raw_connection()
-        rcursor = rconn.cursor()
-
-        #rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
-        #rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
-
-        f = open(os.path.join(schemadir, 'db.sql'))
-        s = f.read()
-        f.close()
-        r = re.compile(r'^[A-Z]', re.MULTILINE)
-        for l in sqlparse.parse(s):
-            strl = str(l)
-            # we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
-            if not re.search(r, strl):
-                logg.warning('skipping parsed query line {}'.format(strl))
-                continue
-            rcursor.execute(strl)
-        rconn.commit()
-
-        rcursor.execute('SET search_path TO public')
-
-# this doesn't work when run separately, no idea why
-# functions have been manually added to original schema from cic-eth 
-#        f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
-#        s = f.read()
-#        f.close()
-#        rcursor.execute(s)
-# 
-#        f = open(os.path.join(schemadir, 'proc_balances.sql'))
-#        s = f.read()
-#        f.close()
-#        rcursor.execute(s)
-
-        rcursor.close()
+    dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+    migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
+    if not os.path.isdir(migrationsdir):
+        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
+    logg.info('using migrations directory {}'.format(migrationsdir))
 
     session = SessionBase.create_session()
+
+    ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
+    ac.set_main_option('sqlalchemy.url', database_engine)
+    ac.set_main_option('script_location', migrationsdir)
+
+    alembic.command.downgrade(ac, 'base')
+    alembic.command.upgrade(ac, 'head')
+
+    session.commit()
+
     yield session
     session.commit()
     session.close()
@@ -116,3 +89,14 @@ def list_defaults(
     return {
         'block': 420000,
     }
+
+
+@pytest.fixture(scope='function')
+def tags(
+        init_database,
+        ):
+
+    add_tag(init_database, 'foo') 
+    add_tag(init_database, 'baz', domain='bar') 
+    add_tag(init_database, 'xyzzy', domain='bar') 
+    init_database.commit()

apps/cic-cache/tests/test_cache.py

@@ -4,7 +4,7 @@ import datetime
 import logging
 import json
 
-# third-party imports
+# external imports
 import pytest
 
 # local imports

apps/cic-cache/tests/test_tag.py

@@ -0,0 +1,37 @@
+import os
+import datetime
+import logging
+import json
+
+# external imports
+import pytest
+
+# local imports
+from cic_cache.db import tag_transaction
+
+logg = logging.getLogger()
+
+
+def test_cache(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        tags,
+        ):
+
+    tag_transaction(init_database, txs[0], 'foo')
+    tag_transaction(init_database, txs[0], 'baz', domain='bar')
+    tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
+
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
+    assert r[0][0] == txs[0]
+
+
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
+    assert r[0][0] == txs[0]
+
+    
+    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
+    assert r[0][0] == txs[1]

apps/cic-eth/cic_eth/eth/account.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from erc20_faucet import Faucet
 from hexathon import (
         strip_0x,
         )
@@ -20,8 +20,8 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index import AccountRegistry
-from sarafu_faucet import MinterFaucet as Faucet
+from eth_accounts_index.registry import AccountRegistry # TODO, use interface module instead (needs gas limit method)
+from sarafu_faucet import MinterFaucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -127,13 +127,13 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     if writer_address == ZERO_ADDRESS:
         session.close()
         raise RoleMissingError('call address for resgistering {}'.format(account_address))
-    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
+    account_registry_address = registry.by_name('AccountsIndex', sender_address=call_address)
    
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
-    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    gas_oracle = self.create_gas_oracle(rpc, AccountsIndex.gas)
+    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +185,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -338,7 +338,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountRegistry.parse_add_request(tx['data'])
+    tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,7 +6,6 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
-from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -16,6 +15,7 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = AddressDeclarator(chain_spec)
+    c = Declarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = AddressDeclarator.parse_declaration(r)
+        declaration_hex = Declarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -13,9 +13,7 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-# TODO: use sarafu_Faucet for both when inheritance has been implemented
-from erc20_single_shot_faucet import SingleShotFaucet
-from sarafu_faucet import MinterFaucet as Faucet
+from erc20_faucet import Faucet
 
 # local imports
 from .base import SyncFilter
@@ -71,14 +69,13 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        c = SingleShotFaucet(self.chain_spec)
-        o = c.token(faucet_contract, sender_address=self.caller_address)
+        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.amount(faucet_contract, sender_address=self.caller_address)
+        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_amount(r)
+        transfer_data['value'] = c.parse_token_amount(r)
 
         return ('tokengift', transfer_data)
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -36,6 +36,7 @@ from cic_eth.eth import (
 from cic_eth.admin import (
         debug,
         ctrl,
+        token
         )
 from cic_eth.queue import (
         query,

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.12',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b8
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./
@@ -50,8 +50,4 @@ COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
-	mkdir -p /usr/local/share/cic/solidity && \
-	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
-
 COPY util/liveness/health.sh /usr/local/bin/health.sh

apps/cic-eth/requirements.txt

@@ -1,25 +1,24 @@
-cic-base==0.1.2b5
+cic-base==0.1.2b8
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a16
-#cic-bancor~=0.0.6
+cic-eth-registry~=0.5.5a4
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a9
-erc20-transfer-authorization~=0.3.1a5
+eth_accounts_index~=0.0.11a11
+erc20-transfer-authorization~=0.3.1a6
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a9
-chainlib~=0.0.2a20
+eth-address-index~=0.1.1a11
+chainlib~=0.0.3a1
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a2
+chainsyncer[sql]~=0.0.2a4
 chainqueue~=0.0.2a2
-pysha3==1.0.2
+sarafu-faucet==0.0.3a1
 coincurve==15.0.0
-sarafu-faucet==0.0.2a28
+sarafu-faucet==0.0.3a2
 potaahto~=0.0.1a1

apps/cic-eth/tests/fixtures_database.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import alembic
 from alembic.config import Config as AlembicConfig

apps/cic-meta/.config/pgp.ini

@@ -1,5 +1,5 @@
 [pgp]
-exports_dir = pgp
+exports_dir = /root/pgp
 privatekey_file = privatekeys.asc
 passphrase = merman
 publickey_trusted_file = publickeys.asc

apps/cic-meta/docker/Dockerfile

@@ -2,26 +2,31 @@ FROM node:15.3.0-alpine3.10
 
 WORKDIR /tmp/src/cic-meta
 
+RUN apk add --no-cache postgresql bash
+
 COPY cic-meta/package.json \
      ./
 
 COPY cic-meta/src/ src/
 COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
-#COPY docker/*.sh /root/
-
-RUN alias tsc=node_modules/typescript/bin/tsc 
 
 RUN npm install
 
+# see exports_dir gpg.ini
+COPY cic-meta/tests/*.asc /root/pgp/
+RUN alias tsc=node_modules/typescript/bin/tsc 
+
+
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
 # COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
-ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
+#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
 
-# COPY cic-meta/docker/start_server.sh ./start_server.sh
-# RUN chmod 755 ./start_server.sh
+COPY cic-meta/docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/db.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
+set -e 
+
+PGPASSWORD=$DATABASE_PASSWORD psql -v ON_ERROR_STOP=1 -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME -f $SCHEMA_SQL_PATH 
+
 
-PGPASSWORD=$DATABASE_PASSWORD psql -U $DATABASE_USER -h $DATABASE_HOST -p $DATABASE_PORT -d $DATABASE_NAME /usr/local/share/cic-meta/sql/server.sql

apps/cic-meta/docker/start_server.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+# db migration
 sh ./db.sh
 
-/usr/local/bin/node /usr/local/bin/cic-meta-server $@
+# /usr/local/bin/node /usr/local/bin/cic-meta-server $@
+# ./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts $@
+npm run start "$@"

apps/cic-meta/package.json

@@ -10,7 +10,8 @@
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
-		"prepare": "npm run build && npm run build-server"
+		"prepare": "npm run build && npm run build-server",
+                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
 	},
 	"dependencies": {
 		"@ethereumjs/tx": "^3.0.0-beta.1",

apps/cic-meta/scripts/initdb/postgresql.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username grassroots --dbname cic_meta <<-EOSQL
-    create table if not exists store (
-        id serial primary key not null,
-        owner_fingerprint text not null,
-        hash char(64) not null unique,
-        content text not null
-    );
-
-    create index if not exists idx_fp on store ((lower(owner_fingerprint)));
-EOSQL
-
-

apps/cic-meta/scripts/initdb/server.postgres.sql

@@ -1,4 +1,4 @@
-create table if not exists cic_meta.store (
+create table if not exists store (
 	id serial primary key not null,
 	owner_fingerprint text not null,
 	hash char(64) not null unique,

apps/cic-ussd/cic_ussd/account.py

@@ -20,7 +20,7 @@ def define_account_tx_metadata(user: Account):
     )
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     account_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/balance.py

@@ -80,7 +80,7 @@ def get_cached_operational_balance(blockchain_address: str):
     """
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     if cached_balance:

apps/cic-ussd/cic_ussd/error.py

@@ -38,3 +38,13 @@ class MetadataStoreError(Exception):
     pass
 
 
+class SeppukuError(Exception):
+    """Exception base class for all errors that should cause system shutdown"""
+    pass
+
+
+class InitializationError(Exception):
+    """Exception raised when initialization state is insufficient to run component"""
+    pass
+
+

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -118,9 +118,9 @@ class MetadataRequestsHandler(Metadata):
         metadata_http_error_handler(result=result)
         response_data = result.content
         data = json.loads(response_data.decode('utf-8'))
-        if result.status_code == 200 and self.cic_type == 'cic.person':
+        if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -9,4 +9,4 @@ from .base import MetadataRequestsHandler
 class PersonMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.person', identifier=identifier)
+        super().__init__(cic_type=':cic.person', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -10,4 +10,4 @@ from .base import MetadataRequestsHandler
 class PhonePointerMetadata(MetadataRequestsHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type='cic.msisdn', identifier=identifier)
+        super().__init__(cic_type=':cic.phone', identifier=identifier)

apps/cic-ussd/cic_ussd/operations.py

@@ -48,10 +48,9 @@ def define_response_with_content(headers: list, response: str) -> tuple:
     content_length_header = ('Content-Length', str(content_length))
     # check for content length defaulted to zero in error headers
     for position, header in enumerate(headers):
-        if header[0] == 'Content-Length':
-            headers[position] = content_length_header
-        else:
-            headers.append(content_length_header)
+        if 'Content-Length' in header:
+            headers.pop(position)
+    headers.append(content_length_header)
     return response_bytes, headers
 
 
@@ -326,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -7,6 +7,7 @@ from typing import Optional
 # third party imports
 import celery
 from sqlalchemy import desc
+from cic_eth.api import Api
 from tinydb.table import Document
 
 # local imports
@@ -15,7 +16,7 @@ from cic_ussd.balance import BalanceManager, compute_operational_balance, get_ca
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.db.models.ussd_session import UssdSession
-from cic_ussd.error import MetadataNotFoundError
+from cic_ussd.error import MetadataNotFoundError, SeppukuError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.phone_number import get_user_by_phone_number
@@ -28,6 +29,38 @@ from cic_types.models.person import generate_metadata_pointer, get_contact_data_
 logg = logging.getLogger(__name__)
 
 
+def get_default_token_data():
+    chain_str = Chain.spec.__str__()
+    cic_eth_api = Api(chain_str=chain_str)
+    default_token_request_task = cic_eth_api.default_token()
+    default_token_data = default_token_request_task.get()
+    return default_token_data
+
+
+def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()):
+    """
+    :param chain_str:
+    :type chain_str:
+    :return:
+    :rtype:
+    """
+    cache_key = create_cached_data_key(
+        identifier=chain_str.encode('utf-8'),
+        salt=':cic.default_token_data'
+    )
+    cached_data = get_cached_data(key=cache_key)
+    if cached_data:
+        default_token_data = json.loads(cached_data)
+        return default_token_data.get('symbol')
+    else:
+        logg.warning('Cached default token data not found. Attempting retrieval from default token API')
+        default_token_data = get_default_token_data()
+        if default_token_data:
+            return default_token_data.get('symbol')
+        else:
+            raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
+
+
 def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
     """
     This method provides translation for all ussd menu entries that follow the pin authorization pattern.
@@ -73,7 +106,9 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     # compile response data
     user_input = ussd_session.get('user_input').split('*')[-1]
     transaction_amount = to_wei(value=int(user_input))
-    token_symbol = 'SRF'
+
+    # get default data
+    token_symbol = retrieve_token_symbol()
 
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
@@ -102,7 +137,7 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     :rtype: str
     """
     transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount')))
-    token_symbol = 'SRF'
+    token_symbol = retrieve_token_symbol()
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
     recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
@@ -137,7 +172,7 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     tx_recipient_information = define_account_tx_metadata(user=recipient)
     tx_sender_information = define_account_tx_metadata(user=user)
 
-    token_symbol = 'SRF'
+    token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
@@ -168,18 +203,18 @@ def process_account_balances(user: Account, display_key: str, ussd_session: dict
     logg.debug('Requires call to retrieve tax and bonus amounts')
     tax = ''
     bonus = ''
-
+    token_symbol = retrieve_token_symbol()
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
         operational_balance=operational_balance,
         tax=tax,
         bonus=bonus,
-        token_symbol='SRF'
+        token_symbol=token_symbol
     )
 
 
-def format_transactions(transactions: list, preferred_language: str):
+def format_transactions(transactions: list, preferred_language: str, token_symbol: str):
     
     formatted_transactions = ''
     if len(transactions) > 0:
@@ -190,7 +225,7 @@ def format_transactions(transactions: list, preferred_language: str):
             timestamp = transaction.get('timestamp')
             action_tag = transaction.get('action_tag')
             direction = transaction.get('direction')
-            token_symbol = 'SRF'
+            token_symbol = token_symbol
 
             if action_tag == 'SENT' or action_tag == 'ULITUMA':
                 formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {recipient_phone_number} {timestamp}.\n'
@@ -214,7 +249,7 @@ def process_display_user_metadata(user: Account, display_key: str):
     """
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key)
     if user_metadata:
@@ -251,9 +286,11 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
     """
     # retrieve cached statement
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address)
-    key = create_cached_data_key(identifier=identifier, salt='cic.statement')
+    key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
     transactions = get_cached_data(key=key)
 
+    token_symbol = retrieve_token_symbol()
+
     first_transaction_set = []
     middle_transaction_set = []
     last_transaction_set = []
@@ -277,7 +314,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             first_transaction_set=format_transactions(
                 transactions=first_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
     elif display_key == 'ussd.kenya.middle_transaction_set':
@@ -286,7 +324,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             middle_transaction_set=format_transactions(
                 transactions=middle_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
 
@@ -296,7 +335,8 @@ def process_account_statement(user: Account, display_key: str, ussd_session: dic
             preferred_language=user.preferred_language,
             last_transaction_set=format_transactions(
                 transactions=last_transaction_set,
-                preferred_language=user.preferred_language
+                preferred_language=user.preferred_language,
+                token_symbol=token_symbol
             )
         )
 
@@ -312,37 +352,28 @@ def process_start_menu(display_key: str, user: Account):
     :return: Corresponding translation text response
     :rtype: str
     """
+    token_symbol = retrieve_token_symbol()
     chain_str = Chain.spec.__str__()
     blockchain_address = user.blockchain_address
     balance_manager = BalanceManager(address=blockchain_address,
                                      chain_str=chain_str,
-                                     token_symbol='SRF')
+                                     token_symbol=token_symbol)
 
     # get balances synchronously for display on start menu
     balances_data = balance_manager.get_balances()
 
     key = create_cached_data_key(
         identifier=bytes.fromhex(blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cache_data(key=key, data=json.dumps(balances_data))
 
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 
-    # TODO [Philip]: figure out how to get token symbol from a metadata layer of sorts.
-    token_symbol = 'SRF'
-
     return translation_for(
         key=display_key,
         preferred_language=user.preferred_language,
@@ -375,6 +406,13 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
+    # retrieve metadata before any transition
+    key = generate_metadata_pointer(
+        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+        cic_type=':cic.person'
+    )
+    person_metadata = get_cached_data(key=key)
+
     if ussd_session:
         if user_input == "0":
             return UssdMenu.parent_menu(menu_name=ussd_session.get('state'))
@@ -385,12 +423,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
         if user.has_valid_pin():
             last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
 
-            key = generate_metadata_pointer(
-                identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-                cic_type='cic.person'
-            )
-            person_metadata = get_cached_data(key=key)
-
             if last_ussd_session:
                 # get last state
                 last_state = last_ussd_session.state

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -0,0 +1,73 @@
+"""
+This module handles requests originating from CICADA or any other management client for custodial wallets, processing
+requests offering control of user account states to a staff behind the client.
+"""
+
+# standard imports
+import logging
+from urllib.parse import quote_plus
+
+# third-party imports
+from confini import Config
+
+# local imports
+from cic_ussd.db import dsn_from_config
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.operations import define_response_with_content
+from cic_ussd.requests import (get_request_endpoint,
+                               get_query_parameters,
+                               process_pin_reset_requests,
+                               process_locked_accounts_requests)
+from cic_ussd.runnable.server_base import exportable_parser, logg
+args = exportable_parser.parse_args()
+
+# define log levels
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+# parse config
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+
+# set up db
+data_source_name = dsn_from_config(config)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
+# create session for the life time of http request
+SessionBase.session = SessionBase.create_session()
+
+
+# handle requests from CICADA
+def application(env, start_response):
+    """Loads python code for application to be accessible over web server
+    :param env: Object containing server and request information
+    :type env: dict
+    :param start_response: Callable to define responses.
+    :type start_response: any
+    :return: a list containing a bytes representation of the response object
+    :rtype: list
+    """
+
+    # define headers
+    errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
+    headers = [('Content-Type', 'text/plain')]
+
+    if get_request_endpoint(env) == '/pin':
+        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
+        phone_number = quote_plus(phone_number)
+        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
+        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
+        SessionBase.session.close()
+        start_response(message, headers)
+        return [response_bytes]
+
+    # handle requests for locked accounts
+    response, message = process_locked_accounts_requests(env=env)
+    response_bytes, headers = define_response_with_content(headers=headers, response=response)
+    start_response(message, headers)
+    SessionBase.session.close()
+    return [response_bytes]
+

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -1,25 +1,23 @@
-"""Functions defining WSGI interaction with external http requests
-Defines an application function essential for the uWSGI python loader to run th python application code.
+"""This module handles requests originating from the ussd service provider.
 """
+
 # standard imports
-import argparse
-import celery
-import i18n
 import json
 import logging
-import os
-import redis
 
 # third-party imports
-from confini import Config
+import celery
+import i18n
+import redis
 from chainlib.chain import ChainSpec
-from urllib.parse import quote_plus
+from confini import Config
 
 # local imports
 from cic_ussd.chain import Chain
 from cic_ussd.db import dsn_from_config
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.encoder import PasswordEncoder
+from cic_ussd.error import InitializationError
 from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata.signer import Signer
@@ -28,34 +26,17 @@ from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
 from cic_ussd.phone_number import process_phone_number
-from cic_ussd.redis import InMemoryStore
+from cic_ussd.processor import get_default_token_data
+from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
-                               get_request_method,
-                               get_query_parameters,
-                               process_locked_accounts_requests,
-                               process_pin_reset_requests)
+                               get_request_method)
+from cic_ussd.runnable.server_base import exportable_parser, logg
 from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.validator import check_ip, check_request_content_length, check_service_code, validate_phone_number, \
     validate_presence
 
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_directory = '/usr/local/etc/cic-ussd/'
-
-# define arguments
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument('-c', type=str, default=config_directory, help='config directory.')
-arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
-arg_parser.add_argument('-v', action='store_true', help='be verbose')
-arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
-arg_parser.add_argument('--env-prefix',
-                        default=os.environ.get('CONFINI_ENV_PREFIX'),
-                        dest='env_prefix',
-                        type=str,
-                        help='environment prefix for variables to overwrite configuration')
-args = arg_parser.parse_args()
+args = exportable_parser.parse_args()
 
 # define log levels
 if args.vv:
@@ -69,7 +50,14 @@ config.process()
 config.censor('PASSWORD', 'DATABASE')
 logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-# initialize elements
+# set up db
+data_source_name = dsn_from_config(config)
+SessionBase.connect(data_source_name,
+                    pool_size=int(config.get('DATABASE_POOL_SIZE')),
+                    debug=config.true('DATABASE_DEBUG'))
+# create session for the life time of http request
+SessionBase.session = SessionBase.create_session()
+
 # set up translations
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
 i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
@@ -82,12 +70,6 @@ ussd_menu_db = create_local_file_data_stores(file_location=config.get('USSD_MENU
                                              table_name='ussd_menu')
 UssdMenu.ussd_menu_db = ussd_menu_db
 
-# set up db
-data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
-
 # define universal redis cache access
 InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'),
                                         port=config.get('REDIS_PORT'),
@@ -127,6 +109,20 @@ Chain.spec = chain_spec
 UssdStateMachine.states = states
 UssdStateMachine.transitions = transitions
 
+# retrieve default token data
+default_token_data = get_default_token_data()
+chain_str = Chain.spec.__str__()
+
+# cache default token for re-usability
+if default_token_data:
+    cache_key = create_cached_data_key(
+        identifier=chain_str.encode('utf-8'),
+        salt=':cic.default_token_data'
+    )
+    cache_data(key=cache_key, data=json.dumps(default_token_data))
+else:
+    raise InitializationError(f'Default token data for: {chain_str} not found.')
+
 
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
@@ -134,6 +130,8 @@ def application(env, start_response):
     :type env: dict
     :param start_response: Callable to define responses.
     :type start_response: any
+    :return: a list containing a bytes representation of the response object
+    :rtype: list
     """
     # define headers
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
@@ -194,20 +192,3 @@ def application(env, start_response):
         start_response('200 OK,', headers)
         SessionBase.session.close()
         return [response_bytes]
-
-    # handle pin requests
-    if get_request_endpoint(env) == '/pin':
-        phone_number = get_query_parameters(env=env, query_name='phoneNumber')
-        phone_number = quote_plus(phone_number)
-        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
-        response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-        SessionBase.session.close()
-        start_response(message, headers)
-        return [response_bytes]
-
-    # handle requests for locked accounts
-    response, message = process_locked_accounts_requests(env=env)
-    response_bytes, headers = define_response_with_content(headers=headers, response=response)
-    start_response(message, headers)
-    SessionBase.session.close()
-    return [response_bytes]

apps/cic-ussd/cic_ussd/runnable/server_base.py

@@ -0,0 +1,38 @@
+"""This module handles generic wsgi server configurations that can then be subsumed by different server flavors for the
+cic-ussd component.
+"""
+
+# standard imports
+import logging
+import os
+from argparse import ArgumentParser
+
+# third-party imports
+
+# local imports
+
+# define a logging system
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+# define default config directory as would be defined in docker
+default_config_dir = '/usr/local/etc/cic-ussd/'
+
+# define args parser
+arg_parser = ArgumentParser(description='CLI for handling cic-ussd server applications.')
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+exportable_parser = arg_parser
+
+
+
+
+
+

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -64,7 +64,7 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo
     # get cached balance
     key = create_cached_data_key(
         identifier=bytes.fromhex(user.blockchain_address[2:]),
-        salt='cic.balances_data'
+        salt=':cic.balances_data'
     )
     cached_balance = get_cached_data(key=key)
     operational_balance = compute_operational_balance(balances=json.loads(cached_balance))

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -176,7 +176,7 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
     blockchain_address = user.blockchain_address
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key=key)
 

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -23,7 +23,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     # check for user metadata in cache
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     user_metadata = get_cached_data(key=key)
     return user_metadata is not None

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -136,7 +136,7 @@ def process_balances_callback(result: list, param: str, status_code: int):
         blockchain_address = balances_data.get('address')
         key = create_cached_data_key(
             identifier=bytes.fromhex(blockchain_address[2:]),
-            salt='cic.balances_data'
+            salt=':cic.balances_data'
         )
         cache_data(key=key, data=json.dumps(balances_data))
     else:
@@ -226,7 +226,7 @@ def process_statement_callback(result, param: str, status_code: int):
 
         # cache account statement
         identifier = bytes.fromhex(param[2:])
-        key = create_cached_data_key(identifier=identifier, salt='cic.statement')
+        key = create_cached_data_key(identifier=identifier, salt=':cic.statement')
         data = json.dumps(processed_transactions)
 
         # cache statement data

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.9')
+version = (0, 3, 0, 'alpha.10')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile

@@ -38,8 +38,9 @@ COPY cic-ussd/transitions/ cic-ussd/transitions/
 COPY cic-ussd/var/ cic-ussd/var/
 
 COPY cic-ussd/docker/db.sh \
-     cic-ussd/docker/start_tasker.sh \
-     cic-ussd/docker/start_uwsgi.sh \
+     cic-ussd/docker/start_cic_user_tasker.sh \
+     cic-ussd/docker/start_cic_user_ussd_server.sh\
+     cic-ussd/docker/start_cic_user_server.sh\
      /root/
 
 RUN chmod +x /root/*.sh

apps/cic-ussd/docker/start_cic_user_server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /root/db.sh
+
+user_server_port=${SERVER_PORT:-9500}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_server.py --http :"$user_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_cic_user_tasker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /root/db.sh
+
+/usr/local/bin/cic-user-tasker "$@"

apps/cic-ussd/docker/start_cic_user_ussd_server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /root/db.sh
+
+user_ussd_server_port=${SERVER_PORT:-9000}
+
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/daemons/cic_user_ussd_server.py --http :"$user_ussd_server_port" --pyargv "$@"

apps/cic-ussd/docker/start_tasker.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-/usr/local/bin/cic-ussd-tasker $@

apps/cic-ussd/docker/start_uwsgi.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-. /root/db.sh
-
-server_port=${SERVER_PORT:-9000}
-
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :$server_port --pyargv "$@"

apps/cic-ussd/setup.cfg

@@ -35,6 +35,7 @@ packages =
 	cic_ussd.menu
 	cic_ussd.metadata
 	cic_ussd.runnable
+	cic_ussd.runnable.daemons
 	cic_ussd.session
 	cic_ussd.state_machine
 	cic_ussd.state_machine.logic
@@ -44,5 +45,5 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-ussd-tasker = cic_ussd.runnable.tasker:main
+	cic-user-tasker = cic_ussd.runnable.daemons.cic_user_tasker:main
 	cic-ussd-client = cic_ussd.runnable.client:main

apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py

@@ -105,7 +105,7 @@ def test_get_user_metadata(caplog,
         assert 'Get latest data status: 200' in caplog.text
     key = generate_metadata_pointer(
         identifier=identifier,
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cached_user_metadata = get_cached_data(key=key)
     assert cached_user_metadata

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py

@@ -36,7 +36,7 @@ def test_has_cached_user_metadata(create_in_db_ussd_session,
     user = create_activated_user
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cache_data(key=key, data=json.dumps(person_metadata))
     result = has_cached_user_metadata(state_machine_data=state_machine_data)

apps/cic-ussd/tests/fixtures/user.py

@@ -115,6 +115,6 @@ def cached_user_metadata(create_activated_user, init_redis_cache, person_metadat
     user_metadata = json.dumps(person_metadata)
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address),
-        cic_type='cic.person'
+        cic_type=':cic.person'
     )
     cache_data(key=key, data=user_metadata)

apps/contract-migration/docker/Dockerfile

@@ -31,15 +31,6 @@ RUN echo Install confini schema files && \
 	git checkout $cic_config_commit && \	
 	cp -v *.ini $CONFINI_DIR
 
-ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501
-ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/
-RUN echo Install ABI collection for solidity interfaces used across all components && \
-	git clone --depth 1 $cic_contracts_url cic-contracts && \
-	cd cic-contracts && \
-	git fetch --depth 1 origin $cic_contracts_commit && \
-	git checkout $cic_contracts_commit && \
-	make install
-
 # Install nvm with node and npm
 # https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
 ENV NVM_DIR /root/.nvm
@@ -56,54 +47,61 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
 ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
 ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-USER grassroots
+#RUN useradd --create-home grassroots
+# WORKDIR /home/grassroots
+# USER grassroots
 
+ARG pip_extra_args=""
+ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a79
-ARG cic_eth_version=0.11.0b8+build.c2286e5c
-ARG sarafu_faucet_version=0.0.2a28
-ARG sarafu_token_version==0.0.1a6
-ARG cic_contracts_version=0.0.2a2
-RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
+ARG cic_base_version=0.1.2b8
+ARG cic_eth_version=0.11.0b12
+ARG sarafu_token_version=0.0.1a8
+ARG sarafu_faucet_version=0.0.3a2
+RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
+	cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
-	cic-contracts==$cic_contracts_version \
 	sarafu-faucet==$sarafu_faucet_version \
-	sarafu-token==$sarafu_token_version
+	sarafu-token==$sarafu_token_version \
+	cic-eth==$cic_eth_version 
 
+# -------------- begin runtime container ---------------- 
 FROM python:3.8.6-slim-buster as runtime-image
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y --no-install-recommends jq
+RUN apt-get install -y jq bash iputils-ping socat
 
 COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
 COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
+COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
 
-RUN useradd --create-home grassroots
-WORKDIR /home/grassroots
-# COPY python dependencies to user dir
-COPY --from=compile-image /home/grassroots/.local .local
-ENV PATH=/home/grassroots/.local/bin:$PATH
+ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
+# RUN useradd -u 1001 --create-home grassroots
+# RUN adduser grassroots sudo && \
+#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# WORKDIR /home/grassroots
 
 COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
+COPY contract-migration/scripts scripts/ 
 
-# RUN chown grassroots:grassroots .local/
-
-RUN mkdir -p /tmp/cic/config
-RUN chown grassroots:grassroots /tmp/cic/config
 # A shared output dir for environment configs
+RUN mkdir -p /tmp/cic/config
+# RUN chown grassroots:grassroots /tmp/cic/config
 RUN chmod a+rwx /tmp/cic/config 
 
 COPY contract-migration/*.sh ./
-RUN chown grassroots:grassroots -R .
+# RUN chown grassroots:grassroots -R .
 RUN chmod gu+x *.sh
 
+# we copied these from the root build container.
+# this is dumb though...I guess the compile image should have the same user
+# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
 
-USER grassroots
+# USER grassroots
 
 ENTRYPOINT [ ]

apps/contract-migration/scripts/README.md

@@ -104,8 +104,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-ussd-tasker
-* cic-ussd-server
+* cic-user-tasker
+* cic-user-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,6 +169,24 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
+Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
+
+In one terminal run:
+
+`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
+
+This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the current sempo system.
+This csv file will be stored in the pins export dir defined as the positional argument.
+
+Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
+
+- To import the pins:
+
+`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
+
+- To import ussd data:
+`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
+
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/contract-migration/scripts/cic_ussd/import_pins.py

@@ -0,0 +1,70 @@
+# standard import
+import argparse
+import csv
+import logging
+import os
+
+# third-party imports
+import celery
+import confini
+
+# local imports
+from import_task import *
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = './config'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+arg_parser.add_argument('--env-prefix',
+                        default=os.environ.get('CONFINI_ENV_PREFIX'),
+                        dest='env_prefix',
+                        type=str,
+                        help='environment prefix for variables to overwrite configuration')
+arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit transaction tasks to')
+arg_parser.add_argument('-v', help='be verbose', action='store_true')
+arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
+arg_parser.add_argument('pins_dir', default='out', type=str, help='user export directory')
+args = arg_parser.parse_args()
+
+# set log levels
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+# process configs
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+
+
+def main():
+    with open(f'{args.pins_dir}/pins.csv') as pins_file:
+        phone_to_pins = [tuple(row) for row in csv.reader(pins_file)]
+
+    s_import_pins = celery.signature(
+        'import_task.set_pins',
+        (db_configs, phone_to_pins),
+        queue=args.q
+    )
+    s_import_pins.apply_async()
+
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/cic_ussd/import_task.py

@@ -8,6 +8,8 @@ import json
 
 # external imports
 import celery
+import psycopg2
+from psycopg2 import extras
 from hexathon import (
         strip_0x,
         add_0x,
@@ -53,7 +55,7 @@ class MetadataTask(ImportTask):
     def meta_url(self):
         scheme = 'http'
         if self.meta_ssl:
-            scheme += s
+            scheme += 's'
         url = urllib.parse.urlparse('{}://{}:{}/{}'.format(scheme, self.meta_host, self.meta_port, self.meta_path))
         return urllib.parse.urlunparse(url)
 
@@ -91,7 +93,6 @@ def resolve_phone(self, phone):
 def generate_metadata(self, address, phone):
     old_address = old_address_from_phone(self.import_dir, phone)
 
-    logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
     logg.debug('address {}'.format(address))
     old_address_upper = strip_0x(old_address).upper()
     metadata_path = '{}/old/{}/{}/{}.json'.format(
@@ -216,3 +217,60 @@ def send_txs(self, nonce):
 
 
     return nonce
+
+
+@celery_app.task
+def set_pins(config: dict, phone_to_pins: list):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # update db
+    for element in phone_to_pins:
+        sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
+        db_cursor.execute(sql, (element[1], element[0]))
+        logg.debug(f'Updating: {element[0]} with: {element[1]}')
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+
+
+@celery_app.task
+def set_ussd_data(config: dict, ussd_data: dict):
+    # define db connection
+    db_conn = psycopg2.connect(
+        database=config.get('database'),
+        host=config.get('host'),
+        port=config.get('port'),
+        user=config.get('user'),
+        password=config.get('password')
+    )
+    db_cursor = db_conn.cursor()
+
+    # process ussd_data
+    account_status = 1
+    if ussd_data['is_activated'] == 1:
+        account_status = 2
+    preferred_language = ussd_data['preferred_language']
+    phone_number = ussd_data['phone']
+
+    sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s'
+    db_cursor.execute(sql, (account_status, preferred_language, phone_number))
+
+    # commit changes
+    db_conn.commit()
+
+    # close connections
+    db_cursor.close()
+    db_conn.close()
+

apps/contract-migration/scripts/cic_ussd/import_users.py

@@ -87,6 +87,13 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
   
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
@@ -135,57 +142,60 @@ if __name__ == '__main__':
         for y in x[2]:
             if y[len(y)-5:] != '.json':
                 continue
-            filepath = os.path.join(x[0], y)
-            f = open(filepath, 'r')
-            try:
-                o = json.load(f)
-            except json.decoder.JSONDecodeError as e:
+            # handle json containing person object
+            filepath = None
+            if y != 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
                 f.close()
-                logg.error('load error for {}: {}'.format(y, e))
-                continue
-            f.close()
-            u = Person.deserialize(o)
+                u = Person.deserialize(o)
 
-            new_address = register_ussd(i, u)
+                new_address = register_ussd(i, u)
 
-            phone_object = phonenumbers.parse(u.tel)
-            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-            s_phone = celery.signature(
-                    'import_task.resolve_phone',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_phone = celery.signature(
+                        'import_task.resolve_phone',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta = celery.signature(
-                    'import_task.generate_metadata',
-                    [
-                        phone,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_meta = celery.signature(
+                        'import_task.generate_metadata',
+                        [
+                            phone,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_balance = celery.signature(
-                    'import_task.opening_balance_tx',
-                    [
-                        phone,
-                        i,
-                        ],
-                    queue='cic-import-ussd',
-                    )
+                s_balance = celery.signature(
+                        'import_task.opening_balance_tx',
+                        [
+                            phone,
+                            i,
+                            ],
+                        queue='cic-import-ussd',
+                        )
 
-            s_meta.link(s_balance)
-            s_phone.link(s_meta)
-            s_phone.apply_async(countdown=7) # block time plus a bit of time for ussd processing
+                s_meta.link(s_balance)
+                s_phone.link(s_meta)
+                # block time plus a bit of time for ussd processing
+                s_phone.apply_async(countdown=7)
 
-            i += 1
-            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
-        
-            j += 1
-            if j == batch_size:
-                time.sleep(batch_delay)
-                j = 0
+                i += 1
+                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+
+                j += 1
+                if j == batch_size:
+                    time.sleep(batch_delay)
+                    j = 0
 
-    #fi.close()

apps/contract-migration/scripts/cic_ussd/import_ussd_data.py

@@ -0,0 +1,70 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# external imports
+import celery
+from confini import Config
+
+# local imports
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+if __name__ == '__main__':
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # handle ussd_data json object
+            if y == 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    ussd_data = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+
+                s_set_ussd_data = celery.signature(
+                    'import_task.set_ussd_data',
+                    [db_configs, ussd_data]
+                )
+                s_set_ussd_data.apply_async(queue='cic-import-ussd')

apps/contract-migration/scripts/create_import_pins.py

@@ -0,0 +1,90 @@
+# standard imports
+import argparse
+import json
+import logging
+import os
+
+# third-party imports
+import bcrypt
+import celery
+import confini
+import phonenumbers
+import random
+from cic_types.models.person import Person
+from cryptography.fernet import Fernet
+
+# local imports
+
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+default_config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
+
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument('-c', type=str, default=default_config_dir, help='Config dir')
+arg_parser.add_argument('-v', action='store_true', help='Be verbose')
+arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
+arg_parser.add_argument('--userdir', type=str, help='path to users export dir tree')
+arg_parser.add_argument('pins_dir', type=str, help='path to pin export dir tree')
+
+
+args = arg_parser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+logg.info('loaded config\n{}'.format(config))
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+user_dir = args.userdir
+pins_dir = args.pins_dir
+
+
+def generate_password_hash():
+    key = Fernet.generate_key()
+    fnt = Fernet(key)
+    pin = str(random.randint(1000, 9999))
+    return fnt.encrypt(bcrypt.hashpw(pin.encode('utf-8'), bcrypt.gensalt())).decode()
+
+
+user_old_dir = os.path.join(user_dir, 'old')
+logg.debug(f'reading user data from: {user_old_dir}')
+
+pins_file = open(f'{pins_dir}/pins.csv', 'w')
+
+if __name__ == '__main__':
+
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            # skip non-json files
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # define file path for
+            filepath = None
+            if y != 'ussd_data.json':
+                filepath = os.path.join(x[0], y)
+                f = open(filepath, 'r')
+                try:
+                    o = json.load(f)
+                except json.decoder.JSONDecodeError as e:
+                    f.close()
+                    logg.error('load error for {}: {}'.format(y, e))
+                    continue
+                f.close()
+                u = Person.deserialize(o)
+
+                phone_object = phonenumbers.parse(u.tel)
+                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+                password_hash = generate_password_hash()
+                pins_file.write(f'{phone},{password_hash}\n')
+                logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
+
+    pins_file.close()

apps/contract-migration/scripts/create_import_users.py

@@ -130,6 +130,7 @@ def genCats():
 def genAmount():
     return random.randint(0, gift_max) * gift_factor
 
+
 def genDob():
     dob_src = fake.date_of_birth(minimum_age=15)
     dob = {}
@@ -168,8 +169,9 @@ def gen():
             }
     p.location['area_name'] = city
     if random.randint(0, 1):
-        p.identities['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
-        p.identities['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+        p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
+        p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+
     
     return (old_blockchain_checksum_address, phone, p)
 
@@ -215,6 +217,20 @@ if __name__ == '__main__':
         json.dump(o.serialize(), f)
         f.close()
 
+        # create ussd data
+        ussd_data_dir = os.path.join(d, 'ussd_data')
+        os.makedirs(ussd_data_dir)
+        f = open('{}/{}/{}'.format(d, 'ussd_data', 'ussd_data.json'), 'w')
+        ussd_data = {
+            'phone': phone,
+            'is_activated': 1,
+            'preferred_language': random.sample(['en', 'sw'], 1)[0],
+            'is_disabled': False
+        }
+        json.dump(ussd_data, f)
+        f.close()
+
+
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b2
-sarafu-faucet==0.0.2a28
-cic-eth==0.11.0b10
+cic-base[full_graph]==0.1.2b8
+sarafu-faucet==0.0.3a2
+cic-eth==0.11.0b12
 cic-types==0.1.0a10
 crypto-dev-signer==0.4.14b3

apps/contract-migration/scripts/verify.py

@@ -1,52 +1,43 @@
 # standard imports
+import argparse
+import copy
+import csv
+import hashlib
+import json
+import logging
 import os
 import sys
-import logging
-import time
-import argparse
-import sys
-import re
-import hashlib
-import csv
-import json
 import urllib
-import copy
-import uuid
 import urllib.request
+import uuid
+from collections import Counter
 
 # external imports
 import celery
-import eth_abi
 import confini
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+import eth_abi
+import psycopg2
 from chainlib.chain import ChainSpec
+from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainlib.hash import keccak256_string_to_hex
-from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
-        OverrideGasOracle,
-        balance,
-        )
+    OverrideGasOracle,
+    balance,
+)
 from chainlib.eth.tx import TxFactory
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.jsonrpc import jsonrpc_template
-from chainlib.eth.error import EthException
 from cic_types.models.person import (
-        Person,
-        generate_metadata_pointer,
-        )
+    Person,
+    generate_metadata_pointer,
+)
 from erc20_single_shot_faucet import SingleShotFaucet
+from hexathon import (
+    strip_0x,
+    add_0x,
+)
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -72,6 +63,7 @@ eth_tests = [
 
 phone_tests = [
         'ussd',
+        'ussd_pins'
         ]
 
 all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
@@ -171,6 +163,39 @@ if logg.isEnabledFor(logging.DEBUG):
     outfunc = logg.debug
 
 
+def send_ussd_request(address, data_dir):
+    upper_address = strip_0x(address).upper()
+    f = open(os.path.join(
+        data_dir,
+        'new',
+        upper_address[:2],
+        upper_address[2:4],
+        upper_address + '.json',
+    ), 'r'
+    )
+    o = json.load(f)
+    f.close()
+
+    p = Person.deserialize(o)
+    phone = p.tel
+
+    session = uuid.uuid4().hex
+    data = {
+        'sessionId': session,
+        'serviceCode': config.get('APP_SERVICE_CODE'),
+        'phoneNumber': phone,
+        'text': '',
+    }
+
+    req = urllib.request.Request(config.get('_USSD_PROVIDER'))
+    data_str = json.dumps(data)
+    data_bytes = data_str.encode('utf-8')
+    req.add_header('Content-Type', 'application/json')
+    req.data = data_bytes
+    response = urllib.request.urlopen(req)
+    return response.read().decode('utf-8')
+
+
 class VerifierState:
 
     def __init__(self, item_keys, active_tests=None):
@@ -354,42 +379,18 @@ class Verifier:
 
 
     def verify_ussd(self, address, balance=None):
-        upper_address = strip_0x(address).upper()
-        f = open(os.path.join(
-            self.data_dir,
-            'new',
-            upper_address[:2],
-            upper_address[2:4],
-            upper_address + '.json',
-            ), 'r'
-            )
-        o = json.load(f)
-        f.close()
-
-        p = Person.deserialize(o)
-        phone = p.tel
-
-        session = uuid.uuid4().hex
-        data = {
-                'sessionId': session,
-                'serviceCode': config.get('APP_SERVICE_CODE'),
-                'phoneNumber': phone,
-                'text': config.get('APP_SERVICE_CODE'),
-            }
-
-        req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-        data_str = json.dumps(data)
-        data_bytes = data_str.encode('utf-8')
-        req.add_header('Content-Type', 'application/json')
-        req.data = data_bytes
-        response = urllib.request.urlopen(req)
-        response_data = response.read().decode('utf-8')
+        response_data = send_ussd_request(address, self.data_dir)
         state = response_data[:3]
         out = response_data[4:]
         m = '{} {}'.format(state, out[:7])
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
+    def verify_ussd_pins(self, address, balance):
+        response_data = send_ussd_request(address, self.data_dir)
+        if response_data[:11] != 'CON Balance':
+            raise VerifierError(response_data, 'pins')
+
 
     def verify(self, address, balance, debug_stem=None):
   

apps/contract-migration/seed_cic_eth.sh

@@ -99,7 +99,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer sarafu tokens to token gifter address"
->&2 eth-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 

ci_templates/.cic-template.yml

@@ -3,7 +3,7 @@ image:
   entrypoint: [""]
 
 variables:
-  KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h"
+  KANIKO_CACHE_ARGS: "--cache=false --cache-copy-layers=true --cache-ttl=24h"
   CONTEXT: $CI_PROJECT_DIR/apps/
 
 .py_build_merge_request:

docker-compose.yml

@@ -53,8 +53,6 @@ services:
     command: [ "-c", "max_connections=200" ]
     volumes:
       - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql 
-      - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
-      - ./apps/cic-cache/db/psycopg2/db.sql:/docker-entrypoint-initdb.d/3-init-cic-meta.sql
       - postgres-db:/var/lib/postgresql/data
 
   redis:
@@ -78,6 +76,9 @@ services:
 
   contract-migration:
     build:
+      args:
+        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
+        pip_extra_args: $PIP_EXTRA_ARGS
       context: apps/
       dockerfile: contract-migration/docker/Dockerfile
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
@@ -477,6 +478,7 @@ services:
       PGP_PUBLICKEY_TRUSTED_FILE: publickeys.asc
       PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc
       PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc
+      SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql
     ports:
       - ${HTTP_PORT_CIC_META:-63380}:8000
     depends_on:
@@ -488,8 +490,7 @@ services:
       - ${LOCAL_VOLUME_DIR:-/tmp/cic}/pgp:/tmp/cic/pgp
     # command: "/root/start_server.sh -vv"
 
-  cic-ussd-server:
-    # image: grassrootseconomics:cic-ussd
+  cic-user-ussd-server:
     build:
       context: apps/
       dockerfile: cic-ussd/docker/Dockerfile
@@ -507,7 +508,7 @@ services:
       SERVER_PORT: 9000
       CIC_META_URL: ${CIC_META_URL:-http://meta:8000}
     ports:
-      - ${HTTP_PORT_CIC_USSD:-63315}:9000
+      - ${HTTP_PORT_CIC_USER_USSD_SERVER:-63315}:9000
     depends_on:
       - postgres
       - redis
@@ -516,10 +517,31 @@ services:
     deploy:
       restart_policy:
         condition: on-failure
-    command: "/root/start_uwsgi.sh -vv"
+    command: "/root/start_cic_user_ussd_server.sh -vv"
 
-  cic-ussd-tasker:
-    # image: grassrootseconomics:cic-ussd
+  cic-user-server:
+    build:
+      context: apps
+      dockerfile: cic-ussd/docker/Dockerfile
+    environment:
+      DATABASE_USER: grassroots
+      DATABASE_HOST: postgres
+      DATABASE_PORT: 5432
+      DATABASE_PASSWORD: tralala
+      DATABASE_NAME: cic_ussd
+      DATABASE_ENGINE: postgresql
+      DATABASE_DRIVER: psycopg2
+      DATABASE_POOL_SIZE: 0
+    ports:
+      - ${HTTP_PORT_CIC_USER_SERVER:-63415}:9500
+    depends_on:
+      - postgres
+    deploy:
+      restart_policy:
+        condition: on-failure
+    command: "/root/start_cic_user_server.sh -vv"
+
+  cic-user-tasker:
     build:
       context: apps
       dockerfile: cic-ussd/docker/Dockerfile
@@ -544,4 +566,4 @@ services:
     deploy:
       restart_policy:
         condition: on-failure
-    command: "/root/start_tasker.sh -q cic-ussd -vv"
+    command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv"