Merge remote-tracking branch 'origin/master' into lash/bloxberg-seeding

fix: update default pip to new url

Closes #171

See merge request grassrootseconomics/cic-internal-integration!316

.gitlab-ci.yml

@@ -30,6 +30,8 @@ version:
   #image: python:3.7-stretch
   image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
   stage: version
+  tags:
+    - integration
   script:
     - mkdir -p ~/.ssh && chmod 700 ~/.ssh
     - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts

.gitlab/issue_templates/bug.md

@@ -0,0 +1,44 @@
+<!---
+Please read this!
+
+Before opening a new issue, make sure to search for keywords in the issues
+filtered by the "bug" label:
+
+- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
+
+and verify the issue you're about to submit isn't a duplicate.
+--->
+
+### Summary
+
+<!-- Summarize the bug encountered concisely. -->
+
+### Steps to reproduce
+
+<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
+
+### Example Project
+
+<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic 
+behavior, and link to it here in the bug report. If you are using an older version of GitLab, this 
+will also determine whether the bug is fixed in a more recent version. -->
+
+### What is the current *bug* behavior?
+
+<!-- Describe what actually happens. -->
+
+### What is the expected *correct* behavior?
+
+<!-- Describe what you should see instead. -->
+
+### Relevant logs and/or screenshots
+
+<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
+ as it's tough to read otherwise. -->
+
+
+### Possible fixes
+
+<!-- If you can, link to the line of code that might be responsible for the problem. -->
+
+/label ~"bug"

CONTRIBUTING.md

@@ -0,0 +1,16 @@
+Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider. 
+
+__To request a change to the code please fork this repository and sumbit a merge request.__
+
+__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
+
+__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
+
+__Ask a question in our dev chat:__
+
+[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
+
+[Discord](https://discord.gg/XWunwAsX)
+
+[Matrix, IRC soon?]
+

README.md

@@ -1,41 +1,19 @@
-# cic-internal-integration
+# Community Inclusion Currency Stack (CIC Stack) 
+
+A custodial evm wallet for executing transactions via USSD
 
 ## Getting started 
 
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
-
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1
 ```
 
-start services, database, redis and local ethereum node
-```
-docker-compose up -d
-```
+To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
 
-Run app/contract-migration to deploy contracts
-```
-RUN_MASK=3 docker-compose up contract-migration
-```
+## Documentation 
 
-stop cluster
-```
-docker-compose down
-```
+[https://docs.grassecon.org/cic_stack/](https://docs.grassecon.org/cic_stack/)
 
-stop cluster and delete data
-```
-docker-compose down -v --remove-orphans
-```
-
-rebuild an images
-```
-docker-compose up --build <service_name>
-```
-
-to delete the buildkit cache
-```
-docker builder prune --filter type=exec.cachemount
-```

apps/cic-cache/cic_cache/cli/arg.py

@@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser):
         if local_arg_flags & CICFlag.CELERY:
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -95,10 +95,10 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
-        initial_block_offset = block_offset
+        initial_block_start = int(config.get('SYNCER_OFFSET'))
+        initial_block_offset = int(block_offset)
         if config.get('SYNCER_NO_HISTORY'):
-            initial_block_start = block_offset
+            initial_block_start = initial_block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
         logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))

apps/cic-cache/docker/Dockerfile

@@ -4,7 +4,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 
 COPY requirements.txt .
 
-ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL="https://pypi.org/simple"
 

apps/cic-cache/docker/db.sh

@@ -2,5 +2,5 @@
 
 set -e
 >&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
+python scripts/migrate.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
 set +e

apps/cic-cache/docker/run_tests.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+pip install --extra-index-url https://pip.grassrootseconomics.net \
 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
 -r test_requirements.txt 
 

apps/cic-eth/cic_eth/api/admin.py

@@ -123,7 +123,7 @@ class AdminApi:
         return s_lock.apply_async()
 
 
-    def tag_account(self, tag, address_hex, chain_spec):
+    def tag_account(self, chain_spec, tag, address):
         """Persistently associate an address with a plaintext tag.
 
         Some tags are known by the system and is used to resolve addresses to use for certain transactions. 
@@ -138,7 +138,7 @@ class AdminApi:
             'cic_eth.eth.account.set_role',
             [
                 tag,
-                address_hex,
+                address,
                 chain_spec.asdict(),
                 ],
             queue=self.queue,
@@ -146,6 +146,30 @@ class AdminApi:
         return s_tag.apply_async()
 
 
+    def get_tag_account(self, chain_spec, tag=None, address=None):
+        if address != None:
+            s_tag = celery.signature(
+                'cic_eth.eth.account.role',
+                [
+                    address,
+                    chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+
+        else:
+            s_tag = celery.signature(
+                'cic_eth.eth.account.role_account',
+                [
+                    tag,
+                    chain_spec.asdict(),
+                ],
+                queue=self.queue,
+                )
+
+        return s_tag.apply_async()
+
+
     def have_account(self, address_hex, chain_spec):
         s_have = celery.signature(
             'cic_eth.eth.account.have',
@@ -503,7 +527,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()
+                role = t.get()[0][1]
                 if role != None:
                     tx['sender_description'] = role
 
@@ -556,7 +580,7 @@ class AdminApi:
                     queue=self.queue,
                     )
                 t = s.apply_async()
-                role = t.get()
+                role = t.get()[0][1]
                 if role != None:
                     tx['recipient_description'] = role
 

apps/cic-eth/cic_eth/check/gas.py

@@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.admin.ctrl import check_lock
+from cic_eth.eth.gas import have_gas_minimum
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 
 def health(*args, **kwargs):
@@ -31,18 +32,15 @@ def health(*args, **kwargs):
         return True
 
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
+    if config.get('ETH_MIN_FEE_PRICE'):
+        min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
+
+    r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
+
     session.close()
+    
+    if not r:
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
 
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    o = balance(gas_provider)
-    r = rpc.do(o)
-    try: 
-        r = int(r, 16)
-    except TypeError:
-        r = int(r)
-    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
-    if r < gas_min:
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
-        return False
-
-    return True
+    return r

apps/cic-eth/cic_eth/check/start.py

@@ -0,0 +1,18 @@
+# external imports
+from chainlib.chain import ChainSpec
+
+# local imports
+from cic_eth.admin.ctrl import check_lock
+from cic_eth.enum import LockEnum
+from cic_eth.error import LockedError
+
+
+def health(*args, **kwargs):
+    config = kwargs['config']
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+    try:
+        check_lock(None, chain_spec.asdict(), LockEnum.START)
+    except LockedError as e:
+        return False
+    return True

apps/cic-eth/cic_eth/cli/arg.py

@@ -16,16 +16,22 @@ class ArgumentParser(BaseArgumentParser):
             self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
             self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
         if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
             self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
         if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
+            self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
+            self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
+            self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
+            self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
+            self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
+            self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
+            self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
+            self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
         if local_arg_flags & CICFlag.CHAIN:
             self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
-
-
-

apps/cic-eth/cic_eth/cli/base.py

@@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum):
     # sync - nibble 4
     SYNCER = 4096
 
-
-argflag_local_task = CICFlag.CELERY 
+argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
+argflag_local_task = CICFlag.CELERY
 argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
 argflag_local_chain = CICFlag.CHAIN
 argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/config.py

@@ -1,12 +1,18 @@
 # standard imports
 import os
 import logging
+import urllib.parse
+import copy
 
 # external imports
 from chainlib.eth.cli import (
         Config as BaseConfig,
         Flag,
         )
+from urlybird.merge import (
+        urlhostmerge,
+        urlmerge,
+        )
 
 # local imports
 from .base import CICFlag
@@ -40,6 +46,7 @@ class Config(BaseConfig):
         if local_arg_flags & CICFlag.CHAIN:
             local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
 
+        
         if local_arg_flags & CICFlag.CELERY:
             local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
 
@@ -49,15 +56,61 @@ class Config(BaseConfig):
 
         config.dict_override(local_args_override, 'local cli args')
 
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
+        local_celery_args_override = {}
         if local_arg_flags & CICFlag.CELERY:
+            hostport = urlhostmerge(
+                    None,
+                    config.get('REDIS_HOST'),
+                    config.get('REDIS_PORT'),
+                    )
+            redis_url = (
+                    'redis',
+                    hostport,
+                    getattr(args, 'redis_db', None),
+                    )
+            celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
+            hostport = urlhostmerge(
+                    celery_config_url[1],
+                    getattr(args, 'celery_host', None),
+                    getattr(args, 'celery_port', None),
+                    )
+            celery_arg_url = (
+                    getattr(args, 'celery_scheme', None),
+                    hostport,
+                    getattr(args, 'celery_db', None),
+                    )
+            celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
+            celery_url_string = urllib.parse.urlunsplit(celery_url)
+            local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
+            if not getattr(args, 'celery_no_result'):
+                local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
+                if local_celery_args_override['CELERY_RESULT_URL'] == None:
+                    local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
+                celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
+                hostport = urlhostmerge(
+                        celery_config_url[1],
+                        getattr(args, 'celery_result_host', None),
+                        getattr(args, 'celery_result_port', None),
+                        )
+                celery_arg_url = (
+                        getattr(args, 'celery_result_scheme', None),
+                        hostport,
+                        getattr(args, 'celery_result_db', None),
+                        )
+                celery_url = urlmerge(celery_config_url, celery_arg_url)
+                logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
+                celery_url_string = urllib.parse.urlunsplit(celery_url)
+                local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
             config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
 
+        config.dict_override(local_celery_args_override, 'local celery cli args')
+
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
+            redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
+            config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
+            config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
+
         logg.debug('config loaded:\n{}'.format(config))
 
         return config
-
-

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -1,5 +1,5 @@
 [celery]
-broker_url = redis://localhost:6379
+broker_url = 
 result_url = 
 queue = cic-eth
 debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -2,5 +2,5 @@
 registry_address = 
 trust_address =
 default_token_symbol =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
 run_dir = /run

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,2 +1,6 @@
 [eth]
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_holder_minimum_units = 180000
+gas_holder_refill_units = 15
+gas_holder_refill_threshold = 3
+gas_gifter_refill_buffer = 3
+min_fee_price = 1

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -23,7 +23,7 @@ def upgrade():
     op.create_table(
             'lock',
             sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String(42), nullable=True),
+            sa.Column("address", sa.String, nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py

@@ -0,0 +1,31 @@
+"""Add gas cache
+
+Revision ID: c91cafc3e0c1
+Revises: aee12aeb47ec
+Create Date: 2021-10-28 20:45:34.239865
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c91cafc3e0c1'
+down_revision = 'aee12aeb47ec'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'gas_cache',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column("address", sa.String, nullable=False),
+            sa.Column("tx_hash", sa.String, nullable=True),
+            sa.Column("method", sa.String, nullable=True),
+            sa.Column("value", sa.BIGINT(), nullable=False),
+            )
+
+
+def downgrade():
+    op.drop_table('gas_cache')

apps/cic-eth/cic_eth/db/models/gas_cache.py

@@ -0,0 +1,27 @@
+# standard imports
+import logging
+
+# external imports
+from sqlalchemy import Column, String, NUMERIC
+
+# local imports
+from .base import SessionBase
+
+logg = logging.getLogger(__name__)
+
+
+class GasCache(SessionBase):
+    """Provides gas budget cache for token operations
+    """
+    __tablename__ = 'gas_cache'
+
+    address = Column(String())
+    tx_hash = Column(String())
+    method = Column(String())
+    value = Column(NUMERIC())
+
+    def __init__(self, address, method, value, tx_hash):
+        self.address = address
+        self.tx_hash = tx_hash
+        self.method = method
+        self.value = value

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -12,7 +12,7 @@ from cic_eth.error import (
         IntegrityError,
         )
 
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
     __tablename__ = 'nonce'
 
     nonce = Column(Integer)
-    address_hex = Column(String(42))
+    address_hex = Column(String(40))
 
 
     @staticmethod

apps/cic-eth/cic_eth/db/models/role.py

@@ -24,8 +24,22 @@ class AccountRole(SessionBase):
     tag = Column(Text)
     address_hex = Column(String(42))
 
-    
-    # TODO: 
+
+    @staticmethod
+    def all(session=None):
+        session = SessionBase.bind_session(session)
+
+        pairs = []
+
+        q = session.query(AccountRole.tag, AccountRole.address_hex)
+        for r in q.all():
+            pairs.append((r[1], r[0]),)
+
+        SessionBase.release_session(session)
+        
+        return pairs
+
+
     @staticmethod
     def get_address(tag, session):
         """Get Ethereum address matching the given tag

apps/cic-eth/cic_eth/enum.py

@@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum):
 class LockEnum(enum.IntEnum):
     """
     STICKY: When set, reset is not possible
+    INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
+    START: When set, startup is not possible, regardless of state
     CREATE: Disable creation of accounts
     SEND: Disable sending to network
     QUEUE: Disable queueing new or modified transactions
+    QUERY: Disable all queue state and transaction queries
     """
     STICKY=1
     INIT=2
@@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum):
     SEND=8
     QUEUE=16
     QUERY=32
-    ALL=int(0xfffffffffffffffe)
+    START=int(0x80000000)
+    ALL=int(0x7ffffffe)
 
 
 def status_str(v, bits_only=False):

apps/cic-eth/cic_eth/error.py

@@ -64,8 +64,10 @@ class LockedError(Exception):
 
 class SeppukuError(Exception):
     """Exception base class for all errors that should cause system shutdown
-
     """
+    def __init__(self, message, lockdown=False):
+        self.message = message
+        self.lockdown = lockdown
 
 
 class SignerError(SeppukuError):

apps/cic-eth/cic_eth/eth/account.py

@@ -136,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
     account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -192,7 +192,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -266,19 +266,46 @@ def set_role(self, tag, address, chain_spec_dict):
 
 @celery_app.task(bind=True, base=BaseTask)
 def role(self, address, chain_spec_dict):
-    """Return account role for address
+    """Return account role for address and/or role
 
     :param account: Account to check
     :type account: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
+    :param chain_spec_dict: Chain spec dict representation
+    :type chain_spec_dict: dict
     :returns: Account, or None if not exists
     :rtype: Varies
     """
     session = self.create_session()
     role_tag =  AccountRole.role_for(address, session=session)
     session.close()
-    return role_tag
+    return [(address, role_tag,)]
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def role_account(self, role_tag, chain_spec_dict):
+    """Return address for role.
+
+    If the role parameter is None, will return addresses for all roles.
+
+    :param role_tag: Role to match
+    :type role_tag: str
+    :param chain_spec_dict: Chain spec dict representation
+    :type chain_spec_dict: dict
+    :returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
+    :rtype: list
+    """
+    session = self.create_session()
+
+    pairs = None
+    if role_tag != None:
+        addr = AccountRole.get_address(role_tag, session=session)
+        pairs = [(addr, role_tag,)]
+    else:
+        pairs = AccountRole.all(session=session)
+
+    session.close()
+
+    return pairs
 
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)

apps/cic-eth/cic_eth/eth/erc20.py

@@ -10,6 +10,9 @@ from chainlib.eth.tx import (
         TxFormat,
         unpack,
         )
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import (
@@ -31,10 +34,8 @@ from cic_eth.error import (
         YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
-from cic_eth.eth.gas import (
-        create_check_gas_task,
-        MaxGasOracle,
-        )
+from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.eth.util import CacheGasOracle
 from cic_eth.ext.address import translate_address
 from cic_eth.task import (
         CriticalSQLAlchemyTask,
@@ -154,8 +155,12 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    enc = ABIContractEncoder()
+    enc.method('transferFrom')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -225,8 +230,12 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
+    enc = ABIContractEncoder()
+    enc.method('transfer')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -294,8 +303,12 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
 
     session = self.create_session()
+
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    enc = ABIContractEncoder()
+    enc.method('approve')
+    method = enc.get()
+    gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)

apps/cic-eth/cic_eth/eth/gas.py

@@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
+from cic_eth.db.models.gas_cache import GasCache
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 from cic_eth.error import (
@@ -65,17 +66,56 @@ from cic_eth.encode import (
         ZERO_ADDRESS_NORMAL,
         unpack_normal,
         )
+from cic_eth.error import SeppukuError
+from cic_eth.eth.util import MAXIMUM_FEE_UNITS
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-MAXIMUM_FEE_UNITS = 8000000
 
-class MaxGasOracle:
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def apply_gas_value_cache(address, method, value, tx_hash):
+    return apply_gas_value_cache_local(address, method, value, tx_hash)
 
-    def gas(code=None):
-        return MAXIMUM_FEE_UNITS
+
+def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
+    address = tx_normalize.executable_address(address)
+    tx_hash = tx_normalize.tx_hash(tx_hash)
+    value = int(value)
+
+    session = SessionBase.bind_session(session)
+    q = session.query(GasCache)
+    q = q.filter(GasCache.address==address)
+    q = q.filter(GasCache.method==method)
+    o = q.first()
+
+    if o == None:
+        o = GasCache(address, method, value, tx_hash)
+    elif tx.gas_used > o.value:
+        o.value = value
+        o.tx_hash = strip_0x(tx_hash)
+
+    session.add(o)
+    session.commit()
+
+    SessionBase.release_session(session)
+
+
+def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
+    if rpc == None:
+        rpc = RPCConnection.connect(chain_spec, 'default')
+    o = balance(add_0x(address))
+    r = rpc.do(o)
+    try: 
+        r = int(r)
+    except ValueError:
+        r = strip_0x(r)
+        r = int(r, 16)
+    logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
+    if r < min_gas:
+        return False
+    return True
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -357,6 +397,13 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # set up evm RPC connection
     rpc = RPCConnection.connect(chain_spec, 'default')
 
+    # check the gas balance of the gifter
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
+        raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
+
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
+        logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
+    
     # set up transaction builder
     nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc)

apps/cic-eth/cic_eth/eth/util.py

@@ -0,0 +1,54 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.eth.gas import RPCGasOracle
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.db.models.gas_cache import GasCache
+from cic_eth.encode import tx_normalize
+from cic_eth.db.models.base import SessionBase
+
+MAXIMUM_FEE_UNITS = 8000000
+
+logg = logging.getLogger(__name__)
+
+
+class MaxGasOracle(RPCGasOracle):
+
+    def get_fee_units(self, code=None):
+        return MAXIMUM_FEE_UNITS
+
+
+class CacheGasOracle(MaxGasOracle):
+    """Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
+
+    :todo: instead of max units, connect a pluggable gas heuristics engine.
+    """
+
+    def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
+        super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
+        self.value = None
+        self.address = address
+        self.method = method
+
+        address = tx_normalize.executable_address(address)
+        session = SessionBase.bind_session(session)
+        q = session.query(GasCache)
+        q = q.filter(GasCache.address==address)
+        if method != None:
+            method = strip_0x(method)
+            q = q.filter(GasCache.method==method)
+        o = q.first()
+        if o != None:
+            self.value = int(o.value)
+
+        SessionBase.release_session(session)
+
+
+    def get_fee_units(self, code=None):
+        if self.value != None:
+            logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
+            return self.value
+        return super(CacheGasOracle, self).get_fee_units(code=code)

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -8,15 +8,14 @@ import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(os.path.dirname(script_dir))
+config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
 logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.environ.get('CONFINI_DIR')
-    if config_dir == None:
-        config_dir = os.path.join(root_dir, 'config/test')
-    conf = confini.Config(config_dir, 'CICTEST')
+    override_config_dir = os.path.join(root_dir, 'config', 'test')
+    conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py

@@ -3,3 +3,4 @@ from .tx import TxFilter
 from .gas import GasFilter
 from .register import RegistrationFilter
 from .transferauth import TransferAuthFilter
+from .token import TokenFilter

apps/cic-eth/cic_eth/runnable/daemons/filters/token.py

@@ -0,0 +1,63 @@
+# standard imports
+import logging
+
+# external imports
+from eth_erc20 import ERC20
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.address import is_same_address
+from chainlib.eth.error import RequestMismatchException
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
+from eth_token_index import TokenUniqueSymbolIndex
+import celery
+
+# local imports
+from .base import SyncFilter
+
+logg = logging.getLogger(__name__)
+
+
+class TokenFilter(SyncFilter):
+
+    def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
+        self.queue = queue
+        self.chain_spec = chain_spec
+        self.caller_address = call_address
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        if not tx.payload:
+            return (None, None)
+
+        try:
+            r = ERC20.parse_transfer_request(tx.payload)
+        except RequestMismatchException:
+            return (None, None)
+
+        token_address = tx.inputs[0]
+        token = ERC20Token(self.chain_spec, conn, token_address)
+
+        registry = CICRegistry(self.chain_spec, conn)
+        r = registry.by_name(token.symbol, sender_address=self.caller_address)
+        if is_same_address(r, ZERO_ADDRESS):
+            return None
+
+        enc = ABIContractEncoder()
+        enc.method('transfer')
+        method = enc.get()
+
+        s = celery.signature(
+                'cic_eth.eth.gas.apply_gas_value_cache',
+                [
+                    token_address,
+                    method,
+                    tx.gas_used,
+                    tx.hash,
+                    ],
+                queue=self.queue,
+                )
+        return s.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -67,7 +67,10 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import BaseTask
+from cic_eth.task import (
+        BaseTask,
+        CriticalWeb3Task,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -76,18 +79,18 @@ arg_flags = cic_eth.cli.argflag_std_read
 local_arg_flags = cic_eth.cli.argflag_local_task
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.process_local_flags(local_arg_flags)
-#argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
+argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei')
 argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
 args = argparser.parse_args()
 
 # process config
 extra_args = {
-#    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
     'aux_all': None,
     'aux': None,
     'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
+    'min_fee_price': 'ETH_MIN_FEE_PRICE',
         }
 config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
@@ -215,6 +218,7 @@ def main():
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
 
+    # TODO: More elegant way of setting queue-wide settings
     BaseTask.default_token_symbol = default_token_symbol
     BaseTask.default_token_address = default_token_address
     default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
@@ -222,6 +226,14 @@ def main():
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
     BaseTask.trusted_addresses = trusted_addresses
+    CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
+    CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
+    CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
+    if config.get('ETH_MIN_FEE_PRICE'):
+        BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
+        CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
+        CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
+        CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
 
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -36,6 +36,7 @@ from cic_eth.runnable.daemons.filters import (
         TxFilter,
         RegistrationFilter,
         TransferAuthFilter,
+        TokenFilter,
         )
 from cic_eth.stat import init_chain_stat
 from cic_eth.registry import (
@@ -99,10 +100,10 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
-        initial_block_offset = block_offset
+        initial_block_start = int(config.get('SYNCER_OFFSET'))
+        initial_block_offset = int(block_offset)
         if config.true('SYNCER_NO_HISTORY'):
-            initial_block_start = block_offset
+            initial_block_start = initial_block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
         logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
@@ -154,6 +155,8 @@ def main():
 
     gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
 
+    token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE'))
+
     #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
 
     i = 0
@@ -163,6 +166,7 @@ def main():
         syncer.add_filter(registration_filter)
         # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
         syncer.add_filter(tx_filter)
+        syncer.add_filter(token_gas_cache_filter)
         #syncer.add_filter(transfer_auth_filter)
         for cf in callback_filters:
             syncer.add_filter(cf)

apps/cic-eth/cic_eth/runnable/tag.py

@@ -8,6 +8,7 @@ import re
 # external imports
 import cic_eth.cli
 from chainlib.chain import ChainSpec
+from chainlib.eth.address import is_address
 from xdg.BaseDirectory import xdg_config_home
 
 # local imports
@@ -21,12 +22,18 @@ logg = logging.getLogger()
 arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_positional('tag', type=str, help='address tag')
-argparser.add_positional('address', type=str, help='address')
+argparser.add_argument('--set', action='store_true', help='sets the given tag')
+argparser.add_argument('--tag', type=str, help='operate on the given tag')
+argparser.add_positional('address', required=False, type=str, help='address associated with tag')
 argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+extra_args = {
+        'set': None,
+        'tag': None,
+        'address': None,
+        }
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
 celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
@@ -39,7 +46,17 @@ api = AdminApi(None)
 
 
 def main():
-    admin_api.tag_account(args.tag, args.address, chain_spec)
+    if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')):
+        sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS')))
+        sys.exit(1)
+
+    if config.get('_SET'):
+        admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS'))
+    else:
+        t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS'))
+        r = t.get()
+        for v in r:
+            sys.stdout.write('{}\t{}\n'.format(v[1], v[0]))
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -18,7 +18,7 @@ from cic_eth.api import Api
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger('create_account_script')
 
-arg_flags = cic_eth.cli.argflag_std_base
+arg_flags = cic_eth.cli.argflag_local_base
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')

apps/cic-eth/cic_eth/runnable/view.py

@@ -16,9 +16,14 @@ import confini
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
-from hexathon import add_0x
+from hexathon import (
+        add_0x,
+        strip_0x,
+        uniform as hex_uniform,
+        )
 
 # local imports
+import cic_eth.cli
 from cic_eth.api.admin import AdminApi
 from cic_eth.db.enum import (
     StatusEnum,
@@ -31,59 +36,35 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 default_format = 'terminal'
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
+arg_flags = cic_eth.cli.argflag_std_base
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
-argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
 argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"')
+argparser.process_local_flags(local_arg_flags)
 args = argparser.parse_args()
 
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
 
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+extra_args = {
+    'f': '_FORMAT',
+    'query': '_QUERY',
         }
-# override args
-config.dict_override(args_override, 'cli args')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-try:
-    config.add(add_0x(args.query), '_QUERY', True)
-except:
-    config.add(args.query, '_QUERY', True)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
+queue = config.get('CELERY_QUEUE')
 
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-queue = args.q
+# connect to celery
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True)
+conn = rpc.get_default()
 
-rpc = EthHTTPConnection(args.p)
-
-#registry_address = config.get('CIC_REGISTRY_ADDRESS')
-
-admin_api = AdminApi(rpc)
+admin_api = AdminApi(conn)
 
 t = admin_api.registry()
 registry_address = t.get()
@@ -113,7 +94,7 @@ def render_tx(o, **kwargs):
     
     for v in o.get('status_log', []):
         d = datetime.datetime.fromisoformat(v[0])
-        e = status_str(v[1], args.status_raw)
+        e = status_str(v[1], config.get('_RAW'))
         content += '{}: {}\n'.format(d, e)
 
     return content
@@ -154,20 +135,24 @@ def render_lock(o, **kwargs):
 def main():
     txs  = []
     renderer = render_tx
-    if len(config.get('_QUERY')) > 66:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
-    elif len(config.get('_QUERY')) > 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
 
-    elif len(config.get('_QUERY')) == 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
-        txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
+    query = config.get('_QUERY')
+    try:
+        query = hex_uniform(strip_0x(query))
+    except TypeError:
+        pass
+    except ValueError:
+        pass
+
+    if len(query) > 64:
+        admin_api.tx(chain_spec, tx_raw=query, renderer=renderer)
+    elif len(query) > 40:
+        admin_api.tx(chain_spec, tx_hash=query, renderer=renderer)
+
+    elif len(query) == 40:
+        txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account)
         renderer = render_account
-    elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
+    elif len(query) >= 4 and query[:4] == 'lock':
         t = admin_api.get_lock()
         txs = t.get()
         renderer = render_lock
@@ -175,7 +160,7 @@ def main():
             r = renderer(txs)
             sys.stdout.write(r + '\n')
     else:
-        raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
+        raise ValueError('cannot parse argument {}'.format(query))
                    
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/task.py

@@ -17,6 +17,7 @@ from cic_eth_registry.error import UnknownContractError
 # local imports
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
+from cic_eth.eth.util import CacheGasOracle
 
 #logg = logging.getLogger().getChild(__name__)
 logg = logging.getLogger()
@@ -29,14 +30,32 @@ class BaseTask(celery.Task):
     session_func = SessionBase.create_session
     call_address = ZERO_ADDRESS
     trusted_addresses = []
-    create_nonce_oracle = RPCNonceOracle
-    create_gas_oracle = RPCGasOracle
+    min_fee_price = 1
     default_token_address = None
     default_token_symbol = None
     default_token_name = None
     default_token_decimals = None
     run_dir = '/run'
 
+
+    def create_gas_oracle(self, conn, address=None, *args, **kwargs):
+        if address == None:
+            return RPCGasOracle(
+                conn,
+                code_callback=kwargs.get('code_callback'),
+                min_price=self.min_fee_price,
+                id_generator=kwargs.get('id_generator'),
+                )
+
+        return CacheGasOracle(
+                conn,
+                address,
+                method=kwargs.get('method'),
+                min_price=self.min_fee_price,
+                id_generator=kwargs.get('id_generator'),
+                )
+
+
     def create_session(self):
         return BaseTask.session_func()
 
@@ -78,19 +97,18 @@ class CriticalWeb3Task(CriticalTask):
     autoretry_for = (
         ConnectionError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_threshold_amount = 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+    safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100
 
 
-class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
+class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
-    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
 class CriticalSQLAlchemyAndSignerTask(CriticalTask):
@@ -100,14 +118,11 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.ResourceClosedError,
         ) 
 
-class CriticalWeb3AndSignerTask(CriticalTask):
+class CriticalWeb3AndSignerTask(CriticalWeb3Task):
     autoretry_for = (
         ConnectionError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
-    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
-
-
+    
 @celery_app.task()
 def check_health(self):
     pass

apps/cic-eth/config/test/accounts.ini

@@ -1,2 +0,0 @@
-[accounts]
-writer_address = 

apps/cic-eth/config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = tests/testdata/bancor

apps/cic-eth/config/test/celery.ini

@@ -1,5 +1,3 @@
 [celery]
 broker_url = filesystem://
 result_url = filesystem://
-#broker_url = redis://
-#result_url = redis://

apps/cic-eth/config/test/chain.ini

@@ -1,2 +0,0 @@
-[chain]
-spec =

apps/cic-eth/config/test/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = 
-trust_address =

apps/cic-eth/config/test/dispatcher.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 0.1

apps/cic-eth/config/test/eth.ini

@@ -1,8 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-gas_provider_address = 
-#chain_id = 
-abi_dir = 
-faucet_giver_address = 

apps/cic-eth/config/test/signer.ini

@@ -1,5 +1,2 @@
 [signer]
-socket_path = /run/crypto-dev-signer/jsonrpc.ipc
-secret = deedbeef
-database_name = signer_test
-dev_keys_path = 
+provider = /run/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/config/test/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = false
-cert_file = 
-key_file =
-password = 
-ca_file =

apps/cic-eth/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[SYNCER]
-loop_interval = 1

apps/cic-eth/docker/Dockerfile

@@ -7,7 +7,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 # TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
 #COPY cic-eth/requirements.txt .
 
-ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple
 
@@ -15,7 +15,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url $PIP_INDEX_URL \
     --pre \
     --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-        cic-eth-aux-erc20-demurrage-token~=0.0.2a7
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a7
 
 
 COPY *requirements.txt ./
@@ -26,7 +26,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt 
-  
+
 COPY . .
 RUN python setup.py install
 

apps/cic-eth/docker/run_tests.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
 -r admin_requirements.txt \
 -r services_requirements.txt \
 -r test_requirements.txt  

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
 celery==4.4.7
-chainlib-eth>=0.0.10a16,<0.1.0
+chainlib-eth>=0.0.10a20,<0.1.0
 semver==2.13.0
-crypto-dev-signer>=0.4.15rc2,<0.5.0
+urlybird~=0.0.1a2

apps/cic-eth/setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 name = cic-eth
 #version = attr: cic_eth.version.__version_string__
-version = 0.12.4a13
+version = 0.12.5a2
 description = CIC Network Ethereum interaction 
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth/tests/filters/test_token_filter.py

@@ -0,0 +1,97 @@
+# external imports
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        receipt,
+        raw,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        Block,
+        block_latest,
+        block_by_number,
+        )
+from chainlib.eth.address import is_same_address
+from chainlib.eth.contract import ABIContractEncoder
+from hexathon import strip_0x
+from eth_token_index import TokenUniqueSymbolIndex
+
+# local imports
+from cic_eth.runnable.daemons.filters.token import TokenFilter
+from cic_eth.db.models.gas_cache import GasCache
+from cic_eth.db.models.base import SessionBase
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        contract_roles,
+        agent_roles,
+        token_roles,
+        foo_token,
+        token_registry,
+        register_lookups,
+        celery_session_worker,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE'])
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    tx.apply_receipt(rcpt)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert t == None
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(GasCache)
+    q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex))
+    o = q.first()
+
+    assert is_same_address(o.address, strip_0x(foo_token))
+    assert o.value > 0
+
+    enc = ABIContractEncoder()
+    enc.method('transfer')
+    method = enc.get()
+
+    assert o.method == method

apps/cic-eth/tests/run_tests.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
 -r admin_requirements.txt
 -r services_requirements.txt
 -r test_requirements.txt 

apps/cic-eth/tests/task/api/test_admin.py

@@ -103,11 +103,11 @@ def test_tag_account(
 
     api = AdminApi(eth_rpc, queue=None)
 
-    t = api.tag_account('foo', agent_roles['ALICE'], default_chain_spec)
+    t = api.tag_account(default_chain_spec, 'foo', agent_roles['ALICE'])
     t.get()
-    t = api.tag_account('bar', agent_roles['BOB'], default_chain_spec)
+    t = api.tag_account(default_chain_spec, 'bar', agent_roles['BOB'])
     t.get()
-    t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec)
+    t = api.tag_account(default_chain_spec, 'bar', agent_roles['CAROL'])
     t.get()
 
     assert AccountRole.get_address('foo', init_database) == tx_normalize.wallet_address(agent_roles['ALICE'])

apps/cic-eth/tests/task/test_task_account.py

@@ -141,9 +141,57 @@ def test_role_task(
             )
     t = s.apply_async()
     r = t.get()
-    assert r == 'foo'
+    assert r[0][0] == address
+    assert r[0][1] == 'foo'
 
 
+def test_get_role_task(
+        init_database,
+        celery_session_worker,
+        default_chain_spec,
+        ):
+    address_foo = '0x' + os.urandom(20).hex()
+    role_foo = AccountRole.set('foo', address_foo)
+    init_database.add(role_foo)
+
+    address_bar = '0x' + os.urandom(20).hex()
+    role_bar = AccountRole.set('bar', address_bar)
+    init_database.add(role_bar)
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.account.role_account',
+            [
+                'bar',
+                default_chain_spec.asdict(),
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get()
+    assert r[0][0] == address_bar
+    assert r[0][1] == 'bar'
+
+    s = celery.signature(
+            'cic_eth.eth.account.role_account',
+            [
+                None,
+                default_chain_spec.asdict(),
+                ],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get()
+    x_tags = ['foo', 'bar']
+    x_addrs = [address_foo, address_bar]
+
+    for v in r:
+        x_addrs.remove(v[0])
+        x_tags.remove(v[1])
+    
+    assert len(x_tags) == 0
+    assert len(x_addrs) == 0
 
 def test_gift(
     init_database,

apps/cic-notify/docker/Dockerfile

@@ -6,7 +6,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
 RUN apt-get install libffi-dev -y
 
 
-ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple
 

apps/cic-notify/docker/run_tests.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+pip install --extra-index-url https://pip.grassrootseconomics.net \
 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
 -r test_requirements.txt 
 

apps/cic-signer/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get install libffi-dev -y
 
 COPY requirements.txt . 
 
-ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL="https://pypi.org/simple"
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \

apps/cic-signer/cic_signer/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine = postgres
+driver = psycopg2
+host = localhost
+port = 5432
+name = cic_signer
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-signer/cic_signer/data/config/signer.ini

@@ -0,0 +1,3 @@
+[signer]
+provider =
+secret =

apps/cic-signer/requirements.txt

@@ -1 +1,2 @@
 funga-eth[sql]>=0.5.1a1,<0.6.0
+chainlib-eth>=0.0.10a18

apps/cic-signer/scripts/sweep.py

@@ -0,0 +1,128 @@
+# standard imports
+import os
+import logging
+import uuid
+import random
+import sys
+
+# external imports
+from chainlib.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.gas import (
+        balance,
+        Gas,
+        )
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.eth.connection import EthHTTPSignerConnection
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.sql import SQLKeystore
+from chainlib.cli.wallet import Wallet
+from chainlib.eth.address import AddressChecksum
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        )
+import chainlib.eth.cli
+
+       
+script_dir = os.path.dirname(os.path.realpath(__file__))
+config_dir = os.path.join(script_dir, '..', 'cic_signer', 'data', 'config')
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+arg_flags = chainlib.eth.cli.argflag_std_write | chainlib.eth.cli.Flag.WALLET
+argparser = chainlib.eth.cli.ArgumentParser(arg_flags)
+args = argparser.parse_args()
+
+config = chainlib.eth.cli.Config.from_args(args, arg_flags, base_config_dir=config_dir)
+
+# set up rpc
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+
+# connect to database
+dsn = 'postgresql://{}:{}@{}:{}/{}'.format(
+        config.get('DATABASE_USER'),
+        config.get('DATABASE_PASSWORD'),
+        config.get('DATABASE_HOST'),
+        config.get('DATABASE_PORT'),
+        config.get('DATABASE_NAME'),    
+    )
+logg.info('using dsn {}'.format(dsn))
+
+keystore = SQLKeystore(dsn, symmetric_key=bytes.fromhex(config.get('SIGNER_SECRET')))
+wallet = Wallet(EIP155Signer, keystore=keystore, checksummer=AddressChecksum)
+
+rpc = chainlib.eth.cli.Rpc(wallet=wallet)
+conn = rpc.connect_by_config(config)
+
+wallet.init()
+
+def main():
+    if config.get('_RECIPIENT') == None:
+        sys.stderr.write('Missing sink address\n')
+        sys.exit(1)
+
+    sink_address = config.get('_RECIPIENT')
+    if config.get('_UNSAFE'):
+        sink_address = to_checksum_address(sink_address)
+    if not is_checksum_address(sink_address):
+        sys.stderr.write('Invalid sink address {}\n'.format(sink_address))
+        sys.exit(1)
+
+    if (config.get('_RPC_SEND')):
+        verify_string = random.randbytes(4).hex()
+        verify_string_check = input("\033[;31m*** WARNING! WARNING! WARNING! ***\033[;39m\nThis action will transfer all remaining gas from all accounts in custodial care to account {}. To confirm, please enter the string: {}\n".format(config.get('_RECIPIENT'), verify_string))
+        if verify_string != verify_string_check:
+            sys.stderr.write('Verify string mismatch. Aborting!\n')
+            sys.exit(1)
+
+    signer = rpc.get_signer()
+
+    gas_oracle = rpc.get_gas_oracle()
+    gas_pair = gas_oracle.get_fee()
+    gas_price = gas_pair[0]
+    gas_limit = 21000
+    gas_cost = gas_price * gas_limit
+    gas_oracle = OverrideGasOracle(price=gas_price, limit=gas_limit)
+    logg.info('using gas price {}'.format(gas_price))
+
+    for r in keystore.list():
+        account = r[0]
+
+        o = balance(add_0x(account))
+        r = conn.do(o)
+        account_balance = 0
+        try:
+            r = strip_0x(r)
+            account_balance = int(r, 16)
+        except ValueError:
+            account_balance = int(r)
+
+        transfer_amount = account_balance - gas_cost
+        if transfer_amount <= 0:
+            logg.warning('address {} has balance {} which is less than gas cost {}, skipping'.format(account, account_balance, gas_cost))
+            continue
+
+        nonce_oracle = RPCNonceOracle(account, conn)
+        c = Gas(chain_spec, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, signer=signer)
+        tx_hash_hex = None
+        if (config.get('_RPC_SEND')):
+            (tx_hash_hex, o) = c.create(account, config.get('_RECIPIENT'), transfer_amount)
+            r = conn.do(o)
+        else:
+            (tx_hash_hex, o) = c.create(account, config.get('_RECIPIENT'), transfer_amount, tx_format=TxFormat.RLP_SIGNED)
+
+        logg.info('address {} balance {} net transfer {} tx {}'.format(account, account_balance, transfer_amount, tx_hash_hex))
+
+if __name__ == '__main__':
+    main()

apps/cic-ussd/cic_ussd/account/balance.py

@@ -2,7 +2,7 @@
 
 import json
 import logging
-from typing import Optional
+from typing import Union, Optional
 
 # third-party imports
 from cic_eth.api import Api
@@ -14,7 +14,7 @@ from cic_ussd.account.transaction import from_wei
 from cic_ussd.cache import cache_data_key, get_cached_data
 from cic_ussd.error import CachedDataNotFoundError
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 def get_balances(address: str,
@@ -43,7 +43,7 @@ def get_balances(address: str,
     :return: A list containing balance data if called synchronously. | None
     :rtype: list | None
     """
-    logg.debug(f'retrieving balance for address: {address}')
+    logg.debug(f'retrieving {token_symbol} balance for address: {address}')
     if asynchronous:
         cic_eth_api = Api(
             chain_str=chain_str,
@@ -60,11 +60,13 @@ def get_balances(address: str,
         return balance_request_task.get()
 
 
-def calculate_available_balance(balances: dict) -> float:
+def calculate_available_balance(balances: dict, decimals: int) -> float:
     """This function calculates an account's balance at a specific point in time by computing the difference from the
     outgoing balance and the sum of the incoming and network balances.
     :param balances: incoming, network and outgoing balances.
     :type balances: dict
+    :param decimals:
+    :type decimals: int
     :return: Token value of the available balance.
     :rtype: float
     """
@@ -73,7 +75,7 @@ def calculate_available_balance(balances: dict) -> float:
     network_balance = balances.get('balance_network')
 
     available_balance = (network_balance + incoming_balance) - outgoing_balance
-    return from_wei(value=available_balance)
+    return from_wei(decimals=decimals, value=available_balance)
 
 
 def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str):
@@ -94,24 +96,25 @@ def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_sym
     return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result
 
 
-def get_cached_available_balance(blockchain_address: str) -> float:
+def get_cached_available_balance(decimals: int, identifier: Union[list, bytes]) -> float:
     """This function attempts to retrieve balance data from the redis cache.
-    :param blockchain_address: Ethereum address of an account.
-    :type blockchain_address: str
+    :param decimals:
+    :type decimals: int
+    :param identifier: An identifier needed to create a unique pointer to a balances resource.
+    :type identifier: bytes | list
     :raises CachedDataNotFoundError: No cached balance data could be found.
     :return: Operational balance of an account.
     :rtype: float
     """
-    identifier = bytes.fromhex(blockchain_address)
-    key = cache_data_key(identifier, salt=MetadataPointer.BALANCES)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.BALANCES)
     cached_balances = get_cached_data(key=key)
     if cached_balances:
-        return calculate_available_balance(json.loads(cached_balances))
+        return calculate_available_balance(balances=json.loads(cached_balances), decimals=decimals)
     else:
-        raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}')
+        raise CachedDataNotFoundError(f'No cached available balance at {key}')
 
 
-def get_cached_adjusted_balance(identifier: bytes):
+def get_cached_adjusted_balance(identifier: Union[list, bytes]):
     """
     :param identifier:
     :type identifier:
@@ -120,3 +123,22 @@ def get_cached_adjusted_balance(identifier: bytes):
     """
     key = cache_data_key(identifier, MetadataPointer.BALANCES_ADJUSTED)
     return get_cached_data(key)
+
+
+def get_account_tokens_balance(blockchain_address: str, chain_str: str, token_symbols_list: list):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param chain_str:
+    :type chain_str:
+    :param token_symbols_list:
+    :type token_symbols_list:
+    :return:
+    :rtype:
+    """
+    for token_symbol in token_symbols_list:
+        get_balances(address=blockchain_address,
+                     chain_str=chain_str,
+                     token_symbol=token_symbol,
+                     asynchronous=True,
+                     callback_param=f'{blockchain_address},{token_symbol}')

apps/cic-ussd/cic_ussd/account/statement.py

@@ -69,7 +69,8 @@ def parse_statement_transactions(statement: list):
     parsed_transactions = []
     for transaction in statement:
         action_tag = transaction.get('action_tag')
-        amount = from_wei(transaction.get('token_value'))
+        decimals = transaction.get('token_decimals')
+        amount = from_wei(decimals, transaction.get('token_value'))
         direction_tag = transaction.get('direction_tag')
         token_symbol = transaction.get('token_symbol')
         metadata_id = transaction.get('metadata_id')

apps/cic-ussd/cic_ussd/account/tokens.py

@@ -1,19 +1,115 @@
 # standard imports
+import hashlib
 import json
 import logging
-from typing import Dict, Optional
+from typing import Optional, Union
 
 # external imports
 from cic_eth.api import Api
 from cic_types.condiments import MetadataPointer
 
 # local imports
+from cic_ussd.account.balance import get_cached_available_balance
 from cic_ussd.account.chain import Chain
-from cic_ussd.cache import cache_data_key, get_cached_data
-from cic_ussd.error import SeppukuError
+from cic_ussd.cache import cache_data, cache_data_key, get_cached_data
+from cic_ussd.error import CachedDataNotFoundError, SeppukuError
+from cic_ussd.metadata.tokens import query_token_info, query_token_metadata
+from cic_ussd.processor.util import wait_for_cache
+from cic_ussd.translation import translation_for
+
+logg = logging.getLogger(__file__)
 
 
-logg = logging.getLogger(__name__)
+def collate_token_metadata(token_info: dict, token_metadata: dict) -> dict:
+    """
+    :param token_info:
+    :type token_info:
+    :param token_metadata:
+    :type token_metadata:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Collating token info: {token_info} and token metadata: {token_metadata}')
+    description = token_info.get('description')
+    issuer = token_info.get('issuer')
+    location = token_metadata.get('location')
+    contact = token_metadata.get('contact')
+    return {
+        'description': description,
+        'issuer': issuer,
+        'location': location,
+        'contact': contact
+    }
+
+
+def create_account_tokens_list(blockchain_address: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    token_symbols_list = get_cached_token_symbol_list(blockchain_address=blockchain_address)
+    token_list_entries = []
+    if token_symbols_list:
+        logg.debug(f'Token symbols: {token_symbols_list} for account: {blockchain_address}')
+        for token_symbol in token_symbols_list:
+            entry = {}
+            logg.debug(f'Processing token data for: {token_symbol}')
+            key = cache_data_key([bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')], MetadataPointer.TOKEN_DATA)
+            token_data = get_cached_data(key)
+            token_data = json.loads(token_data)
+            logg.debug(f'Retrieved token data: {token_data} for: {token_symbol}')
+            token_name = token_data.get('name')
+            entry['name'] = token_name
+            token_symbol = token_data.get('symbol')
+            entry['symbol'] = token_symbol
+            token_issuer = token_data.get('issuer')
+            entry['issuer'] = token_issuer
+            token_contact = token_data['contact'].get('phone')
+            entry['contact'] = token_contact
+            token_location = token_data.get('location')
+            entry['location'] = token_location
+            decimals = token_data.get('decimals')
+            identifier = [bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')]
+            wait_for_cache(identifier, f'Cached available balance for token: {token_symbol}', MetadataPointer.BALANCES)
+            token_balance = get_cached_available_balance(decimals=decimals, identifier=identifier)
+            entry['balance'] = token_balance
+            token_list_entries.append(entry)
+    account_tokens_list = order_account_tokens_list(token_list_entries, bytes.fromhex(blockchain_address))
+    key = cache_data_key(bytes.fromhex(blockchain_address), MetadataPointer.TOKEN_DATA_LIST)
+    cache_data(key, json.dumps(account_tokens_list))
+
+
+def get_active_token_symbol(blockchain_address: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    identifier = bytes.fromhex(blockchain_address)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_ACTIVE)
+    active_token_symbol = get_cached_data(key)
+    if not active_token_symbol:
+        raise CachedDataNotFoundError('No active token set.')
+    return active_token_symbol
+
+
+def get_cached_token_data(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    identifier = [bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')]
+    key = cache_data_key(identifier, MetadataPointer.TOKEN_DATA)
+    logg.debug(f'Retrieving token data for: {token_symbol} at: {key}')
+    token_data = get_cached_data(key)
+    return json.loads(token_data)
 
 
 def get_cached_default_token(chain_str: str) -> Optional[str]:
@@ -49,6 +145,132 @@ def get_default_token_symbol():
             raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
 
 
+def get_cached_token_symbol_list(blockchain_address: str) -> Optional[list]:
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(identifier=bytes.fromhex(blockchain_address), salt=MetadataPointer.TOKEN_SYMBOLS_LIST)
+    token_symbols_list = get_cached_data(key)
+    if token_symbols_list:
+        return json.loads(token_symbols_list)
+    return token_symbols_list
+
+
+def get_cached_token_data_list(blockchain_address: str) -> Optional[list]:
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(bytes.fromhex(blockchain_address), MetadataPointer.TOKEN_DATA_LIST)
+    token_data_list = get_cached_data(key)
+    if token_data_list:
+        return json.loads(token_data_list)
+    return token_data_list
+
+
+def handle_token_symbol_list(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    token_symbol_list = get_cached_token_symbol_list(blockchain_address)
+    if token_symbol_list:
+        if token_symbol not in token_symbol_list:
+            token_symbol_list.append(token_symbol)
+    else:
+        token_symbol_list = [token_symbol]
+
+    identifier = bytes.fromhex(blockchain_address)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_SYMBOLS_LIST)
+    data = json.dumps(token_symbol_list)
+    cache_data(key, data)
+
+
+def hashed_token_proof(token_proof: Union[dict, str]) -> str:
+    """
+    :param token_proof:
+    :type token_proof:
+    :return:
+    :rtype:
+    """
+    if isinstance(token_proof, dict):
+        token_proof = json.dumps(token_proof)
+    logg.debug(f'Hashing token proof: {token_proof}')
+    hash_object = hashlib.new("sha256")
+    hash_object.update(token_proof.encode('utf-8'))
+    return hash_object.digest().hex()
+
+
+def order_account_tokens_list(account_tokens_list: list, identifier: bytes) -> list:
+    """
+    :param account_tokens_list:
+    :type account_tokens_list:
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    ordered_tokens_list = []
+    # get last sent token
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_SENT)
+    last_sent_token_symbol = get_cached_data(key)
+
+    # get last received token
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_RECEIVED)
+    last_received_token_symbol = get_cached_data(key)
+
+    last_sent_token_data, remaining_accounts_token_list = remove_from_account_tokens_list(account_tokens_list, last_sent_token_symbol)
+    if last_sent_token_data:
+        ordered_tokens_list.append(last_sent_token_data[0])
+
+    last_received_token_data, remaining_accounts_token_list = remove_from_account_tokens_list(remaining_accounts_token_list, last_received_token_symbol)
+    if last_received_token_data:
+        ordered_tokens_list.append(last_received_token_data[0])
+
+    # order the by balance
+    ordered_by_balance = sorted(remaining_accounts_token_list, key=lambda d: d['balance'], reverse=True)
+    return ordered_tokens_list + ordered_by_balance
+
+
+def parse_token_list(account_token_list: list):
+    parsed_token_list = []
+    for i in range(len(account_token_list)):
+        token_symbol = account_token_list[i].get('symbol')
+        token_balance = account_token_list[i].get('balance')
+        token_data_repr = f'{i+1}. {token_symbol} {token_balance}'
+        parsed_token_list.append(token_data_repr)
+    return parsed_token_list
+
+
+def process_token_data(blockchain_address: str, token_symbol: str):
+    """
+
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'Processing token data for token: {token_symbol}')
+    identifier = token_symbol.encode('utf-8')
+    query_token_metadata(identifier=identifier)
+    token_info = query_token_info(identifier=identifier)
+    hashed_token_info = hashed_token_proof(token_proof=token_info)
+    query_token_data(blockchain_address=blockchain_address,
+                     hashed_proofs=[hashed_token_info],
+                     token_symbols=[token_symbol])
+
+
 def query_default_token(chain_str: str):
     """This function synchronously queries cic-eth for the deployed system's default token.
     :param chain_str: Chain name and network id.
@@ -60,3 +282,60 @@ def query_default_token(chain_str: str):
     cic_eth_api = Api(chain_str=chain_str)
     default_token_request_task = cic_eth_api.default_token()
     return default_token_request_task.get()
+
+
+def query_token_data(blockchain_address: str, hashed_proofs: list, token_symbols: list):
+    """"""
+    logg.debug(f'Retrieving token metadata for tokens: {", ".join(token_symbols)}')
+    api = Api(callback_param=blockchain_address,
+              callback_queue='cic-ussd',
+              chain_str=Chain.spec.__str__(),
+              callback_task='cic_ussd.tasks.callback_handler.token_data_callback')
+    api.tokens(token_symbols=token_symbols, proof=hashed_proofs)
+
+
+def remove_from_account_tokens_list(account_tokens_list: list, token_symbol: str):
+    """
+    :param account_tokens_list:
+    :type account_tokens_list:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    removed_token_data = []
+    for i in range(len(account_tokens_list)):
+        if account_tokens_list[i]['symbol'] == token_symbol:
+            removed_token_data.append(account_tokens_list[i])
+            del account_tokens_list[i]
+            break
+    return removed_token_data, account_tokens_list
+
+
+def set_active_token(blockchain_address: str, token_symbol: str):
+    """
+    :param blockchain_address:
+    :type blockchain_address:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    logg.info(f'Active token set to: {token_symbol}')
+    key = cache_data_key(identifier=bytes.fromhex(blockchain_address), salt=MetadataPointer.TOKEN_ACTIVE)
+    cache_data(key=key, data=token_symbol)
+
+
+def token_list_set(preferred_language: str, token_data_reprs: list):
+    """
+    :param preferred_language:
+    :type preferred_language:
+    :param token_data_reprs:
+    :type token_data_reprs:
+    :return:
+    :rtype:
+    """
+    if not token_data_reprs:
+        return translation_for('helpers.no_tokens_list', preferred_language)
+    return ''.join(f'{token_data_repr}\n' for token_data_repr in token_data_reprs)
+

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -1,7 +1,6 @@
 # standard import
-import decimal
-import json
 import logging
+from math import trunc
 from typing import Dict, Tuple
 
 # external import
@@ -9,8 +8,6 @@ from cic_eth.api import Api
 from sqlalchemy.orm.session import Session
 
 # local import
-from cic_ussd.account.chain import Chain
-from cic_ussd.account.tokens import get_cached_default_token
 from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.error import UnknownUssdRecipient
@@ -55,32 +52,32 @@ def aux_transaction_data(preferred_language: str, transaction: dict) -> dict:
     return transaction
 
 
-def from_wei(value: int) -> float:
+def from_wei(decimals: int, value: int) -> float:
     """This function converts values in Wei to a token in the cic network.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
     :param value: Value in Wei
     :type value: int
     :return: SRF equivalent of value in Wei
     :rtype: float
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
-    token_decimals: int = cached_token_data.get('decimals')
-    value = float(value) / (10**token_decimals)
+    value = float(value) / (10**decimals)
     return truncate(value=value, decimals=2)
 
 
-def to_wei(value: int) -> int:
+def to_wei(decimals: int, value: int) -> int:
     """This functions converts values from a token in the cic network to Wei.
+    :param decimals: The decimals required for wei values.
+    :type decimals: int
     :param value: Value in SRF
     :type value: int
     :return: Wei equivalent of value in SRF
     :rtype: int
     """
-    cached_token_data = json.loads(get_cached_default_token(Chain.spec.__str__()))
-    token_decimals: int = cached_token_data.get('decimals')
-    return int(value * (10**token_decimals))
+    return int(value * (10**decimals))
 
 
-def truncate(value: float, decimals: int):
+def truncate(value: float, decimals: int) -> float:
     """This function truncates a value to a specified number of decimals places.
     :param value: The value to be truncated.
     :type value: float
@@ -89,9 +86,8 @@ def truncate(value: float, decimals: int):
     :return: The truncated value.
     :rtype: int
     """
-    decimal.getcontext().rounding = decimal.ROUND_DOWN
-    contextualized_value = decimal.Decimal(value)
-    return round(contextualized_value, decimals)
+    stepper = 10.0**decimals
+    return trunc(stepper*value) / stepper
 
 
 def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
@@ -104,14 +100,17 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
     """
     destination_token_symbol = transaction.get('destination_token_symbol')
     destination_token_value = transaction.get('destination_token_value') or transaction.get('to_value')
+    destination_token_decimals = transaction.get('destination_token_decimals')
     recipient_blockchain_address = transaction.get('recipient')
     sender_blockchain_address = transaction.get('sender')
     source_token_symbol = transaction.get('source_token_symbol')
     source_token_value = transaction.get('source_token_value') or transaction.get('from_value')
+    source_token_decimals = transaction.get('source_token_decimals')
 
     recipient_transaction_data = {
         "token_symbol": destination_token_symbol,
         "token_value": destination_token_value,
+        "token_decimals": destination_token_decimals,
         "blockchain_address": recipient_blockchain_address,
         "role": "recipient",
     }
@@ -119,6 +118,7 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
         "blockchain_address": sender_blockchain_address,
         "token_symbol": source_token_symbol,
         "token_value": source_token_value,
+        "token_decimals": source_token_decimals,
         "role": "sender",
     }
     return recipient_transaction_data, sender_transaction_data
@@ -166,14 +166,16 @@ class OutgoingTransaction:
         self.from_address = from_address
         self.to_address = to_address
 
-    def transfer(self, amount: int, token_symbol: str):
+    def transfer(self, amount: int, decimals: int, token_symbol: str):
         """This function initiates standard transfers between one account to another
         :param amount: The amount of tokens to be sent
         :type amount: int
+        :param decimals: The decimals for the token being transferred.
+        :type decimals: int
         :param token_symbol: ERC20 token symbol of token to send
         :type token_symbol: str
         """
         self.cic_eth_api.transfer(from_address=self.from_address,
                                   to_address=self.to_address,
-                                  value=to_wei(value=amount),
+                                  value=to_wei(decimals=decimals, value=amount),
                                   token_symbol=token_symbol)

apps/cic-ussd/cic_ussd/cache.py

@@ -1,12 +1,13 @@
 # standard imports
 import hashlib
 import logging
+from typing import Union
 
 # external imports
 from cic_types.condiments import MetadataPointer
 from redis import Redis
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 class Cache:
@@ -39,7 +40,7 @@ def get_cached_data(key: str):
     return cache.get(name=key)
 
 
-def cache_data_key(identifier: bytes, salt: MetadataPointer):
+def cache_data_key(identifier: Union[list, bytes], salt: MetadataPointer):
     """
     :param identifier:
     :type identifier:
@@ -49,6 +50,10 @@ def cache_data_key(identifier: bytes, salt: MetadataPointer):
     :rtype:
     """
     hash_object = hashlib.new("sha256")
-    hash_object.update(identifier)
+    if isinstance(identifier, list):
+        for identity in identifier:
+            hash_object.update(identity)
+    else:
+        hash_object.update(identifier)
     hash_object.update(salt.value.encode(encoding="utf-8"))
     return hash_object.digest().hex()

apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py

@@ -24,6 +24,8 @@ def upgrade():
                     sa.Column('preferred_language', sa.String(), nullable=True),
                     sa.Column('password_hash', sa.String(), nullable=True),
                     sa.Column('failed_pin_attempts', sa.Integer(), nullable=False),
+                    sa.Column('guardians', sa.String(), nullable=True),
+                    sa.Column('guardian_quora', sa.Integer(), nullable=False),
                     sa.Column('status', sa.Integer(), nullable=False),
                     sa.Column('created', sa.DateTime(), nullable=False),
                     sa.Column('updated', sa.DateTime(), nullable=False),

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -4,6 +4,8 @@ import json
 # external imports
 from cic_eth.api import Api
 from cic_types.condiments import MetadataPointer
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.account.metadata import get_cached_preferred_language, parse_account_metadata
@@ -12,8 +14,9 @@ from cic_ussd.db.enum import AccountStatus
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.encoder import check_password_hash, create_password_hash
-from sqlalchemy import Column, Integer, String
-from sqlalchemy.orm.session import Session
+from cic_ussd.phone_number import Support
+
+support_phone = Support.phone_number
 
 
 class Account(SessionBase):
@@ -29,12 +32,16 @@ class Account(SessionBase):
     failed_pin_attempts = Column(Integer)
     status = Column(Integer)
     preferred_language = Column(String)
+    guardians = Column(String)
+    guardian_quora = Column(Integer)
 
     def __init__(self, blockchain_address, phone_number):
         self.blockchain_address = blockchain_address
         self.phone_number = phone_number
         self.password_hash = None
         self.failed_pin_attempts = 0
+        # self.guardians = f'{support_phone}' if support_phone else None
+        self.guardian_quora = 1
         self.status = AccountStatus.PENDING.value
 
     def __repr__(self):
@@ -45,6 +52,28 @@ class Account(SessionBase):
         self.failed_pin_attempts = 0
         self.status = AccountStatus.ACTIVE.value
 
+    def add_guardian(self, phone_number: str):
+        set_guardians = phone_number
+        if self.guardians:
+            set_guardians = self.guardians.split(',')
+            set_guardians.append(phone_number)
+            ','.join(set_guardians)
+        self.guardians = set_guardians
+
+    def remove_guardian(self, phone_number: str):
+        set_guardians = self.guardians.split(',')
+        set_guardians.remove(phone_number)
+        if len(set_guardians) > 1:
+            self.guardians = ','.join(set_guardians)
+        else:
+            self.guardians = set_guardians[0]
+
+    def get_guardians(self) -> list:
+        return self.guardians.split(',') if self.guardians else []
+
+    def set_guardian_quora(self, quora: int):
+        self.guardian_quora = quora
+
     def create_password(self, password):
         """This method takes a password value and hashes the value before assigning it to the corresponding
         `hashed_password` attribute in the user record.

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -287,7 +287,132 @@
             "display_key": "ussd.kenya.dob_edit_pin_authorization",
             "name": "dob_edit_pin_authorization",
             "parent": "metadata_management"
+        },
+        "49": {
+            "description": "Menu to display first set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.first_account_tokens_set",
+            "name": "first_account_tokens_set",
+            "parent": null
+        },
+        "50": {
+            "description": "Menu to display middle set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.middle_account_tokens_set",
+            "name": "middle_account_tokens_set",
+            "parent": null
+        },
+        "51": {
+            "description": "Menu to display last set of tokens in the account's token list.",
+            "display_key": "ussd.kenya.last_account_tokens_set",
+            "name": "last_account_tokens_set",
+            "parent": null
+        },
+        "52": {
+            "description": "Pin entry menu for setting an active token.",
+            "display_key": "ussd.kenya.token_selection_pin_authorization",
+            "name": "token_selection_pin_authorization",
+            "parent": null
+        },
+        "53": {
+            "description": "Exit following a successful active token setting.",
+            "display_key": "ussd.kenya.exit_successful_token_selection",
+            "name": "exit_successful_token_selection",
+            "parent": null
+        },
+        "54": {
+            "description": "Pin management menu for operations related to an account's pin.",
+            "display_key": "ussd.kenya.pin_management",
+            "name": "pin_management",
+            "parent": "start"
+        },
+        "55": {
+            "description": "Phone number entry for account whose pin is being reset.",
+            "display_key": "ussd.kenya.reset_guarded_pin",
+            "name": "reset_guarded_pin",
+            "parent": "pin_management"
+        },
+        "56": {
+            "description": "Pin entry for initiating request to reset an account's pin.",
+            "display_key": "ussd.kenya.reset_guarded_pin_authorization",
+            "name": "reset_guarded_pin_authorization",
+            "parent": "pin_management"
+        },
+        "57": {
+            "description": "Exit menu following successful pin reset initiation.",
+            "display_key": "ussd.kenya.exit_pin_reset_initiated_success",
+            "name": "exit_pin_reset_initiated_success",
+            "parent": "pin_management"
+        },
+        "58": {
+            "description": "Exit menu in the event that an account is not a set guardian.",
+            "display_key": "ussd.kenya.exit_not_authorized_for_pin_reset",
+            "name": "exit_not_authorized_for_pin_reset",
+            "parent": "pin_management"
+        },
+        "59": {
+            "description": "Pin guard menu for handling guardianship operations.",
+            "display_key": "ussd.kenya.guard_pin",
+            "name": "guard_pin",
+            "parent": "pin_management"
+        },
+        "60": {
+            "description": "Pin entry to display a list of set guardians.",
+            "display_key": "ussd.kenya.guardian_list_pin_authorization",
+            "name": "guardian_list_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "61": {
+            "description": "Menu to display list of set guardians.",
+            "display_key": "ussd.kenya.guardian_list",
+            "name": "guardian_list",
+            "parent": "guard_pin"
+        },
+        "62": {
+            "description": "Phone number entry to add an account as a guardian to reset pin.",
+            "display_key": "ussd.kenya.add_guardian",
+            "name": "add_guardian",
+            "parent": "guard_pin"
+        },
+        "63": {
+            "description": "Pin entry to confirm addition of an account as a guardian.",
+            "display_key": "ussd.kenya.add_guardian_pin_authorization",
+            "name": "add_guardian_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "64": {
+            "description": "Exit menu when an account is successfully added as pin reset guardian.",
+            "display_key": "ussd.kenya.exit_guardian_addition_success",
+            "name": "exit_guardian_addition_success",
+            "parent": "guard_pin"
+        },
+        "65": {
+            "description": "Phone number entry to remove an account as a guardian to reset pin.",
+            "display_key": "ussd.kenya.remove_guardian",
+            "name": "remove_guardian",
+            "parent": "guard_pin"
+        },
+        "66": {
+            "description": "Pin entry to confirm removal of an account as a guardian.",
+            "display_key": "ussd.kenya.remove_guardian_pin_authorization",
+            "name": "remove_guardian_pin_authorization",
+            "parent": "guard_pin"
+        },
+        "67": {
+            "description": "Exit menu when an account is successfully removed as pin reset guardian.",
+            "display_key": "ussd.kenya.exit_guardian_removal_success",
+            "name": "exit_guardian_removal_success",
+            "parent": "guard_pin"
+        },
+        "68": {
+            "description": "Exit menu when invalid phone number entry for guardian addition. ",
+            "display_key": "ussd.kenya.exit_invalid_guardian_addition",
+            "name": "exit_invalid_guardian_addition",
+            "parent": "guard_pin"
+        },
+        "69": {
+            "description": "Exit menu when invalid phone number entry for guardian removal. ",
+            "display_key": "ussd.kenya.exit_invalid_guardian_removal",
+            "name": "exit_invalid_guardian_removal",
+            "parent": "guard_pin"
         }
-
     }
 }

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -13,7 +13,7 @@ logg = logging.getLogger(__file__)
 
 
 class UssdMetadataHandler(MetadataRequestsHandler):
-    def __init__(self, cic_type: MetadataPointer, identifier: bytes):
+    def __init__(self, identifier: bytes, cic_type: MetadataPointer = None):
         super().__init__(cic_type, identifier)
 
     def cache_metadata(self, data: str):

apps/cic-ussd/cic_ussd/metadata/tokens.py

@@ -0,0 +1,54 @@
+# standard imports
+from typing import Dict, Optional
+
+# external imports
+import json
+
+from cic_types.condiments import MetadataPointer
+
+# local imports
+from .base import UssdMetadataHandler
+from cic_ussd.cache import cache_data
+from cic_ussd.error import MetadataNotFoundError
+
+
+class TokenMetadata(UssdMetadataHandler):
+    def __init__(self, identifier: bytes, **kwargs):
+        super(TokenMetadata, self).__init__(identifier=identifier, **kwargs)
+
+
+def token_metadata_handler(metadata_client: TokenMetadata) -> Optional[Dict]:
+    """
+    :param metadata_client:
+    :type metadata_client:
+    :return:
+    :rtype:
+    """
+    result = metadata_client.query()
+    token_metadata = result.json()
+    if not token_metadata:
+        raise MetadataNotFoundError(f'No metadata found at: {metadata_client.metadata_pointer} for: {metadata_client.identifier.decode("utf-8")}')
+    cache_data(metadata_client.metadata_pointer, json.dumps(token_metadata))
+    return token_metadata
+
+
+def query_token_metadata(identifier: bytes):
+    """
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    token_metadata_client = TokenMetadata(identifier=identifier, cic_type=MetadataPointer.TOKEN_META_SYMBOL)
+    return token_metadata_handler(token_metadata_client)
+
+
+def query_token_info(identifier: bytes):
+    """
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    token_info_client = TokenMetadata(identifier=identifier, cic_type=MetadataPointer.TOKEN_PROOF_SYMBOL)
+    return token_metadata_handler(token_info_client)

apps/cic-ussd/cic_ussd/phone_number.py

@@ -5,7 +5,6 @@ from typing import Optional
 import phonenumbers
 
 # local imports
-from cic_ussd.db.models.account import Account
 
 
 class E164Format:

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -9,6 +9,7 @@ from cic_types.condiments import MetadataPointer
 
 # local imports
 from cic_ussd.account.balance import (calculate_available_balance,
+                                      get_account_tokens_balance,
                                       get_adjusted_balance,
                                       get_balances,
                                       get_cached_adjusted_balance,
@@ -21,13 +22,20 @@ from cic_ussd.account.statement import (
     query_statement,
     statement_transaction_set
 )
-from cic_ussd.account.tokens import get_default_token_symbol
+from cic_ussd.account.tokens import (create_account_tokens_list,
+                                     get_active_token_symbol,
+                                     get_cached_token_data,
+                                     get_cached_token_symbol_list,
+                                     get_cached_token_data_list,
+                                     parse_token_list,
+                                     token_list_set)
 from cic_ussd.account.transaction import from_wei, to_wei
 from cic_ussd.cache import cache_data_key, cache_data
 from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
 from cic_ussd.processor.util import parse_person_metadata
+from cic_ussd.session.ussd_session import save_session_data
 from cic_ussd.translation import translation_for
 from sqlalchemy.orm.session import Session
 
@@ -48,22 +56,27 @@ class MenuProcessor:
         :return:
         :rtype:
         """
-        available_balance = get_cached_available_balance(self.account.blockchain_address)
+
         adjusted_balance = get_cached_adjusted_balance(self.identifier)
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
         with_available_balance = f'{self.display_key}.available_balance'
         with_fees = f'{self.display_key}.with_fees'
+        decimals = token_data.get('decimals')
+        available_balance = get_cached_available_balance(decimals, [self.identifier, token_symbol.encode('utf-8')])
         if not adjusted_balance:
             return translation_for(key=with_available_balance,
                                    preferred_language=preferred_language,
                                    available_balance=available_balance,
                                    token_symbol=token_symbol)
+
         adjusted_balance = json.loads(adjusted_balance)
-        tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
-        tax = from_wei(int(tax_wei))
+
+        tax_wei = to_wei(decimals, int(available_balance)) - int(adjusted_balance)
+        tax = from_wei(decimals, int(tax_wei))
         return translation_for(key=with_fees,
                                preferred_language=preferred_language,
                                available_balance=available_balance,
@@ -76,21 +89,25 @@ class MenuProcessor:
         :rtype:
         """
         cached_statement = get_cached_statement(self.account.blockchain_address)
-        statement = json.loads(cached_statement)
-        statement_transactions = parse_statement_transactions(statement)
-        transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
+        transaction_sets = []
+        if cached_statement:
+            statement = json.loads(cached_statement)
+            statement_transactions = parse_statement_transactions(statement)
+            transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        first_transaction_set = []
-        middle_transaction_set = []
-        last_transaction_set = []
+        no_transaction_history = statement_transaction_set(preferred_language, transaction_sets)
+        first_transaction_set = no_transaction_history
+        middle_transaction_set = no_transaction_history
+        last_transaction_set = no_transaction_history
         if transaction_sets:
             first_transaction_set = statement_transaction_set(preferred_language, transaction_sets[0])
         if len(transaction_sets) >= 2:
             middle_transaction_set = statement_transaction_set(preferred_language, transaction_sets[1])
         if len(transaction_sets) >= 3:
             last_transaction_set = statement_transaction_set(preferred_language, transaction_sets[2])
+
         if self.display_key == 'ussd.kenya.first_transaction_set':
             return translation_for(
                 self.display_key, preferred_language, first_transaction_set=first_transaction_set
@@ -104,7 +121,63 @@ class MenuProcessor:
                 self.display_key, preferred_language, last_transaction_set=last_transaction_set
             )
 
-    def help(self):
+    def add_guardian_pin_authorization(self):
+        guardian_information = self.guardian_metadata()
+        return self.pin_authorization(guardian_information=guardian_information)
+
+    def guardian_list(self):
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        set_guardians = self.account.get_guardians()
+        if set_guardians:
+            guardians_list = ''
+            guardians_list_header = translation_for('helpers.guardians_list_header', preferred_language)
+            for phone_number in set_guardians:
+                guardian = Account.get_by_phone_number(phone_number, self.session)
+                guardian_information = guardian.standard_metadata_id()
+                guardians_list += f'{guardian_information}\n'
+            guardians_list = guardians_list_header + '\n' + guardians_list
+        else:
+            guardians_list = translation_for('helpers.no_guardians_list', preferred_language)
+        return translation_for(self.display_key, preferred_language, guardians_list=guardians_list)
+
+    def account_tokens(self) -> str:
+        cached_token_data_list = get_cached_token_data_list(self.account.blockchain_address)
+        token_data_list = parse_token_list(cached_token_data_list)
+        token_list_sets = [token_data_list[tds:tds + 3] for tds in range(0, len(token_data_list), 3)]
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        no_token_list = token_list_set(preferred_language, [])
+        first_account_tokens_set = no_token_list
+        middle_account_tokens_set = no_token_list
+        last_account_tokens_set = no_token_list
+        if token_list_sets:
+            data = {
+                'account_tokens_list': cached_token_data_list
+            }
+            save_session_data(data=data, queue='cic-ussd', session=self.session, ussd_session=self.ussd_session)
+            first_account_tokens_set = token_list_set(preferred_language, token_list_sets[0])
+
+        if len(token_list_sets) >= 2:
+            middle_account_tokens_set = token_list_set(preferred_language, token_list_sets[1])
+        if len(token_list_sets) >= 3:
+            last_account_tokens_set = token_list_set(preferred_language, token_list_sets[2])
+        if self.display_key == 'ussd.kenya.first_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, first_account_tokens_set=first_account_tokens_set
+            )
+        if self.display_key == 'ussd.kenya.middle_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, middle_account_tokens_set=middle_account_tokens_set
+            )
+        if self.display_key == 'ussd.kenya.last_account_tokens_set':
+            return translation_for(
+                self.display_key, preferred_language, last_account_tokens_set=last_account_tokens_set
+            )
+
+    def help(self) -> str:
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -155,30 +228,48 @@ class MenuProcessor:
             f'{self.display_key}.retry', preferred_language, retry_pin_entry=retry_pin_entry
         )
 
+    def guarded_account_metadata(self):
+        guarded_account_phone_number = self.ussd_session.get('data').get('guarded_account_phone_number')
+        guarded_account = Account.get_by_phone_number(guarded_account_phone_number, self.session)
+        return guarded_account.standard_metadata_id()
+
+    def guardian_metadata(self):
+        guardian_phone_number = self.ussd_session.get('data').get('guardian_phone_number')
+        guardian = Account.get_by_phone_number(guardian_phone_number, self.session)
+        return guardian.standard_metadata_id()
+
+    def reset_guarded_pin_authorization(self):
+        guarded_account_information = self.guarded_account_metadata()
+        return self.pin_authorization(guarded_account_information=guarded_account_information)
+
     def start_menu(self):
         """
         :return:
         :rtype:
         """
         chain_str = Chain.spec.__str__()
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
         blockchain_address = self.account.blockchain_address
         balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0]
-        key = cache_data_key(self.identifier, MetadataPointer.BALANCES)
+        key = cache_data_key([self.identifier, token_symbol.encode('utf-8')], MetadataPointer.BALANCES)
         cache_data(key, json.dumps(balances))
-        available_balance = calculate_available_balance(balances)
+        available_balance = calculate_available_balance(balances, decimals)
         now = datetime.now()
         if (now - self.account.created).days >= 30:
             if available_balance <= 0:
                 logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.')
             else:
                 timestamp = int((now - timedelta(30)).timestamp())
-                adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol)
-                key = cache_data_key(self.identifier, MetadataPointer.BALANCES_ADJUSTED)
+                adjusted_balance = get_adjusted_balance(to_wei(decimals, int(available_balance)), chain_str, timestamp, token_symbol)
+                key = cache_data_key([self.identifier, token_symbol.encode('utf-8')], MetadataPointer.BALANCES_ADJUSTED)
                 cache_data(key, json.dumps(adjusted_balance))
 
         query_statement(blockchain_address)
-
+        token_symbols_list = get_cached_token_symbol_list(blockchain_address)
+        get_account_tokens_balance(blockchain_address, chain_str, token_symbols_list)
+        create_account_tokens_list(blockchain_address)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -186,6 +277,20 @@ class MenuProcessor:
             self.display_key, preferred_language, account_balance=available_balance, account_token_name=token_symbol
         )
 
+    def token_selection_pin_authorization(self) -> str:
+        """
+        :return:
+        :rtype:
+        """
+        selected_token = self.ussd_session.get('data').get('selected_token')
+        token_name = selected_token.get('name')
+        token_symbol = selected_token.get('symbol')
+        token_issuer = selected_token.get('issuer')
+        token_contact = selected_token.get('contact')
+        token_location = selected_token.get('location')
+        token_data = f'{token_name} ({token_symbol})\n{token_issuer}\n{token_contact}\n{token_location}\n'
+        return self.pin_authorization(token_data=token_data)
+
     def transaction_pin_authorization(self) -> str:
         """
         :return:
@@ -195,36 +300,81 @@ class MenuProcessor:
         recipient = Account.get_by_phone_number(recipient_phone_number, self.session)
         tx_recipient_information = recipient.standard_metadata_id()
         tx_sender_information = self.account.standard_metadata_id()
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
         user_input = self.ussd_session.get('data').get('transaction_amount')
-        transaction_amount = to_wei(value=int(user_input))
+        decimals = token_data.get('decimals')
+        transaction_amount = to_wei(decimals=decimals, value=int(user_input))
         return self.pin_authorization(
             recipient_information=tx_recipient_information,
-            transaction_amount=from_wei(transaction_amount),
+            transaction_amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             sender_information=tx_sender_information
         )
 
+    def exit_guardian_addition_success(self) -> str:
+        guardian_information = self.guardian_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guardian_information=guardian_information)
+
+    def exit_guardian_removal_success(self):
+        guardian_information = self.guardian_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guardian_information=guardian_information)
+
+    def exit_invalid_guardian_addition(self):
+        failure_reason = self.ussd_session.get('data').get('failure_reason')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key, preferred_language, error_exit=failure_reason)
+
+    def exit_invalid_guardian_removal(self):
+        failure_reason = self.ussd_session.get('data').get('failure_reason')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key, preferred_language, error_exit=failure_reason)
+
+    def exit_pin_reset_initiated_success(self):
+        guarded_account_information = self.guarded_account_metadata()
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,
+                               preferred_language,
+                               guarded_account_information=guarded_account_information)
+
     def exit_insufficient_balance(self):
         """
         :return:
         :rtype:
         """
-        available_balance = get_cached_available_balance(self.account.blockchain_address)
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
         session_data = self.ussd_session.get('data')
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
+        available_balance = get_cached_available_balance(decimals, [self.identifier, token_symbol.encode('utf-8')])
         transaction_amount = session_data.get('transaction_amount')
-        transaction_amount = to_wei(value=int(transaction_amount))
-        token_symbol = get_default_token_symbol()
+        transaction_amount = to_wei(decimals=decimals, value=int(transaction_amount))
         recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number')
         recipient = Account.get_by_phone_number(recipient_phone_number, self.session)
         tx_recipient_information = recipient.standard_metadata_id()
         return translation_for(
             self.display_key,
             preferred_language,
-            amount=from_wei(transaction_amount),
+            amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             recipient_information=tx_recipient_information,
             token_balance=available_balance
@@ -242,6 +392,14 @@ class MenuProcessor:
             preferred_language = i18n.config.get('fallback')
         return translation_for('ussd.kenya.exit_pin_blocked', preferred_language, support_phone=Support.phone_number)
 
+    def exit_successful_token_selection(self) -> str:
+        selected_token = self.ussd_session.get('data').get('selected_token')
+        token_symbol = selected_token.get('symbol')
+        preferred_language = get_cached_preferred_language(self.account.blockchain_address)
+        if not preferred_language:
+            preferred_language = i18n.config.get('fallback')
+        return translation_for(self.display_key,preferred_language,token_symbol=token_symbol)
+
     def exit_successful_transaction(self):
         """
         :return:
@@ -251,8 +409,10 @@ class MenuProcessor:
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        transaction_amount = to_wei(amount)
-        token_symbol = get_default_token_symbol()
+        token_symbol = get_active_token_symbol(self.account.blockchain_address)
+        token_data = get_cached_token_data(self.account.blockchain_address, token_symbol)
+        decimals = token_data.get('decimals')
+        transaction_amount = to_wei(decimals, amount)
         recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number')
         recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=self.session)
         tx_recipient_information = recipient.standard_metadata_id()
@@ -260,7 +420,7 @@ class MenuProcessor:
         return translation_for(
             self.display_key,
             preferred_language,
-            transaction_amount=from_wei(transaction_amount),
+            transaction_amount=from_wei(decimals, transaction_amount),
             token_symbol=token_symbol,
             recipient_information=tx_recipient_information,
             sender_information=tx_sender_information
@@ -294,15 +454,42 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if menu_name == 'transaction_pin_authorization':
         return menu_processor.transaction_pin_authorization()
 
+    if menu_name == 'token_selection_pin_authorization':
+        return menu_processor.token_selection_pin_authorization()
+
     if menu_name == 'exit_insufficient_balance':
         return menu_processor.exit_insufficient_balance()
 
+    if menu_name == 'exit_invalid_guardian_addition':
+        return menu_processor.exit_invalid_guardian_addition()
+
+    if menu_name == 'exit_invalid_guardian_removal':
+        return menu_processor.exit_invalid_guardian_removal()
+
     if menu_name == 'exit_successful_transaction':
         return menu_processor.exit_successful_transaction()
 
+    if menu_name == 'exit_guardian_addition_success':
+        return menu_processor.exit_guardian_addition_success()
+
+    if menu_name == 'exit_guardian_removal_success':
+        return menu_processor.exit_guardian_removal_success()
+
+    if menu_name == 'exit_pin_reset_initiated_success':
+        return menu_processor.exit_pin_reset_initiated_success()
+
     if menu_name == 'account_balances':
         return menu_processor.account_balances()
 
+    if menu_name == 'guardian_list':
+        return menu_processor.guardian_list()
+
+    if menu_name == 'add_guardian_pin_authorization':
+        return menu_processor.add_guardian_pin_authorization()
+
+    if menu_name == 'reset_guarded_pin_authorization':
+        return menu_processor.reset_guarded_pin_authorization()
+
     if 'pin_authorization' in menu_name:
         return menu_processor.pin_authorization()
 
@@ -312,6 +499,9 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if 'transaction_set' in menu_name:
         return menu_processor.account_statement()
 
+    if 'account_tokens_set' in menu_name:
+        return menu_processor.account_tokens()
+
     if menu_name == 'display_user_metadata':
         return menu_processor.person_metadata()
 
@@ -321,6 +511,9 @@ def response(account: Account, display_key: str, menu_name: str, session: Sessio
     if menu_name == 'exit_pin_blocked':
         return menu_processor.exit_pin_blocked()
 
+    if menu_name == 'exit_successful_token_selection':
+        return menu_processor.exit_successful_token_selection()
+
     preferred_language = get_cached_preferred_language(account.blockchain_address)
 
     return translation_for(display_key, preferred_language)

apps/cic-ussd/cic_ussd/processor/util.py

@@ -1,15 +1,22 @@
 # standard imports
 import datetime
 import json
+import logging
+import time
+from typing import Union
 
 # external imports
+from cic_types.condiments import MetadataPointer
 from cic_types.models.person import get_contact_data_from_vcard
 from tinydb.table import Document
 
 # local imports
+from cic_ussd.cache import cache_data_key, get_cached_data
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.translation import translation_for
 
+logg = logging.getLogger(__file__)
+
 
 def latest_input(user_input: str) -> str:
     """
@@ -76,3 +83,66 @@ def resume_last_ussd_session(last_state: str) -> Document:
     if last_state in non_reusable_states:
         return UssdMenu.find_by_name('start')
     return UssdMenu.find_by_name(last_state)
+
+
+def wait_for_cache(identifier: Union[list, bytes], resource_name: str, salt: MetadataPointer, interval: int = 1, max_retry: int = 5):
+    """
+    :param identifier:
+    :type identifier:
+    :param interval:
+    :type interval:
+    :param resource_name:
+    :type resource_name:
+    :param salt:
+    :type salt:
+    :param max_retry:
+    :type max_retry:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(identifier=identifier, salt=salt)
+    resource = get_cached_data(key)
+    counter = 0
+    while resource is None:
+        logg.debug(f'Waiting for: {resource_name} at: {key}. Checking after: {interval} ...')
+        time.sleep(interval)
+        counter += 1
+        resource = get_cached_data(key)
+        if resource is not None:
+            logg.debug(f'{resource_name} now available.')
+            break
+        else:
+            if counter == max_retry:
+                logg.debug(f'Could not find: {resource_name} within: {max_retry}')
+                break
+
+
+def wait_for_session_data(resource_name: str, session_data_key: str, ussd_session: dict, interval: int = 1, max_retry: int = 5):
+    """
+    :param interval:
+    :type interval:
+    :param resource_name:
+    :type resource_name:
+    :param session_data_key:
+    :type session_data_key:
+    :param ussd_session:
+    :type ussd_session:
+    :param max_retry:
+    :type max_retry:
+    :return:
+    :rtype:
+    """
+    session_data = ussd_session.get('data').get(session_data_key)
+    counter = 0
+    while session_data is None:
+        logg.debug(f'Waiting for: {resource_name}. Checking after: {interval} ...')
+        time.sleep(interval)
+        counter += 1
+        session_data = ussd_session.get('data').get(session_data_key)
+        if session_data is not None:
+            logg.debug(f'{resource_name} now available.')
+            break
+        else:
+            if counter == max_retry:
+                logg.debug(f'Could not find: {resource_name} within: {max_retry}')
+                break

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -63,7 +63,6 @@ elif ssl == 0:
 else:
     ssl = True
 
-valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
 def main():
 
     # TODO: improve url building 
@@ -79,7 +78,7 @@ def main():
     session = uuid.uuid4().hex
     data = {
             'sessionId': session,
-            'serviceCode': valid_service_codes[0],
+            'serviceCode': config.get('USSD_SERVICE_CODE'),
             'phoneNumber': args.phone,
             'text': "",
         }

apps/cic-ussd/cic_ussd/session/ussd_session.py

@@ -13,7 +13,7 @@ from cic_ussd.cache import Cache
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession as DbUssdSession
 
-logg = logging.getLogger()
+logg = logging.getLogger(__file__)
 
 
 class UssdSession:
@@ -239,11 +239,16 @@ def save_session_data(queue: Optional[str], session: Session, data: dict, ussd_s
     :param ussd_session: A ussd session passed to the state machine.
     :type ussd_session: UssdSession
     """
+    logg.debug(f'Saving: {data} session data to: {ussd_session}')
     cache = Cache.store
     external_session_id = ussd_session.get('external_session_id')
     existing_session_data = ussd_session.get('data')
     if existing_session_data:
-        data = {**existing_session_data, **data}
+        # replace session data entry
+        keys = data.keys()
+        for key in keys:
+            existing_session_data[key] = data[key]
+        data = existing_session_data
     in_redis_ussd_session = cache.get(external_session_id)
     in_redis_ussd_session = json.loads(in_redis_ussd_session)
     create_or_update_session(

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -81,6 +81,18 @@ def menu_six_selected(state_machine_data: Tuple[str, dict, Account, Session]) ->
     return user_input == '6'
 
 
+def menu_nine_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '6'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '6'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '9'
+
+
 def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """
     This function checks that user input matches a string with value '00'
@@ -93,6 +105,30 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account, Sessio
     return user_input == '00'
 
 
+def menu_eleven_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '11'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '99'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '11'
+
+
+def menu_twenty_two_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
+    """
+    This function checks that user input matches a string with value '22'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '99'
+    :rtype: bool
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    return user_input == '22'
+
+
 def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """
     This function checks that user input matches a string with value '99'

apps/cic-ussd/cic_ussd/state_machine/logic/pin.py

@@ -3,7 +3,6 @@ user's pin.
 """
 
 # standard imports
-import json
 import logging
 import re
 from typing import Tuple
@@ -16,6 +15,7 @@ from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.enum import AccountStatus
 from cic_ussd.encoder import create_password_hash, check_password_hash
+from cic_ussd.processor.util import wait_for_session_data
 from cic_ussd.session.ussd_session import create_or_update_session, persist_ussd_session
 
 
@@ -31,11 +31,8 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    pin_is_valid = False
     matcher = r'^\d{4}$'
-    if re.match(matcher, user_input):
-        pin_is_valid = True
-    return pin_is_valid
+    return bool(re.match(matcher, user_input))
 
 
 def is_authorized_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
@@ -68,7 +65,7 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Accoun
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
     initial_pin = create_password_hash(user_input)
     if ussd_session.get('data'):
         data = ussd_session.get('data')
@@ -97,7 +94,8 @@ def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
+    wait_for_session_data('Initial pin', session_data_key='initial_pin', ussd_session=ussd_session)
     initial_pin = ussd_session.get('data').get('initial_pin')
     return check_password_hash(user_input, initial_pin)
 
@@ -107,11 +105,12 @@ def complete_pin_change(state_machine_data: Tuple[str, dict, Account, Session]):
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user, session = state_machine_data
+    user_input, ussd_session, account, session = state_machine_data
     session = SessionBase.bind_session(session=session)
+    wait_for_session_data('Initial pin', session_data_key='initial_pin', ussd_session=ussd_session)
     password_hash = ussd_session.get('data').get('initial_pin')
-    user.password_hash = password_hash
-    session.add(user)
+    account.password_hash = password_hash
+    session.add(account)
     session.flush()
     SessionBase.release_session(session=session)
 
@@ -134,6 +133,6 @@ def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account, Session]) ->
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user, session = state_machine_data
-    is_old_pin = user.verify_password(password=user_input)
+    user_input, ussd_session, account, session = state_machine_data
+    is_old_pin = account.verify_password(password=user_input)
     return is_valid_pin(state_machine_data=state_machine_data) and not is_old_pin

apps/cic-ussd/cic_ussd/state_machine/logic/pin_guard.py

@@ -0,0 +1,200 @@
+# standard imports
+import logging
+from typing import Tuple
+
+# external imports
+import celery
+import i18n
+from phonenumbers.phonenumberutil import NumberParseException
+from sqlalchemy.orm.session import Session
+
+# local imports
+from cic_ussd.account.metadata import get_cached_preferred_language
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.phone_number import process_phone_number, E164Format
+from cic_ussd.session.ussd_session import save_session_data
+from cic_ussd.translation import translation_for
+
+
+logg = logging.getLogger(__file__)
+
+
+def save_guardian_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data') or {}
+    guardian_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    session_data['guardian_phone_number'] = guardian_phone_number
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+
+
+def save_guarded_account_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data') or {}
+    guarded_account_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    session_data['guarded_account_phone_number'] = guarded_account_phone_number
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+
+
+def retrieve_person_metadata(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = process_phone_number(user_input, E164Format.region)
+    guardian = Account.get_by_phone_number(guardian_phone_number, session)
+    blockchain_address = guardian.blockchain_address
+    s_query_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address], queue='cic-ussd')
+    s_query_person_metadata.apply_async()
+
+
+def is_valid_guardian_addition(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    try:
+        phone_number = process_phone_number(user_input, E164Format.region)
+    except NumberParseException:
+        phone_number = None
+
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+
+    is_valid_account = Account.get_by_phone_number(phone_number, session) is not None
+    is_initiator = phone_number == account.phone_number
+    is_existent_guardian = phone_number in account.get_guardians()
+
+    failure_reason = ''
+    if not is_valid_account:
+        failure_reason = translation_for('helpers.error.no_matching_account', preferred_language)
+
+    if is_initiator:
+        failure_reason = translation_for('helpers.error.is_initiator', preferred_language)
+
+    if is_existent_guardian:
+        failure_reason = translation_for('helpers.error.is_existent_guardian', preferred_language)
+
+    if failure_reason:
+        session_data = ussd_session.get('data') or {}
+        session_data['failure_reason'] = failure_reason
+        save_session_data('cic-ussd', session, session_data, ussd_session)
+
+    return phone_number is not None and is_valid_account and not is_existent_guardian and not is_initiator
+
+
+def add_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = ussd_session.get('data').get('guardian_phone_number')
+    account.add_guardian(guardian_phone_number)
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+
+def is_set_pin_guardian(account: Account, checked_number: str, preferred_language: str, session: Session, ussd_session: dict):
+    """"""
+    failure_reason = ''
+    set_guardians = []
+    if account:
+        set_guardians = account.get_guardians()
+    else:
+        failure_reason = translation_for('helpers.error.no_matching_account', preferred_language)
+
+    is_set_guardian = checked_number in set_guardians
+    is_initiator = checked_number == account.phone_number
+
+    if not is_set_guardian:
+        failure_reason = translation_for('helpers.error.is_not_existent_guardian', preferred_language)
+
+    if is_initiator:
+        failure_reason = translation_for('helpers.error.is_initiator', preferred_language)
+
+    if failure_reason:
+        session_data = ussd_session.get('data') or {}
+        session_data['failure_reason'] = failure_reason
+        save_session_data('cic-ussd', session, session_data, ussd_session)
+
+    return is_set_guardian and not is_initiator
+
+
+def is_dialers_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, account, session = state_machine_data
+    phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+    return is_set_pin_guardian(account, phone_number, preferred_language, session, ussd_session)
+
+
+def is_others_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, account, session = state_machine_data
+    preferred_language = get_cached_preferred_language(account.blockchain_address)
+    phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    guarded_account = Account.get_by_phone_number(phone_number, session)
+    if not preferred_language:
+        preferred_language = i18n.config.get('fallback')
+    return is_set_pin_guardian(guarded_account, account.phone_number, preferred_language, session, ussd_session)
+
+
+def remove_pin_guardian(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    guardian_phone_number = ussd_session.get('data').get('guardian_phone_number')
+    account.remove_guardian(guardian_phone_number)
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+
+def initiate_pin_reset(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data')
+    quorum_count = session_data['quorum_count'] if session_data.get('quorum_count') else 0
+    quorum_count += 1
+    session_data['quorum_count'] = quorum_count
+    save_session_data('cic-ussd', session, session_data, ussd_session)
+    guarded_account_phone_number = session_data.get('guarded_account_phone_number')
+    guarded_account = Account.get_by_phone_number(guarded_account_phone_number, session)
+    if quorum_count >= guarded_account.guardian_quora:
+        guarded_account.reset_pin(session)
+        logg.debug(f'Reset initiated for: {guarded_account.phone_number}')
+        session_data['quorum_count'] = 0
+        save_session_data('cic-ussd', session, session_data, ussd_session)

apps/cic-ussd/cic_ussd/state_machine/logic/sms.py

@@ -6,7 +6,7 @@ from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.account.metadata import get_cached_preferred_language
-from cic_ussd.account.tokens import get_default_token_symbol
+from cic_ussd.account.tokens import get_active_token_symbol
 from cic_ussd.db.models.account import Account
 from cic_ussd.notifications import Notifier
 from cic_ussd.phone_number import Support
@@ -18,7 +18,7 @@ def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account,
     notifier = Notifier()
     phone_number = ussd_session.get('data')['recipient_phone_number']
     preferred_language = get_cached_preferred_language(account.blockchain_address)
-    token_symbol = get_default_token_symbol()
+    token_symbol = get_active_token_symbol(account.blockchain_address)
     tx_sender_information = account.standard_metadata_id()
     notifier.send_sms_notification('sms.upsell_unregistered_recipient',
                                    phone_number,

apps/cic-ussd/cic_ussd/state_machine/logic/tokens.py

@@ -0,0 +1,69 @@
+# standard imports
+from typing import Tuple
+
+# external imports
+from sqlalchemy.orm.session import Session
+
+# local imports
+from cic_ussd.account.tokens import set_active_token
+from cic_ussd.db.models.account import Account
+from cic_ussd.processor.util import wait_for_session_data
+from cic_ussd.session.ussd_session import save_session_data
+
+
+def is_valid_token_selection(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    session_data = ussd_session.get('data')
+    account_tokens_list = session_data.get('account_tokens_list')
+    if not account_tokens_list:
+        wait_for_session_data('Account token list', session_data_key='account_tokens_list', ussd_session=ussd_session)
+    if user_input not in ['00', '22']:
+        try:
+            user_input = int(user_input)
+            return user_input <= len(account_tokens_list)
+        except ValueError:
+            user_input = user_input.upper()
+            return any(token_data['symbol'] == user_input for token_data in account_tokens_list)
+
+
+def process_token_selection(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    account_tokens_list = ussd_session.get('data').get('account_tokens_list')
+    try:
+        user_input = int(user_input)
+        selected_token = account_tokens_list[user_input-1]
+    except ValueError:
+        user_input = user_input.upper()
+        selected_token = next(token_data for token_data in account_tokens_list if token_data['symbol'] == user_input)
+    data = {
+        'selected_token': selected_token
+    }
+    save_session_data(queue='cic-ussd', session=session, data=data, ussd_session=ussd_session)
+
+
+def set_selected_active_token(state_machine_data: Tuple[str, dict, Account, Session]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, account, session = state_machine_data
+    wait_for_session_data(resource_name='Selected token', session_data_key='selected_token', ussd_session=ussd_session)
+    selected_token = ussd_session.get('data').get('selected_token')
+    token_symbol = selected_token.get('symbol')
+    set_active_token(blockchain_address=account.blockchain_address, token_symbol=token_symbol)
+
+

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -5,18 +5,17 @@ from typing import Tuple
 # third party imports
 import celery
 from phonenumbers.phonenumberutil import NumberParseException
+from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.account.balance import get_cached_available_balance
 from cic_ussd.account.chain import Chain
-from cic_ussd.account.tokens import get_default_token_symbol
+from cic_ussd.account.tokens import get_active_token_symbol, get_cached_token_data
 from cic_ussd.account.transaction import OutgoingTransaction
-from cic_ussd.db.enum import AccountStatus
 from cic_ussd.db.models.account import Account
-from cic_ussd.db.models.base import SessionBase
 from cic_ussd.phone_number import process_phone_number, E164Format
 from cic_ussd.session.ussd_session import save_session_data
-from sqlalchemy.orm.session import Session
+
 
 logg = logging.getLogger(__file__)
 
@@ -63,7 +62,11 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account, Session
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    return int(user_input) <= get_cached_available_balance(account.blockchain_address)
+    identifier = bytes.fromhex(account.blockchain_address)
+    token_symbol = get_active_token_symbol(account.blockchain_address)
+    token_data = get_cached_token_data(account.blockchain_address, token_symbol)
+    decimals = token_data.get('decimals')
+    return int(user_input) <= get_cached_available_balance(decimals, [identifier, token_symbol.encode('utf-8')])
 
 
 def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
@@ -122,9 +125,10 @@ def process_transaction_request(state_machine_data: Tuple[str, dict, Account, Se
     to_address = recipient.blockchain_address
     from_address = account.blockchain_address
     amount = int(ussd_session.get('data').get('transaction_amount'))
-    token_symbol = get_default_token_symbol()
-
+    token_symbol = get_active_token_symbol(account.blockchain_address)
+    token_data = get_cached_token_data(account.blockchain_address, token_symbol)
+    decimals = token_data.get('decimals')
     outgoing_tx_processor = OutgoingTransaction(chain_str=chain_str,
                                                 from_address=from_address,
                                                 to_address=to_address)
-    outgoing_tx_processor.transfer(amount=amount, token_symbol=token_symbol)
+    outgoing_tx_processor.transfer(amount=amount, decimals=decimals, token_symbol=token_symbol)

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -1,4 +1,5 @@
 # standard imports
+
 import json
 import logging
 from datetime import timedelta
@@ -7,7 +8,6 @@ from datetime import timedelta
 import celery
 from cic_types.condiments import MetadataPointer
 
-
 # local imports
 from cic_ussd.account.balance import get_balances, calculate_available_balance
 from cic_ussd.account.statement import generate
@@ -15,8 +15,15 @@ from cic_ussd.cache import Cache, cache_data, cache_data_key, get_cached_data
 from cic_ussd.account.chain import Chain
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.account import Account
+from cic_ussd.processor.util import wait_for_cache
 from cic_ussd.account.statement import filter_statement_transactions
 from cic_ussd.account.transaction import transaction_actors
+from cic_ussd.account.tokens import (collate_token_metadata,
+                                     get_cached_token_data,
+                                     get_default_token_symbol,
+                                     handle_token_symbol_list,
+                                     process_token_data,
+                                     set_active_token)
 from cic_ussd.error import AccountCreationDataNotFound
 from cic_ussd.tasks.base import CriticalSQLAlchemyTask
 
@@ -58,6 +65,9 @@ def account_creation_callback(self, result: str, url: str, status_code: int):
     session.close()
     logg.debug(f'recorded account with identifier: {result}')
 
+    token_symbol = get_default_token_symbol()
+    set_active_token(blockchain_address=result, token_symbol=token_symbol)
+
     queue = self.request.delivery_info.get('routing_key')
     s_phone_pointer = celery.signature(
         'cic_ussd.tasks.metadata.add_phone_pointer', [result, phone_number], queue=queue
@@ -88,8 +98,16 @@ def balances_callback(result: list, param: str, status_code: int):
         raise ValueError(f'Unexpected status code: {status_code}.')
 
     balances = result[0]
-    identifier = bytes.fromhex(param)
-    key = cache_data_key(identifier, MetadataPointer.BALANCES)
+    identifier = []
+    param = param.split(',')
+    for identity in param:
+        try:
+            i = bytes.fromhex(identity)
+            identifier.append(i)
+        except ValueError:
+            i = identity.encode('utf-8')
+            identifier.append(i)
+    key = cache_data_key(identifier=identifier, salt=MetadataPointer.BALANCES)
     cache_data(key, json.dumps(balances))
 
 
@@ -122,6 +140,38 @@ def statement_callback(self, result, param: str, status_code: int):
             generate(param, queue, sender_transaction)
 
 
+@celery_app.task
+def token_data_callback(result: dict, param: str, status_code: int):
+    """
+    :param result:
+    :type result:
+    :param param:
+    :type param:
+    :param status_code:
+    :type status_code:
+    :return:
+    :rtype:
+    """
+
+    if status_code != 0:
+        raise ValueError(f'Unexpected status code: {status_code}.')
+
+    token = result[0]
+    token_symbol = token.get('symbol')
+    identifier = token_symbol.encode('utf-8')
+    token_meta_key = cache_data_key(identifier, MetadataPointer.TOKEN_META_SYMBOL)
+    token_info_key = cache_data_key(identifier, MetadataPointer.TOKEN_PROOF_SYMBOL)
+    token_meta = get_cached_data(token_meta_key)
+    token_meta = json.loads(token_meta)
+    token_info = get_cached_data(token_info_key)
+    token_info = json.loads(token_info)
+    token_data = collate_token_metadata(token_info=token_info, token_metadata=token_meta)
+    token_data = {**token_data, **token}
+    token_data_key = cache_data_key([bytes.fromhex(param), identifier], MetadataPointer.TOKEN_DATA)
+    cache_data(token_data_key, json.dumps(token_data))
+    handle_token_symbol_list(blockchain_address=param, token_symbol=token_symbol)
+
+
 @celery_app.task(bind=True)
 def transaction_balances_callback(self, result: list, param: dict, status_code: int):
     """
@@ -138,10 +188,15 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     """
     if status_code != 0:
         raise ValueError(f'Unexpected status code: {status_code}.')
-
     balances_data = result[0]
-    available_balance = calculate_available_balance(balances_data)
     transaction = param
+    token_symbol = transaction.get('token_symbol')
+    blockchain_address = transaction.get('blockchain_address')
+    identifier = [bytes.fromhex(blockchain_address), token_symbol.encode('utf-8')]
+    wait_for_cache(identifier, f'Cached token data for: {token_symbol}', MetadataPointer.TOKEN_DATA)
+    token_data = get_cached_token_data(blockchain_address, token_symbol)
+    decimals = token_data.get('decimals')
+    available_balance = calculate_available_balance(balances_data, decimals)
     transaction['available_balance'] = available_balance
     queue = self.request.delivery_info.get('routing_key')
 
@@ -175,6 +230,8 @@ def transaction_callback(result: dict, param: str, status_code: int):
     source_token_symbol = result.get('source_token_symbol')
     source_token_value = result.get('source_token_value')
 
+    process_token_data(blockchain_address=recipient_blockchain_address, token_symbol=destination_token_symbol)
+
     recipient_metadata = {
         "alt_blockchain_address": sender_blockchain_address,
         "blockchain_address": recipient_blockchain_address,

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -6,6 +6,7 @@ import logging
 import celery
 
 # local imports
+from cic_ussd.account.tokens import get_cached_token_data
 from cic_ussd.account.transaction import from_wei
 from cic_ussd.notifications import Notifier
 from cic_ussd.phone_number import Support
@@ -25,11 +26,15 @@ def transaction(notification_data: dict):
     """
     role = notification_data.get('role')
     token_value = notification_data.get('token_value')
-    amount = token_value if token_value == 0 else from_wei(token_value)
+    token_symbol = notification_data.get('token_symbol')
+    blockchain_address = notification_data.get('blockchain_address')
+    token_data = get_cached_token_data(blockchain_address, token_symbol)
+    decimals = token_data.get('decimals')
+    amount = token_value if token_value == 0 else from_wei(decimals, token_value)
     balance = notification_data.get('available_balance')
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
-    token_symbol = notification_data.get('token_symbol')
+
     alt_metadata_id = notification_data.get('alt_metadata_id')
     metadata_id = notification_data.get('metadata_id')
     transaction_type = notification_data.get('transaction_type')

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -47,7 +47,8 @@ def cache_statement(parsed_transaction: dict, querying_party: str):
     statement_transactions = []
     if cached_statement:
         statement_transactions = json.loads(cached_statement)
-    statement_transactions.append(parsed_transaction)
+    if parsed_transaction not in statement_transactions:
+        statement_transactions.append(parsed_transaction)
     data = json.dumps(statement_transactions)
     identifier = bytes.fromhex(querying_party)
     key = cache_data_key(identifier, MetadataPointer.STATEMENT)
@@ -74,6 +75,14 @@ def parse_transaction(transaction: dict) -> dict:
     role = transaction.get('role')
     alt_blockchain_address = transaction.get('alt_blockchain_address')
     blockchain_address = transaction.get('blockchain_address')
+    identifier = bytes.fromhex(blockchain_address)
+    token_symbol = transaction.get('token_symbol')
+    if role == 'recipient':
+        key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_RECEIVED)
+        cache_data(key, token_symbol)
+    if role == 'sender':
+        key = cache_data_key(identifier=identifier, salt=MetadataPointer.TOKEN_LAST_SENT)
+        cache_data(key, token_symbol)
     account = validate_transaction_account(blockchain_address, role, session)
     alt_account = session.query(Account).filter_by(blockchain_address=alt_blockchain_address).first()
     if alt_account:

apps/cic-ussd/docker/Dockerfile

@@ -10,7 +10,7 @@ RUN mkdir -vp pgp/keys
 RUN mkdir -vp cic-ussd
 RUN mkdir -vp data
 
-ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
+ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
 ARG EXTRA_PIP_ARGS=""
 ARG PIP_INDEX_URL=https://pypi.org/simple
 
@@ -18,7 +18,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url $PIP_INDEX_URL \
     --pre \
     --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-        cic-eth-aux-erc20-demurrage-token~=0.0.2a7
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a7
 
 
 COPY *requirements.txt ./

apps/cic-ussd/docker/run_tests.sh

@@ -3,7 +3,7 @@
 set -e
 
 
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+pip install --extra-index-url https://pip.grassrootseconomics.net \
 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
 -r test_requirements.txt 
 

apps/cic-ussd/requirements.txt

@@ -6,7 +6,7 @@ celery==4.4.7
 cffi==1.14.6
 cic-eth~=0.12.5a1
 cic-notify~=0.4.0a11
-cic-types~=0.2.1a2
+cic-types~=0.2.1a7
 confini>=0.3.6rc4,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6

apps/cic-ussd/states/exit_states.json

@@ -11,5 +11,7 @@
   "account_creation_prompt",
   "exit_successful_transaction",
   "exit_insufficient_balance",
+  "exit_invalid_guardian_addition",
+  "exit_invalid_guardian_removal",
   "complete"
 ]

apps/cic-ussd/states/pin_management_states.json

@@ -0,0 +1,16 @@
+[
+  "pin_management",
+  "reset_guarded_pin",
+  "reset_guarded_pin_authorization",
+  "exit_pin_reset_initiated_success",
+  "exit_not_authorized_for_pin_reset",
+  "guard_pin",
+  "guardian_list_pin_authorization",
+  "guardian_list",
+  "add_guardian",
+  "add_guardian_pin_authorization",
+  "exit_guardian_addition_success",
+  "remove_guardian",
+  "remove_guardian_pin_authorization",
+  "exit_guardian_removal_success"
+]

apps/cic-ussd/states/token_management_states.json

@@ -0,0 +1,7 @@
+[
+  "first_account_tokens_set",
+  "middle_account_tokens_set",
+  "last_account_tokens_set",
+  "token_selection_pin_authorization",
+  "exit_successful_token_selection"
+]

apps/cic-ussd/transitions/account_management_transitions.json

@@ -50,7 +50,7 @@
   {
     "trigger": "scan_data",
     "source": "account_management",
-    "dest": "enter_current_pin",
+    "dest": "pin_management",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected"
   },
   {

apps/cic-ussd/transitions/account_statement_transitions.json

@@ -8,7 +8,7 @@
   {
     "trigger": "scan_data",
     "source": "first_transaction_set",
-    "dest": "start",
+    "dest": "exit",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
   },
   {
@@ -31,7 +31,7 @@
   {
     "trigger": "scan_data",
     "source": "middle_transaction_set",
-    "dest": "start",
+    "dest": "exit",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
   },
   {
@@ -48,7 +48,7 @@
   {
     "trigger": "scan_data",
     "source": "last_transaction_set",
-    "dest": "start",
+    "dest": "exit",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
   },
   {

apps/cic-ussd/transitions/exit_transitions.json

@@ -58,5 +58,17 @@
     "source": "exit_successful_transaction",
     "dest": "exit",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_ninety_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_successful_token_selection",
+    "dest": "start",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_zero_zero_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_successful_token_selection",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_ninety_nine_selected"
   }
 ]

apps/cic-ussd/transitions/pin_guard_transitions.json

@@ -0,0 +1,119 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "guard_pin",
+    "dest": "guardian_list_pin_authorization",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_one_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guardian_list_pin_authorization",
+    "dest": "guardian_list",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guardian_list_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_blocked_pin"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guard_pin",
+    "dest": "add_guardian",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_two_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "add_guardian",
+    "dest": "add_guardian_pin_authorization",
+    "after": [
+      "cic_ussd.state_machine.logic.pin_guard.save_guardian_to_session_data",
+      "cic_ussd.state_machine.logic.pin_guard.retrieve_person_metadata"
+    ],
+    "conditions": "cic_ussd.state_machine.logic.pin_guard.is_valid_guardian_addition"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "add_guardian",
+    "dest": "exit_invalid_guardian_addition",
+    "unless": "cic_ussd.state_machine.logic.pin_guard.is_valid_guardian_addition"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "add_guardian_pin_authorization",
+    "dest": "exit_guardian_addition_success",
+    "after": "cic_ussd.state_machine.logic.pin_guard.add_pin_guardian",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "add_guardian_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guard_pin",
+    "dest": "remove_guardian",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "remove_guardian",
+    "dest": "remove_guardian_pin_authorization",
+    "after": [
+      "cic_ussd.state_machine.logic.pin_guard.save_guardian_to_session_data",
+      "cic_ussd.state_machine.logic.pin_guard.retrieve_person_metadata"
+    ],
+    "conditions": "cic_ussd.state_machine.logic.pin_guard.is_dialers_pin_guardian"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "remove_guardian",
+    "dest": "exit_invalid_guardian_removal",
+    "unless": "cic_ussd.state_machine.logic.pin_guard.is_dialers_pin_guardian"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "remove_guardian_pin_authorization",
+    "dest": "exit_guardian_removal_success",
+    "after": "cic_ussd.state_machine.logic.pin_guard.remove_pin_guardian",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "remove_guardian_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_guardian_removal_success",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_invalid_guardian_addition",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_invalid_guardian_removal",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guardian_list",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "guard_pin",
+    "dest": "exit_invalid_menu_option"
+  }
+]

apps/cic-ussd/transitions/pin_management_transitions.json

@@ -0,0 +1,20 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "pin_management",
+    "dest": "enter_current_pin",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_one_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "pin_management",
+    "dest": "reset_guarded_pin",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_two_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "pin_management",
+    "dest": "guard_pin",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
+  }
+]

apps/cic-ussd/transitions/pin_reset_transitions.json

@@ -0,0 +1,43 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "reset_guarded_pin",
+    "dest": "reset_guarded_pin_authorization",
+    "after": [
+      "cic_ussd.state_machine.logic.pin_guard.save_guarded_account_session_data",
+      "cic_ussd.state_machine.logic.pin_guard.retrieve_person_metadata"
+    ],
+    "conditions": "cic_ussd.state_machine.logic.pin_guard.is_others_pin_guardian"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "reset_guarded_pin_authorization",
+    "dest": "exit_pin_reset_initiated_success",
+    "after": "cic_ussd.state_machine.logic.pin_guard.initiate_pin_reset",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_pin_reset_initiated_success",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "reset_guarded_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "reset_guarded_pin",
+    "dest": "exit_not_authorized_for_pin_reset",
+    "unless": "cic_ussd.state_machine.logic.pin_guard.is_others_pin_guardian"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "exit_not_authorized_for_pin_reset",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_nine_selected"
+  }
+]

apps/cic-ussd/transitions/start_menu_transitions.json

@@ -8,15 +8,21 @@
   {
     "trigger": "scan_data",
     "source": "start",
-    "dest": "account_management",
+    "dest": "first_account_tokens_set",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_two_selected"
   },
   {
     "trigger": "scan_data",
     "source": "start",
-    "dest": "help",
+    "dest": "account_management",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
   },
+  {
+    "trigger": "scan_data",
+    "source": "start",
+    "dest": "help",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_four_selected"
+  },
   {
     "trigger": "scan_data",
     "source": "start",