Consider faucet amount in verify balance

Move to chainlib-eth dependency

See merge request grassrootseconomics/cic-internal-integration!199

.gitignore

@@ -8,3 +8,8 @@ gmon.out
 *.egg-info
 dist/
 build/
+**/*sqlite
+**/.nyc_output
+**/coverage
+**/.venv
+.idea

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -89,7 +89,7 @@ def process_transactions_all_data(session, env):
 
     offset = r[1]
     end = r[2]
-    if r[2] < r[1]:
+    if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,6 +16,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -28,10 +29,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -113,10 +112,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,12 +1,13 @@
-cic-base~=0.1.2b10
+cic-base==0.1.3a3+build.4aa03607
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.3a3
+erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,6 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-web3==5.12.2
-cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8
+cic_base[full]==0.1.3a3+build.4aa03607
+sarafu-faucet~=0.0.4a1

apps/cic-eth/.coveragerc

@@ -5,3 +5,5 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
+	cic_eth/queue/state.py
+	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -5,18 +5,29 @@
 
 .cic_eth_changes_target:
     rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      #changes:
+          #- $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
-        - .cic_eth_changes_target
-        - .py_build_merge_request
         - .cic_eth_variables
+        - .cic_eth_changes_target
+        - .py_build_target_test
+
+test-mr-cic-eth:
+    extends:
+        - .cic_eth_variables
+        - .cic_eth_changes_target
+    stage: test
+    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+    script:
+        - cd apps/$APP_NAME/
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
-        
-

apps/cic-eth/MANIFEST.in

@@ -0,0 +1,2 @@
+include *requirements.txt
+

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,11 +4,18 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
-from chainqueue.query import get_tx
-from chainqueue.state import set_cancel
+from chainlib.connection import RPCConnection
+from chainlib.eth.tx import (
+        unpack,
+        TxFactory,
+        )
+from chainlib.eth.gas import OverrideGasOracle
+from chainqueue.sql.query import get_tx
+from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
+from hexathon import strip_0x
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -21,13 +28,14 @@ from cic_eth.admin.ctrl import (
         )
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True)
-def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
+@celery_app.task(bind=True, base=BaseTask)
+def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -38,25 +46,29 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
+    chain_spec = ChainSpec.from_dict(chainspec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_brief = get_tx(tx_hash_orig_hex)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
+    session = BaseTask.session_func()
+    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
 
-    lock_queue(None, chain_str, address)
-    lock_send(None, chain_str, address)
+    lock_queue(None, chain_spec.asdict(), address=address)
+    lock_send(None, chain_spec.asdict(), address=address)
+
+    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -69,49 +81,57 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
+        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        del(tx_new['hash'])
-        del(tx_new['hash_unsigned'])
+        tx_new['gas_price'] += 1
+        tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
 
-        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
+        logg.debug('tx_new {}'.format(tx_new))
+
+        del(tx_new['hash'])
+        del(tx_new['hash_unsigned'])
+        del(tx_new['hashUnsigned'])
+
+        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
+        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            nonce=tx_new['nonce'],
-            address=tx_new['from'],
-            tx_hash=tx_hash_hex,
-            signed_tx=tx_signed_raw_hex,
-                )
+            tx_new['nonce'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            )
         session.add(otx)
-        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(tx_previous_hash_hex, True)
+        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
+        session.commit()
 
     session.close()
 
-    s = create_check_gas_and_send_task(
+    s = create_check_gas_task(
          txs, 
-         chain_str,
+         chain_spec,
          tx_new['from'],
-         tx_new['gas'],
-         tx_hashes, 
-         queue,
+         gas=tx_new['gas'],
+         tx_hashes_hex=tx_hashes, 
+         queue=queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,
@@ -119,7 +139,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/cic_eth/api/api_admin.py

@@ -8,6 +8,7 @@ from chainlib.eth.constant import (
         ZERO_ADDRESS,
         )
 from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.contract import code
@@ -30,13 +31,14 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
+from chainqueue.sql.query import get_tx
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
-from cic_eth.queue.query import get_tx
 
 app = celery.current_app
 
@@ -188,6 +190,7 @@ class AdminApi:
         s_manual = celery.signature(
             'cic_eth.queue.state.set_manual',
             [
+                chain_spec.asdict(),
                 tx_hash_hex,
                 ],
             queue=self.queue,
@@ -206,8 +209,9 @@ class AdminApi:
             s.link(s_gas)
 
         return s_manual.apply_async()
-                        
-    def check_nonce(self, address):
+                    
+
+    def check_nonce(self, chain_spec, address):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -228,13 +232,12 @@ class AdminApi:
             s_get_tx = celery.signature(
                     'cic_eth.queue.query.get_tx',
                     [
-                    chain_spec.asdict(),
+                        chain_spec.asdict(),
                         k,
                         ],
                     queue=self.queue,
                     )
             tx = s_get_tx.apply_async().get()
-            #tx = get_tx(k)
             logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
             if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@@ -242,7 +245,9 @@ class AdminApi:
                 blocking_tx = k
                 blocking_nonce = nonce_otx
             elif nonce_otx - last_nonce > 1:
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from']))
+                logg.debug('tx {}'.format(tx))
+                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
+                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
                 blocking_tx = k
                 blocking_nonce = nonce_otx
                 break
@@ -256,12 +261,13 @@ class AdminApi:
                 'blocking': blocking_nonce,
             },
             'tx': {
-                'blocking': blocking_tx,
-                }
+                'blocking': add_0x(blocking_tx),
             }
+        }
 
 
-    def fix_nonce(self, address, nonce, chain_spec):
+    # TODO: is risky since it does not validate that there is actually a nonce problem?
+    def fix_nonce(self, chain_spec, address, nonce):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -275,15 +281,17 @@ class AdminApi:
         txs = s.apply_async().get()
 
         tx_hash_hex = None
+        session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(k)
+            tx_dict = get_tx(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
+        session.close()
 
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    self.rpc.chain_spec.asdict(),
+                    chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -388,12 +396,13 @@ class AdminApi:
     
         t = s.apply_async()
         tx = t.get()
-  
+
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
+            source_token_declaration = None
             if registry != None:
                 try:
-                    source_token = registry.by_address(tx['source_token'])
+                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
             else:
@@ -406,16 +415,21 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                source_token = t.get()
-                if source_token == None:
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
+                source_token_declaration = t.get()
+
+            if source_token_declaration != None:
+                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
+                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
+                logg.debug('source token set tup {}'.format(source_token))
+
 
 
         destination_token = None
         if tx['destination_token'] != ZERO_ADDRESS:
+            destination_token_declaration = None
             if registry != None:
                 try:
-                    destination_token = registry.by_address(tx['destination_token'])
+                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
             else:
@@ -428,10 +442,10 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                destination_token = t.get()
-                if destination_token == None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
-
+                destination_token_declaration = t.get()
+            if destination_token_declaration != None:
+                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
+                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
@@ -543,13 +557,19 @@ class AdminApi:
                 if role != None:
                     tx['recipient_description'] = role
 
+        erc20_c = ERC20(chain_spec)
         if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol()
-            tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call()
+            tx['source_token_symbol'] = source_token.symbol
+            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['sender_token_balance'] = erc20_c.parse_balance(r)
 
         if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol()
-            tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call()
+            tx['destination_token_symbol'] = destination_token.symbol
+            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
+            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
         tx['network_status'] = 'Not in node' 

apps/cic-eth/cic_eth/api/api_task.py

@@ -74,134 +74,134 @@ class Api:
         return s_token.apply_async()
 
 
-    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of receipient
-        :type to_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
-        :type  target_return: int
-        :param minimum_return: The least value of destination token return to allow
-        :type minimum_return: int
-        :param from_token_symbol: ERC20 token symbol of token being converted
-        :type from_token_symbol: str
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        raise NotImplementedError('out of service until new DEX migration is done')
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [from_token_symbol, to_token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        s_convert = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
-                [
-                    from_address,
-                    target_return,
-                    minimum_return,
-                    to_address,
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_convert.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
-        else:
-            s_tokens.link(s_convert)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
-
-
-    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
-        :type  target_return: int
-        :param minimum_return: The least value of destination token return to allow
-        :type minimum_return: int
-        :param from_token_symbol: ERC20 token symbol of token being converted
-        :type from_token_symbol: str
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        raise NotImplementedError('out of service until new DEX migration is done')
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [from_token_symbol, to_token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_convert = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
-                [
-                    from_address,
-                    target_return,
-                    minimum_return,
-                    from_address,
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_convert.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
-        else:
-            s_tokens.link(s_convert)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
+#    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param to_address: Ethereum address of receipient
+#        :type to_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_str,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    to_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
+#
+#
+#    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    from_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
 
 
     def transfer(self, from_address, to_address, value, token_symbol):

apps/cic-eth/cic_eth/db/util.py

@@ -1,8 +0,0 @@
-import math
-
-def num_serialize(n):
-    if n == 0:
-        return b'\x00'
-    binlog = math.log2(n)
-    bytelength = int(binlog / 8 + 1)
-    return n.to_bytes(bytelength, 'big')

apps/cic-eth/cic_eth/eth/gas.py

@@ -57,10 +57,12 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+MAXIMUM_FEE_UNITS = 8000000
+
 class MaxGasOracle:
 
     def gas(code=None):
-        return 8000000
+        return MAXIMUM_FEE_UNITS
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -150,7 +152,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,24 +172,30 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    if len(txs) == 0:
-        for i in range(len(tx_hashes)):
-            o = get_tx(tx_hashes[i])
-            txs.append(o['signed_tx'])
-            if address == None:
-                address = o['address']
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
+
+    addresspass = None
+    if len(txs) == 0:
+        addresspass = []
+        for i in range(len(tx_hashes)):
+            o = get_tx(chain_spec_dict, tx_hashes[i])
+            txs.append(o['signed_tx'])
+            logg.debug('sender {}'.format(o))
+            tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
+            if address == None:
+                address = tx['from']
+            elif address != tx['from']:
+                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
+            addresspass.append(address)
 
-    #if not web3.Web3.isChecksumAddress(address):
     if not is_checksum_address(address):
         raise ValueError('invalid address {}'.format(address))
 
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
-    # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     gas_balance = 0
     try:
         o = balance(address)
@@ -198,6 +206,9 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
         conn.disconnect()
         raise EthError('gas_balance call for {}: {}'.format(address, e))
 
+    if gas_required == None:
+        gas_required = MAXIMUM_FEE_UNITS
+
     logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
     session = SessionBase.create_session()
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@@ -268,7 +279,8 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
             queue=queue,
             )
         ready_tasks.append(s)
-    celery.group(ready_tasks)()
+    t = celery.group(ready_tasks)()
+    logg.debug('group {}'.format(t))
 
     return txs
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -21,6 +21,7 @@ from chainqueue.db.models.tx import Otx
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db import SessionBase
@@ -58,6 +59,9 @@ def hashes_to_txs(self, tx_hashes):
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
+    for i in range(len(tx_hashes)):
+        tx_hashes[i] = strip_0x(tx_hashes[i])
+
     queue = self.request.delivery_info['routing_key']
 
     session = SessionBase.create_session()
@@ -148,7 +152,7 @@ def send(self, txs, chain_spec_dict):
 
 @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
 def sync_tx(self, tx_hash_hex, chain_spec_dict):
-    """Force update of network status of a simgle transaction
+    """Force update of network status of a single transaction
 
     :param tx_hash_hex: Transaction hash
     :type tx_hash_hex: str, 0x-hex
@@ -173,12 +177,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
 
     # TODO: apply receipt in tx object to validate and normalize input
     if rcpt != None:
+        rcpt = snake_and_camel(rcpt)
         success = rcpt['status'] == 1
-        logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
+        logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
 
         s = celery.signature(
             'cic_eth.queue.state.set_final',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 rcpt['blockNumber'],
                 rcpt['transactionIndex'],
@@ -186,12 +192,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
                 ],
                 queue=queue,
             )
+    # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
     else:
         logg.debug('sync tx {} mempool'.format(tx_hash_hex))
 
         s = celery.signature(
             'cic_eth.queue.state.set_sent',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 ],
                 queue=queue,

apps/cic-eth/cic_eth/ext/tx.py

@@ -19,7 +19,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
-from chainqueue.query import get_tx_cache
+from chainqueue.sql.query import get_tx_cache
 from eth_erc20 import ERC20
 
 # local imports

apps/cic-eth/cic_eth/queue/query.py

@@ -5,7 +5,7 @@ import datetime
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.query
+import chainqueue.sql.query
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -28,7 +28,7 @@ celery_app = celery.current_app
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r =  chainqueue.query.get_tx(chain_spec, tx_hash)
+    r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -46,7 +46,7 @@ def get_tx(chain_spec_dict, tx_hash):
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
+    r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
     session.close()
     return r
 
@@ -55,17 +55,17 @@ def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True,
 def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):

apps/cic-eth/cic_eth/queue/state.py

@@ -1,6 +1,6 @@
 # external imports
 from chainlib.chain import ChainSpec
-import chainqueue.state
+import chainqueue.sql.state
 
 # local imports
 import celery
@@ -14,7 +14,7 @@ celery_app = celery.current_app
 def set_sent(chain_spec_dict, tx_hash, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session)
+    r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
     session.close()
     return r
 
@@ -23,7 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
+    r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
     session.close()
     return r
 
@@ -32,7 +32,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
+    r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
     session.close()
     return r
 
@@ -41,7 +41,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 def set_rejected(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -50,7 +50,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 def set_fubar(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -59,7 +59,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 def set_manual(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -68,7 +68,7 @@ def set_manual(chain_spec_dict, tx_hash):
 def set_ready(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -77,7 +77,7 @@ def set_ready(chain_spec_dict, tx_hash):
 def set_reserved(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -86,7 +86,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 def set_waitforgas(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
     session.close()
     return r
     
@@ -95,7 +95,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 def get_state_log(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -104,6 +104,6 @@ def get_state_log(chain_spec_dict, tx_hash):
 def obsolete(chain_spec_dict, tx_hash, final):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
+    r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
     session.close()
     return r

apps/cic-eth/cic_eth/queue/tx.py

@@ -15,14 +15,14 @@ from sqlalchemy import tuple_
 from sqlalchemy import func
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.state
+import chainqueue.sql.state
 from chainqueue.db.enum import (
         StatusEnum,
         StatusBits,
         is_alive,
         dead,
         )
-from chainqueue.tx import create
+from chainqueue.sql.tx import create
 from chainqueue.error import NotLocalTxError
 from chainqueue.db.enum import status_str
 

apps/cic-eth/cic_eth/registry.py

@@ -5,29 +5,30 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
 
 logg = logging.getLogger()
 
 
-def connect_token_registry(rpc, chain_spec):
+def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    token_registry_address = registry.by_name('TokenRegistry')
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
     logg.debug('using token registry address {}'.format(token_registry_address))
     lookup = TokenIndexLookup(chain_spec, token_registry_address)
     CICRegistry.add_lookup(lookup)
 
 
-def connect_declarator(rpc, chain_spec, trusted_addresses):
+def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator')
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
     logg.debug('using declarator address {}'.format(declarator_address))
     lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
     CICRegistry.add_lookup(lookup)
 
 
-def connect(rpc, chain_spec, registry_address):
+def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry')
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
     return registry
 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -21,7 +21,7 @@ from chainqueue.db.enum import (
     StatusBits,
     )
 from chainqueue.error import NotLocalTxError
-from chainqueue.state import set_reserved
+from chainqueue.sql.state import set_reserved
 
 # local imports
 import cic_eth

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,14 +10,15 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.query import get_paused_tx_cache as get_paused_tx
+from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 
 class GasFilter(SyncFilter):
@@ -27,11 +28,11 @@ class GasFilter(SyncFilter):
         self.chain_spec = chain_spec
 
 
-    def filter(self, conn, block, tx, session):
+    def filter(self, conn, block, tx, db_session):
         if tx.value > 0:
             tx_hash_hex = add_0x(tx.hash)
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(session)
+            session = SessionBase.bind_session(db_session)
             q = session.query(TxCache.recipient)
             q = q.join(Otx)
             q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@@ -56,7 +57,7 @@ class GasFilter(SyncFilter):
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
-                s.apply_async()
+                return s.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -50,7 +50,8 @@ class RegistrationFilter(SyncFilter):
                     queue=self.queue,
                     )
                 s_nonce.link(s_gift)
-                s_nonce.apply_async()
+                t = s_nonce.apply_async()
+                return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/straggler.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from chainqueue.state import obsolete_by_cache
+from chainqueue.sql.state import obsolete_by_cache
 
 logg = logging.getLogger()
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
         self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
 
 
-    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
+    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
 
         if tx.payload == None:
             logg.debug('no payload')
@@ -45,16 +45,17 @@ class TransferAuthFilter(SyncFilter):
             return False
 
         recipient = tx.inputs[0]
-        if recipient != self.transfer_request_contract.address():
+        #if recipient != self.transfer_request_contract.address():
+        if recipient != self.transfer_request_contract:
             logg.debug('not our transfer auth contract address {}'.format(recipient))
             return False
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
-           
-        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
-        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
-        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
-        value = abi_decode_single(ABIContractType.UINT256, r[3])
+          
+        sender = r[0]
+        recipient = r[1]
+        token = r[2]
+        value = r[3]
 
         token_data = {
             'address': token,
@@ -64,6 +65,7 @@ class TransferAuthFilter(SyncFilter):
             'cic_eth.eth.nonce.reserve_nonce',
             [
                 [token_data],
+                self.chain_spec.asdict(), 
                 sender,
                 ],
             queue=self.queue,
@@ -80,7 +82,7 @@ class TransferAuthFilter(SyncFilter):
             )
         s_nonce.link(s_approve)
         t = s_nonce.apply_async()
-        return True
+        return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,136 +0,0 @@
-# standard imports
-import os
-import re
-import logging
-import argparse
-import json
-
-# third-party imports
-import web3
-import confini
-import celery
-from json.decoder import JSONDecodeError
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import dsn_from_config
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_eth', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-queue = args.q
-
-re_something = r'^/something/?'
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-def process_something(session, env):
-    r = re.match(re_something, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    #if env.get('CONTENT_TYPE') != 'application/json':
-    #    raise AttributeError('content type')
-
-    #if env.get('REQUEST_METHOD') != 'POST':
-    #    raise AttributeError('method')
-
-    #post_data = json.load(env.get('wsgi.input'))
-    
-    #return ('text/plain', 'foo'.encode('utf-8'),)
-
-
-# uwsgi application
-def application(env, start_response):
-    
-    for k in env.keys():
-        logg.debug('env {} {}'.format(k, env[k]))
-
-    headers = []
-    content = b''
-    err = None
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_something,
-            ]:
-        try:
-            r = handler(session, env)
-        except AttributeError as e:
-            logg.error('handler fail attribute {}'.format(e))
-            err = '400 Impertinent request'
-            break
-        except JSONDecodeError as e:
-            logg.error('handler fail json {}'.format(e))
-            err = '400 Invalid data format'
-            break
-        except KeyError as e:
-            logg.error('handler fail key {}'.format(e))
-            err = '400 Invalid JSON'
-            break
-        except ValueError as e:
-            logg.error('handler fail value {}'.format(e))
-            err = '400 Invalid data'
-            break
-        except RuntimeError as e:
-            logg.error('task fail value {}'.format(e))
-            err = '500 Task failed, sorry I cannot tell you more'
-            break
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    if err != None:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response(err, headers)
-        session.close()
-        return [content]
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,6 +194,7 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
+    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,10 +27,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/stat.py

@@ -20,7 +20,11 @@ def init_chain_stat(rpc, block_start=0):
     if block_start == 0:
         o = block_latest()
         r = rpc.do(o)
-        block_start = int(r, 16)
+        try:
+            block_start = int(r, 16)
+        except TypeError:
+            block_start = int(r)
+        logg.debug('blockstart {}'.format(block_start))
 
     for i in range(BLOCK_SAMPLES):
         o = block_by_number(block_start-10+i)

apps/cic-eth/cic_eth/task.py

@@ -20,7 +20,8 @@ import liveness.linux
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 celery_app = celery.current_app
 
@@ -118,12 +119,13 @@ def registry():
     return CICRegistry.address
 
 
-@celery_app.task()
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
+@celery_app.task(bind=True, base=BaseTask)
+def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
+    r = registry.by_address(address, sender_address=self.call_address)
+    return r
 
 
 @celery_app.task(throws=(UnknownContractError,))
@@ -131,7 +133,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
+    return registry.by_name(name, sender_address=self.call_address)
 
 
 @celery_app.task()

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         11,
-        0,
-        'beta.14',
+        1,
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,48 +1,62 @@
-# FROM grassrootseconomics:cic
-
-#FROM python:3.8.6-alpine
-FROM python:3.8.6-slim-buster
-
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+FROM python:3.8.6-slim-buster as compile
 
 WORKDIR /usr/src/cic-eth
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
-
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file:  ${root_requirement_file}"
-#COPY $root_requirement_file .
-#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
-RUN /usr/local/bin/python -m pip install --upgrade pip
-#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
-#	cd cic-base && \
-#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
-#	python merge_requirements.py | tee merged_requirements.txt 
-#RUN cd cic-base && \
-#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+#RUN python -m venv venv && . venv/bin/activate
 
-COPY cic-eth/scripts/ scripts/
-COPY cic-eth/setup.cfg cic-eth/setup.py ./
-COPY cic-eth/cic_eth/ cic_eth/
-# Copy app specific requirements
-COPY cic-eth/requirements.txt .
-COPY cic-eth/test_requirements.txt .
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+RUN /usr/local/bin/python -m pip install --upgrade pip
+RUN pip install semver
+
+# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
+COPY cic-eth/ .
 RUN	pip install $pip_extra_index_url_flag .
 
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
+
+COPY cic-eth/test_requirements.txt .
+RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
+
+COPY cic-eth . 
+
+ENV PYTHONPATH .
+
+ENTRYPOINT ["pytest"]
+
+# --- RUNTIME ---
+FROM python:3.8.6-slim-buster as runtime 
+
+RUN apt-get update && \
+	apt install -y gnupg libpq-dev procps
+
+WORKDIR /usr/src/cic-eth
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
-COPY cic-eth/tests/ tests/
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -51,3 +65,4 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
+

apps/cic-eth/requirements.txt

@@ -1,24 +1,25 @@
-cic-base~=0.1.2b11
+cic-base==0.1.3a3+build.4aa03607
 celery==4.4.7
-crypto-dev-signer~=0.4.14b3
+crypto-dev-signer~=0.4.14b6
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a6
+eth_accounts_index~=0.0.12a1
+erc20-transfer-authorization~=0.3.2a1
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3a2
+eth-address-index~=0.1.2a1
+chainlib-eth~=0.0.5a1
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a4
-chainqueue~=0.0.2a2
-sarafu-faucet==0.0.3a3
-erc20-faucet==0.2.1a4
+chainsyncer[sql]~=0.0.3a3
+chainqueue~=0.0.2b5
+sarafu-faucet~=0.0.4a1
+erc20-faucet~=0.2.2a1
 coincurve==15.0.0
 potaahto~=0.0.1a2
+pycryptodome==3.10.1

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/setup.py

@@ -11,17 +11,6 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
 setup(
-    install_requires=requirements,
-    tests_require=test_requirements,
+    install_requires=requirements
         )

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,3 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/check/test_check_db.py

@@ -0,0 +1,8 @@
+# local imports
+from cic_eth.check.db import health
+
+def test_check_health(
+    init_database,
+        ):
+
+    assert health()

apps/cic-eth/tests/check/test_check_gas.py

@@ -0,0 +1,20 @@
+# local imports
+from cic_eth.check.gas import health
+from cic_eth.db.models.role import AccountRole
+
+def test_check_gas(
+    config,
+    init_database,
+    default_chain_spec,
+    eth_rpc,
+    custodial_roles,
+    whoever,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
+    assert health(config=config)
+
+    AccountRole.set('GAS_GIFTER', whoever, session=init_database)
+    init_database.commit()
+    assert not health(config=config)

apps/cic-eth/tests/check/test_check_redis.py

@@ -0,0 +1,16 @@
+# external imports
+import pytest
+
+# local imports
+from cic_eth.check.redis import health
+
+
+def test_check_redis(
+        config,
+        have_redis,
+        ):
+
+    if have_redis != None:
+        pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
+
+    assert health(unit='test', config=config)

apps/cic-eth/tests/check/test_check_signer.py

@@ -0,0 +1,13 @@
+# local imports
+from cic_eth.check.signer import health
+
+
+def test_check_signer(
+        default_chain_spec,
+        config,
+        eth_signer,
+        eth_rpc,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    assert health(config=config)

apps/cic-eth/tests/conftest.py

@@ -2,9 +2,11 @@
 import os
 import sys
 import logging
+import uuid
 
 # external imports
 from eth_erc20 import ERC20
+import redis
 
 # local imports
 from cic_eth.api import Api
@@ -19,6 +21,7 @@ from tests.fixtures_config import *
 from tests.fixtures_database import *
 from tests.fixtures_celery import *
 from tests.fixtures_role import *
+from tests.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -55,3 +58,28 @@ def default_token(
         ):
     BaseTask.default_token_symbol = foo_token_symbol
     BaseTask.default_token_address = foo_token   
+
+
+
+@pytest.fixture(scope='session')
+def have_redis(
+        config,
+        ):
+
+    r = redis.Redis(
+            host = config.get('REDIS_HOST'),
+            port = config.get('REDIS_PORT'),
+            db = config.get('REDIS_DB'),
+            ) 
+    k = str(uuid.uuid4())
+    try:
+        r.set(k, 'foo')
+        r.delete(k)
+    except redis.exceptions.ConnectionError as e:
+        return e
+    except TypeError as e:
+        return e 
+
+    return None
+
+

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -0,0 +1,38 @@
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+from cic_eth.runnable.daemons.filters.callback import CallbackFilter
+from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
+from cic_eth.runnable.daemons.filters.tx import TxFilter
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+# Hit tx mismatch paths on all filters
+def test_filter_bogus(
+        init_database,
+        bogus_tx_block,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        transfer_auth,
+        cic_registry,
+        contract_roles,
+        register_lookups,
+        ):
+
+    fltrs = [
+        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
+        GasFilter(default_chain_spec, queue=None),
+        TxFilter(default_chain_spec, None),
+        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
+        StragglerFilter(default_chain_spec, None),
+        RegistrationFilter(default_chain_spec, queue=None),
+        ]
+      
+    for fltr in fltrs:
+        r = None
+        try:
+            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
+        except:
+            pass
+        assert not r

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -0,0 +1,101 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.sql.state import (
+        set_waitforgas,
+        )
+from hexathon import strip_0x
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.eth.gas import cache_gas_data
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
+    init_database.commit()
+    tx_hash_hex_wait = tx_hash_hex
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['BOB'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    fltr = GasFilter(default_chain_spec, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

apps/cic-eth/tests/filters/test_register_filter.py

@@ -0,0 +1,78 @@
+# external imports
+from eth_accounts_index.registry import AccountRegistry
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import(
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from erc20_faucet import Faucet
+from hexathon import strip_0x
+from chainqueue.sql.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+def test_register_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        account_registry,
+        faucet,
+        register_lookups,
+        contract_roles,
+        agent_roles,
+        cic_registry,
+        init_celery_tasks,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
+
+    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block, rcpt=rcpt)
+    tx.apply_receipt(rcpt)
+
+    fltr = RegistrationFilter(default_chain_spec, queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(gift_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    gift = Faucet.parse_give_to_request(gift_tx['data'])
+    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_straggler_filter.py

@@ -17,8 +17,8 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.tx import create as queue_create
-from chainqueue.state import (
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
         set_reserved,
         set_ready,
         set_sent,

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -0,0 +1,79 @@
+# external imports
+from erc20_transfer_authorization import TransferAuthorization
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from hexathon import strip_0x
+from chainqueue.sql.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+
+
+def test_filter_transferauth(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        transfer_auth,
+        foo_token,
+        celery_session_worker,
+        register_lookups,
+        init_custodial,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
+    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
+    
+    r = rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(approve_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    c = ERC20(default_chain_spec)
+    approve = c.parse_approve_request(approve_tx['data']) 
+    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -17,13 +17,12 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.tx import create as queue_create
-from chainqueue.state import (
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
         set_reserved,
         set_ready,
         set_sent,
         )
-
 from hexathon import strip_0x
 
 # local imports
@@ -31,7 +30,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
 from cic_eth.eth.gas import cache_gas_data
 
 
-def test_tx(
+def test_filter_tx(
         default_chain_spec,
         init_database,
         eth_rpc,

apps/cic-eth/tests/fixtures_celery.py

@@ -22,7 +22,6 @@ def init_celery_tasks(
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
-#        'cic_eth.eth.bancor',
         'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
@@ -47,8 +46,8 @@ def celery_config():
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
     rq = tempfile.mkdtemp()
-    logg.debug('celery broker queue {} processed {}'.format(bq, bp))
-    logg.debug('celery backend store {}'.format(rq))
+    logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
+    logg.debug('celery backend session store {}'.format(rq))
     yield {
         'broker_url': 'filesystem://',
         'broker_transport_options': {
@@ -58,12 +57,11 @@ def celery_config():
             },
         'result_backend': 'file://{}'.format(rq),
             }
-    logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq))
+    logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
     shutil.rmtree(bq)
     shutil.rmtree(bp)
     shutil.rmtree(rq)
 
-
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {

apps/cic-eth/tests/fixtures_contract.py

@@ -0,0 +1,77 @@
+# standard imports
+import os
+
+# external imports
+import pytest
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        TxFactory,
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from hexathon import strip_0x
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+
+
+@pytest.fixture(scope='function')
+def bogus_tx_block(
+    default_chain_spec,
+    eth_rpc,
+    eth_signer,
+    contract_roles,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
+
+    f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
+    bytecode = f.read()
+    f.close()
+
+    c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
+    tx = c.set_code(tx, bytecode)
+    (tx_hash_hex, o) = c.build(tx)
+
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+
+    contract_address = r['contract_address']
+
+    enc = ABIContractEncoder()
+    enc.method('poke')
+    data = enc.get()
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
+    tx = c.set_code(tx, data)
+    (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
+    r = eth_rpc.do(o)
+    tx_signed_raw_hex = strip_0x(o['params'][0])
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    return (block, tx)

apps/cic-eth/tests/task/api/test_admin.py

@@ -9,8 +9,14 @@ from chainlib.eth.tx import (
         unpack,
         TxFormat,
         )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import Gas
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
 from chainlib.eth.address import to_checksum_address
 from hexathon import (
         strip_0x,
@@ -23,7 +29,15 @@ from chainqueue.db.enum import (
         StatusBits,
         status_str,
         )
-from chainqueue.query import get_tx
+from chainqueue.sql.state import (
+        set_fubar,
+        set_ready,
+        set_reserved,
+        )
+from chainqueue.sql.query import (
+        get_tx,
+        get_nonce_tx_cache,
+        )
 
 # local imports
 from cic_eth.api import AdminApi
@@ -36,150 +50,6 @@ from cic_eth.queue.tx import queue_create
 logg = logging.getLogger()
 
 
-#def test_resend_inplace(
-#    default_chain_spec,
-#    init_database,
-#    init_w3,
-#    celery_session_worker,
-#    ):
-#    
-#    chain_str = str(default_chain_spec)
-#    c = RpcClient(default_chain_spec)
-#    
-#    sigs = []
-#
-#    gas_provider = c.gas_provider()
-#
-#    s_nonce = celery.signature(
-#        'cic_eth.eth.nonce.reserve_nonce',
-#        [
-#            init_w3.eth.accounts[0],
-#            gas_provider,
-#            ],
-#        queue=None,
-#        )
-#    s_refill = celery.signature(
-#        'cic_eth.eth.gas.refill_gas',
-#        [
-#            chain_str,
-#            ],
-#        queue=None,
-#            )
-#    s_nonce.link(s_refill)
-#    t = s_nonce.apply_async()
-#    t.get()
-#    for r in t.collect():
-#        pass
-#    assert t.successful()
-#
-#    q = init_database.query(Otx)
-#    q = q.join(TxCache)
-#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
-#    o = q.first()
-#    tx_raw = o.signed_tx
-#
-#    tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
-#    gas_price_before = tx_dict['gasPrice'] 
-#
-#    s = celery.signature(
-#        'cic_eth.admin.ctrl.lock_send',
-#        [
-#            chain_str,
-#            init_w3.eth.accounts[0],
-#            ],
-#        queue=None,
-#            )
-#    t = s.apply_async()
-#    t.get()
-#    assert t.successful()
-#
-#    api = AdminApi(c, queue=None)
-#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
-#    t.get()
-#    i = 0
-#    tx_hash_new_hex = None
-#    for r in t.collect():
-#        tx_hash_new_hex = r[1]
-#    assert t.successful()
-#  
-#    tx_raw_new = get_tx(tx_hash_new_hex) 
-#    logg.debug('get {}'.format(tx_raw_new))
-#    tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
-#    assert tx_hash_new_hex != tx_dict['hash']
-#    assert tx_dict_new['gasPrice'] > gas_price_before
-#
-#    tx_dict_after = get_tx(tx_dict['hash'])
-#
-#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
-#    assert tx_dict_after['status'] & StatusBits.MANUAL
-
-
-#def test_check_fix_nonce(
-#    default_chain_spec,
-#    init_database,
-#    init_eth_account_roles,
-#    init_w3,
-#    eth_empty_accounts,
-#    celery_session_worker,
-#    ):
-#
-#    chain_str = str(default_chain_spec)
-#    
-#    sigs = []
-#    for i in range(5):
-#        s = celery.signature(
-#            'cic_eth.eth.gas.refill_gas',
-#            [
-#                eth_empty_accounts[i],
-#                chain_str,
-#                ],
-#            queue=None,
-#                )
-#        sigs.append(s)
-#
-#    t = celery.group(sigs)()
-#    txs = t.get()
-#    assert t.successful()
-#
-#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
-#    c = RpcClient(default_chain_spec)
-#    api = AdminApi(c, queue=None)
-#    address = init_eth_account_roles['eth_account_gas_provider']
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['network'] == 0
-#    assert nonce_spec['nonce']['queue'] == 4
-#    assert nonce_spec['nonce']['blocking'] == None
-#
-#    s_set = celery.signature(
-#            'cic_eth.queue.tx.set_rejected',
-#            [
-#                tx_hash.hex(),
-#                ],
-#            queue=None,
-#            )
-#    t = s_set.apply_async()
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['blocking'] == 2
-#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
-#
-#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#    for tx in txs[3:]:
-#        tx_hash = web3.Web3.keccak(hexstr=tx)
-#        tx_dict = get_tx(tx_hash.hex())
-#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
-#
-#
-
-
 def test_have_account(
     default_chain_spec,
     custodial_roles,
@@ -243,28 +113,6 @@ def test_tag_account(
     assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
-#def test_ready(
-#    init_database,
-#    agent_roles,
-#    eth_rpc,
-#    ):
-#
-#    api = AdminApi(eth_rpc)
-#   
-#    with pytest.raises(InitializationError):
-#        api.ready()
-#
-#    bogus_account = os.urandom(20)
-#    bogus_account_hex = '0x' + bogus_account.hex()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
-#    with pytest.raises(KeyError):
-#        api.ready()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
-#    api.ready()
-
-
 def test_tx(
     default_chain_spec,
     cic_registry,
@@ -286,3 +134,168 @@ def test_tx(
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
     tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
     logg.warning('code missing to verify tx contents {}'.format(tx))
+
+
+def test_check_nonce_gap(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        if i == 3:
+            j = 1
+            nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
+
+        queue_create(
+                default_chain_spec,
+                i+j,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
+
+
+def test_check_nonce_localfail(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    set_ready(default_chain_spec, tx_hashes[4], session=init_database)
+    set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
+    set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[4]
+
+
+def test_fix_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        init_celery_tasks,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
+    r = t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+    
+    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
+    ks = txs.keys()
+    assert len(ks) == 2
+    for k in ks:
+        hsh = add_0x(k)
+        otx = Otx.load(hsh, session=init_database)
+        init_database.refresh(otx)
+        logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
+        if add_0x(k) == tx_hashes[3]:
+            assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+        else:
+            assert otx.status == 1

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -0,0 +1,373 @@
+# standard imports
+import logging
+import io
+import json
+
+# external imports
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        nonce,
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.tx import (
+        raw,
+        receipt,
+        TxFormat,
+        Tx,
+        )
+from chainlib.eth.block import block_latest
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        set_sent,
+        set_ready,
+        )
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.query import get_nonce_tx_cache
+from eth_erc20 import ERC20
+from cic_eth_registry import CICRegistry
+
+# local imports
+from cic_eth.api.api_admin import AdminApi
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.eth.erc20 import cache_transfer_data
+
+logg = logging.getLogger()
+
+
+def test_admin_api_tx(
+        default_chain_spec,
+        init_database,
+        init_celery_tasks,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        custodial_roles,
+        celery_session_worker,
+        foo_token,
+        address_declarator,
+        cic_registry,
+        register_tokens,
+        register_lookups,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
+
+    o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
+    r = eth_rpc.do(o)
+    gifter_nonce = int(r, 16)
+
+    #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            gifter_nonce, # will only work if agent starts at 0
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_transfer_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    # lookup by transaction hash, without registry
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
+    logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by RLP transaction, without registry
+    tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, with registry
+    registry = CICRegistry(default_chain_spec, eth_rpc)
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, using writer
+    buf = io.StringIO()
+    api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex 
+
+
+def test_admin_api_account(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes_alice = []
+    txs_alice = []
+
+    for i in range(3):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_alice.append(tx_hash_hex)
+        txs_alice.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
+    tx_hashes_bob = []
+    txs_bob = []
+
+    for i in range(2):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                13+i,
+                agent_roles['BOB'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_bob.append(tx_hash_hex)
+        txs_bob.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'])
+    assert len(r) == 5
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
+    assert len(r) == 2
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
+    assert len(r) == 3
+
+
+def test_admin_api_account_writer(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    buf = io.StringIO()
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
+
+    # TODO: improve eval
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex
+
+
+def test_registry(
+        eth_rpc,
+        cic_registry,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.registry()
+    r = t.get_leaf()
+    assert r == cic_registry
+
+
+def test_proxy_do(
+        default_chain_spec,
+        eth_rpc,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.proxy_do(default_chain_spec, o)
+    rr = t.get_leaf()
+
+    assert r == rr
+
+
+def test_resend_inplace(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
+    r = t.get_leaf()
+    assert t.successful()
+
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 2
+
+
+
+@pytest.mark.xfail()
+def test_resend_clone(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
+    r = t.get_leaf()
+    assert t.successful()
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    otx = Otx.load(txs[0], session=init_database)
+    assert otx.status == 0

apps/cic-eth/tests/task/api/test_app.py

@@ -8,11 +8,20 @@ import pytest
 import celery
 from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
+from eth_accounts_index import AccountsIndex
+from chainlib.eth.tx import (
+        transaction,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        )
 
 # local imports
 from cic_eth.api import Api
+from cic_eth.queue.query import get_tx
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 def test_account_api(
@@ -29,6 +38,47 @@ def test_account_api(
     assert t.successful()
 
 
+def test_account_api_register(
+        default_chain_spec,
+        init_database,
+        account_registry,
+        faucet,
+        custodial_roles,
+        cic_registry,
+        register_lookups,
+        eth_rpc,
+        celery_session_worker,
+        ):
+    api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
+    t = api.create_account('')
+    register_tx_hash = t.get_leaf()
+    assert t.successful()
+
+    set_reserved(default_chain_spec, register_tx_hash, session=init_database)
+
+    tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                [tx['signed_tx']],
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = transaction(register_tx_hash)
+    tx_src = eth_rpc.do(o)
+
+    c = AccountsIndex(default_chain_spec)
+    address = c.parse_add_request(tx_src['data'])
+    o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    assert c.parse_have(r)
+
+
 def test_transfer_api(
         default_chain_spec,
         eth_rpc,
@@ -37,16 +87,15 @@ def test_transfer_api(
         custodial_roles,
         agent_roles,
         cic_registry,
-        register_tokens,
+        token_registry,
         register_lookups,
         celery_session_worker,
+        register_tokens,
+        foo_token_symbol,
         ):
 
-    #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
-    foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
-
     api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol)
+    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) 
     t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -0,0 +1,19 @@
+# local imports
+from cic_eth.api.api_task import Api
+from cic_eth.task import BaseTask
+
+def test_default_token(
+        default_chain_spec,
+        foo_token,
+        default_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None)     
+    t = api.default_token()
+    r = t.get_leaf()
+    assert r['address'] == foo_token

apps/cic-eth/tests/task/test_task_account.py

@@ -156,6 +156,7 @@ def test_gift(
     eth_signer,
     init_celery_tasks,
     cic_registry,
+    register_lookups,
     celery_session_worker,
     ):
 

apps/cic-eth/tests/task/test_task_admin.py

@@ -0,0 +1,88 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.sql.tx import (
+        create as queue_create,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import TxFormat
+from chainqueue.sql.query import get_nonce_tx_cache
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from hexathon import add_0x
+
+# local imports
+from cic_eth.admin.nonce import shift_nonce
+from cic_eth.eth.gas import cache_gas_data
+
+logg = logging.getLogger()
+
+
+def test_shift_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.admin.nonce.shift_nonce',
+            [
+                default_chain_spec.asdict(),
+                tx_hashes[3],
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+
+    for i in range(42+3, 42+10):
+        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
+        for k in txs.keys():
+            hsh = add_0x(k)
+            otx = Otx.load(hsh, session=init_database)
+            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
+            if add_0x(k) == tx_hashes[i-42]:
+                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+            else:
+                assert otx.status == 1

apps/cic-eth/tests/task/test_task_gas.py

@@ -0,0 +1,286 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        unpack,
+        TxFormat,
+        )
+from chainlib.eth.constant import (
+        MINIMUM_FEE_UNITS,
+        MINIMUM_FEE_PRICE,
+        )
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.query import get_tx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.state import (
+        set_ready,
+        set_reserved,
+        set_sent,
+        )
+from chainqueue.db.models.otx import Otx
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.error import OutOfGasError
+
+logg = logging.getLogger()
+
+
+def test_task_check_gas_ok(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                8000000,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+def test_task_check_gas_insufficient(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(whoever, 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                None,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    try:
+        r = t.get_leaf()
+    except OutOfGasError:
+        pass
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+
+def test_task_check_gas_low(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
+    nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
+    r = eth_rpc.do(o)
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) 
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                ],
+                [],
+                None,
+                None,
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+@pytest.mark.parametrize(
+        '_gas_price,_gas_factor',
+        [
+        (None, 1.1),
+        (MINIMUM_FEE_PRICE * 1.1, 0.9),
+        (None, 1.3),
+        ]
+        )
+def test_task_resend_explicit(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        _gas_price,
+        _gas_factor,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+
+    init_database.commit()
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    s = celery.signature(
+            'cic_eth.eth.gas.resend_with_higher_gas',
+            [
+                tx_hash_hex,
+                default_chain_spec.asdict(),
+                _gas_price,
+                _gas_factor,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(Otx)
+    q = q.filter(Otx.tx_hash==strip_0x(r))
+    otx = q.first()
+    if otx == None:
+        raise NotLocalTxError(r)
+
+    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
+    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
+    assert tx_after['gasPrice'] > tx_before['gasPrice']
+

apps/cic-eth/tests/task/test_task_tx.py

@@ -4,16 +4,27 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.gas import Gas
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
         transaction,
         receipt,
+        raw,
         )
 from hexathon import strip_0x
 from chainqueue.db.models.otx import Otx
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        set_sent,
+        )
+from chainqueue.db.enum import StatusBits
 
 # local imports
 from cic_eth.queue.tx import register_tx
@@ -60,15 +71,6 @@ def test_tx_send(
 
 
 def test_sync_tx(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-    pass
-
-
-def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,
         eth_rpc,
@@ -77,31 +79,48 @@ def test_resend_with_higher_gas(
         celery_session_worker,
         ):
 
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
-    register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
-    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
-    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    o = raw(tx_signed_raw_hex)
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
 
     s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
+            'cic_eth.eth.tx.sync_tx',
             [
                 tx_hash_hex,
                 default_chain_spec.asdict(),
                 ],
-            queue=None,
+            queue=None
             )
     t = s.apply_async()
     r = t.get_leaf()
+    assert t.successful() 
 
-    q = init_database.query(Otx)
-    q = q.filter(Otx.tx_hash==strip_0x(r))
-    otx = q.first()
-    if otx == None:
-        raise NotLocalTxError(r)
-
-    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
-    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
-    assert tx_after['gasPrice'] > tx_before['gasPrice']
+    init_database.commit()
 
+    o = Otx.load(tx_hash_hex, session=init_database)
+    assert o.status & StatusBits.FINAL == StatusBits.FINAL

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -0,0 +1,170 @@
+# standard imports
+import os
+import logging
+
+# external imports
+import pytest
+import celery
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        receipt,
+        )
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        )
+
+logg = logging.getLogger()
+
+
+def test_hashes_to_txs(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+
+    #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    init_database.commit()
+
+    bogus_one = add_0x(os.urandom(32).hex())
+    bogus_two = add_0x(os.urandom(32).hex())
+
+    yarrgs = [
+                bogus_one,
+                tx_hash_hex_two,
+                bogus_two,
+                tx_hash_hex_one,
+            ]   
+    s = celery.signature(
+        'cic_eth.eth.tx.hashes_to_txs',
+        [
+            yarrgs,
+                ],
+        queue=None,
+        )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    assert len(r) == 2
+
+    logg.debug('r {}'.format(r))
+    txs = [
+        tx_signed_raw_hex_two,
+        tx_signed_raw_hex_one,
+            ]
+    for tx in r:
+        txs.remove(add_0x(tx))
+    assert len(txs) == 0
+
+
+
+def test_double_send(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+    set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
+    init_database.commit()
+
+    yarrgs = [
+        tx_signed_raw_hex_one,
+        tx_signed_raw_hex_two,
+            ]
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                yarrgs,
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = receipt(tx_hash_hex_one)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    o = receipt(tx_hash_hex_two)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+
+

apps/cic-eth/tests/testdata/Bogus.bin

@@ -0,0 +1 @@
+60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

apps/cic-eth/tests/testdata/Bogus.sol

@@ -0,0 +1,10 @@
+pragma solidity ^0.8.0;
+
+contract Bogus {
+
+	uint256 public a = 0;
+
+	function poke() public {
+		a++;
+	}
+}

apps/cic-eth/tests/unit/ext/test_address.py

@@ -19,6 +19,7 @@ def test_translate(
         agent_roles,
         cic_registry,
         init_celery_tasks,
+        register_lookups,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)

apps/cic-eth/tests/unit/test_registry_connect.py

@@ -0,0 +1,22 @@
+# local imports
+from cic_eth.registry import *
+
+def test_registry_connect(
+        eth_rpc,
+        default_chain_spec,
+        address_declarator,
+        token_registry,
+        contract_roles,
+        purge_lookups,
+        registry,
+        agent_roles,
+        ):
+
+    r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+

apps/cic-eth/tests/unit/test_stat.py

@@ -0,0 +1,18 @@
+# standard imports
+import datetime
+
+# local imports
+from cic_eth.stat import init_chain_stat
+
+
+def test_chain_stat(
+        eth_rpc,
+        init_eth_tester,
+        ):
+
+    now = int(datetime.datetime.now().timestamp()) + 1
+    for i in range(11):
+        init_eth_tester.time_travel(now + (i * 2))
+    
+    s = init_chain_stat(eth_rpc, block_start=0)
+    assert s.block_average() == 2

apps/cic-meta/.gitignore

@@ -3,3 +3,5 @@ dist
 dist-web
 dist-server
 scratch
+coverage
+.nyc_output

apps/cic-meta/.gitlab-ci.yml

@@ -3,17 +3,38 @@
     variables:
         APP_NAME: cic-meta
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
 .cic_meta_changes_target:
     rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+#        - changes:
+#            - $CONTEXT/$APP_NAME/*
+        - when: always
 
-build-mr-cic-meta:
+cic-meta-build-mr:
+  stage: build
+  extends:
+    - .cic_meta_variables
+    - .cic_meta_changes_target
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
+
+test-mr-cic-meta:
     extends:
-        - .cic_meta_changes_target
-        - .py_build_merge_request
         - .cic_meta_variables
+        - .cic_meta_changes_target
+    stage: test
+    image: $IMAGE_TAG 
+    script:
+        - cd /tmp/src/cic-meta
+        - npm install --dev 
+        - npm run test
+        - npm run test:coverage
+    needs: ["cic-meta-build-mr"]
 
 build-push-cic-meta:
     extends:

apps/cic-meta/docker/Dockerfile

@@ -4,29 +4,28 @@ WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
-COPY cic-meta/package.json \
-     ./
-
+# required to build the cic-client-meta module
 COPY cic-meta/src/ src/
-COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
 
+# copy the dependencies
+COPY cic-meta/package.json .
+COPY cic-meta/tsconfig.json . 
+COPY cic-meta/webpack.config.js .
+     
 RUN npm install
 
-# see exports_dir gpg.ini
+COPY cic-meta/tests/ tests/
 COPY cic-meta/tests/*.asc /root/pgp/
-RUN alias tsc=node_modules/typescript/bin/tsc 
-
 
+# copy runtime configs
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-# COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
+# db migrations
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
-#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
-
+RUN alias tsc=node_modules/typescript/bin/tsc 
 COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/package.json

@@ -11,6 +11,7 @@
 	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
+		"test:coverage": "nyc mocha tests/*.ts --timeout 3000 --check-coverage=true",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
@@ -34,7 +35,9 @@
 	"devDependencies": {
 		"@types/mocha": "^8.0.3",
 		"mocha": "^8.2.0",
+		"nock": "^13.1.0",
 		"node-localstorage": "^2.1.6",
+		"nyc": "^15.1.0",
 		"ts-node": "^9.0.0",
 		"typescript": "^4.0.5",
 		"webpack": "^5.4.0",
@@ -50,5 +53,26 @@
 	"license": "GPL-3.0-or-later",
 	"engines": {
 		"node": ">=14.16.1"
+	},
+	"nyc": {
+		"include": [
+			"src/**/*.ts"
+		],
+		"extension": [
+			".ts"
+		],
+		"require": [
+			"ts-node/register"
+		],
+		"reporter": [
+			"text",
+			"html"
+		],
+		"sourceMap": true,
+		"instrument": true,
+		"branches": ">80",
+		"lines": ">80",
+		"functions": ">80",
+		"statements": ">80"
 	}
 }

apps/cic-meta/scripts/server/handlers.ts

@@ -31,7 +31,7 @@ function handleNoMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('mesage', e);
+				console.error('message', e);
 				doh(e);
 			});
 		})
@@ -46,7 +46,7 @@ function handleServerMergePost(data, db, digest, keystore, signer) {
 			let e = undefined;
 			let s = undefined;
 			if (v === undefined) {
-				s = new Syncable(digest, data);
+				s = new Syncable(digest, o);
 				s.onwrap = (e) => {
 					whohoo(e.toJSON());
 				};

apps/cic-meta/scripts/server/server.ts

@@ -204,7 +204,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty onctent', data);
+			console.error('empty content', data);
 			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/cic-meta/src/custom.ts

@@ -9,7 +9,7 @@ class Custom extends Syncable implements Addressable {
         super('', v);
         Custom.toKey(name).then((cid) => {
             this.id = cid;
-            this.value = v;
+            this.name = name;
         });
     }
 

apps/cic-meta/src/meta.ts

@@ -100,13 +100,15 @@ class Meta {
             identifier = await User.toKey(name);
         } else if (type === 'phone') {
             identifier = await Phone.toKey(name);
-        } else {
+        } else if (type === 'custom') {
             identifier = await Custom.toKey(name);
+        } else {
+            identifier = await Custom.toKey(name, type);
         }
         return identifier;
     }
 
-    private wrap(syncable: Syncable): Promise<Envelope> {
+    wrap(syncable: Syncable): Promise<Envelope> {
         return new Promise<Envelope>(async (resolve, reject) => {
             syncable.setSigner(this.signer);
             syncable.onwrap = async (env) => {

apps/cic-meta/tests/custom.ts

@@ -0,0 +1,49 @@
+import * as assert from 'assert';
+import {Custom} from "../src";
+
+const testName = 'areas';
+const testObject = {
+    area: ['Nairobi', 'Mombasa', 'Kilifi']
+}
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+describe('custom', () => {
+
+    context('with predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName, testObject);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, testObject);
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    context('without predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, {});
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    describe('#toKey()', () => {
+        context('without a custom identifier', () => {
+            it('should generate a key from the custom name', async () => {
+                assert.strictEqual(await Custom.toKey(testName), testNameKey);
+            });
+        });
+
+        context('with a custom identifier', () => {
+            it('should generate a key from the custom name with a custom identifier', async () => {
+                assert.strictEqual(await Custom.toKey(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/meta.ts

@@ -0,0 +1,176 @@
+import * as assert from 'assert';
+import * as fs from 'fs';
+const nock = require('nock');
+import {Meta} from "../src";
+import {getResponse, metaData, networkErrorResponse, notFoundResponse, putResponse} from "./response";
+import {Syncable} from "@cicnet/crdt-meta";
+
+const metaUrl = 'https://meta.dev.grassrootseconomics.net';
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+const testName = 'areas'
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+function readFile(filename) {
+    if(!fs.existsSync(filename)) {
+        console.error(`File ${filename} not found`);
+        return;
+    }
+    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
+}
+
+const privateKey = readFile('./privatekeys.asc');
+
+describe('meta', () => {
+    beforeEach(() => {
+        nock(metaUrl)
+            .get(`/${testAddressKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testPhoneKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .get(`/${testIdentifier}`)
+            .replyWithError(networkErrorResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddressKey}`)
+            .reply(200, putResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .post('/post')
+            .reply(500);
+    });
+
+    describe('#get()', () => {
+        it('should fetch data from the meta service', async () => {
+            const account = await Meta.get(testAddressKey, metaUrl);
+            assert.strictEqual(account.toJSON(account), getResponse.payload);
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testAddress, metaUrl);
+                assert.strictEqual(account, `404: Not Found`);
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testIdentifier, metaUrl);
+                assert.strictEqual(account, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+            });
+        });
+    })
+
+    describe('#set()', () => {
+        context('object data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testAddressKey, metaData);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('string data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testPhoneKey, testAddress);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testIdentifier, metaData);
+                    assert.strictEqual(response, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+                }
+            });
+        });
+    });
+
+    describe('#updateMeta()', () => {
+        it('should update data in the meta server', async () => {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.updateMeta(syncable, testAddressKey);
+                assert.strictEqual(response, putResponse);
+            }
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', () => {
+                const syncable = new Syncable(testAddress, metaData);
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.updateMeta(syncable, testAddress);
+                    assert.strictEqual(response, notFoundResponse);
+                }
+            });
+        });
+    });
+
+    describe('#wrap()', () => {
+        it('should sign a syncable object', function () {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.wrap(syncable);
+                assert.strictEqual(response.toJSON(), getResponse);
+            }
+        });
+    })
+
+    describe('#getIdentifier()', () => {
+        context('without type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName), testNameKey);
+            });
+        });
+
+        context('with user type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testAddress, 'user'), testAddressKey);
+            });
+        });
+
+        context('with phone type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testPhone, 'phone'), testPhoneKey);
+            });
+        });
+
+        context('with custom type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, 'custom'), testNameKey);
+            });
+        });
+
+        context('with unrecognised type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/phone.ts

@@ -0,0 +1,24 @@
+import * as assert from 'assert';
+import {Phone} from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+
+describe('phone', () => {
+
+    it('should create a phone object', () => {
+        const phone = new Phone(testAddress, testPhone);
+        setTimeout(() => {
+            assert.strictEqual(phone.address, testAddress);
+            assert.strictEqual(phone.m.data.msisdn, testPhone);
+            assert.strictEqual(phone.key(), testPhoneKey)
+        }, 0);
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the phone number', async () => {
+            assert.strictEqual(await Phone.toKey(testPhone), testPhoneKey);
+        });
+    });
+});

apps/cic-meta/tests/user.ts

@@ -0,0 +1,54 @@
+import * as assert from 'assert';
+
+import { User } from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testUser = {
+    user: {
+        firstName: 'Test',
+        lastName: 'User'
+    }
+}
+
+describe('user', () => {
+
+    context('without predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, '');
+                assert.strictEqual(user.m.data.user.lastName, '');
+            }, 0);
+        });
+    });
+
+    context('with predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress, testUser);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+                assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+            }, 0);
+        });
+    });
+
+    describe('#setName()', () => {
+        it('should set user\'s names to metadata', () => {
+            const user = new User(testAddress);
+            user.setName(testUser.user.firstName, testUser.user.lastName);
+            assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+            assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+        });
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the user\'s address', async () => {
+            assert.strictEqual(await User.toKey(testAddress), testAddressKey);
+        });
+    });
+});

apps/cic-meta/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "baseUrl": ".",
     "outDir": "./dist.browser",
-    "target": "es5",
+    "target": "es2015",
     "module": "commonjs",
     "moduleResolution": "node",
     "lib": ["es2016", "dom", "es5"],

apps/cic-notify/cic_notify/api.py

@@ -26,7 +26,7 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
         for q in qs[host]:
             if re.match(re_q, q['name']):
                 host_queues.append((host, q['name'],))
-   
+
     task_prefix_len = len(task_prefix)
     queue_tasks = []
     for (host, queue) in host_queues:
@@ -35,17 +35,18 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
             for task in tasks:
                 if len(task) >= task_prefix_len and task[:task_prefix_len] == task_prefix:
                     queue_tasks.append((queue, task,))
-    
+
     return queue_tasks
 
 
 class Api:
     # TODO: Implement callback strategy
-    def __init__(self, queue='cic-notify'):
+    def __init__(self, queue=None):
         """
         :param queue: The queue on which to execute notification tasks
         :type queue: str
         """
+        self.queue = queue
         self.sms_tasks = get_sms_queue_tasks(app)
         logg.debug('sms tasks {}'.format(self.sms_tasks))
 
@@ -61,13 +62,19 @@ class Api:
         """
         signatures = []
         for q in self.sms_tasks:
+
+            if not self.queue:
+                queue = q[0]
+            else:
+                queue = self.queue
+
             signature = celery.signature(
                     q[1],
                     [
                         message,
                         recipient,
                         ],
-                    queue=q[0],
+                    queue=queue,
                     )
             signatures.append(signature)
 

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -87,10 +87,18 @@ for key in config.store.keys():
         module = importlib.import_module(config.store[key])
         if key == 'TASKS_AFRICASTALKING':
             africastalking_notifier = module.AfricasTalkingNotifier
+
+            api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID')
+            logg.debug(f'SENDER ID VALUE IS: {api_sender_id}')
+
+            if not api_sender_id:
+                api_sender_id = None
+                logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}')
+
             africastalking_notifier.initialize(
                 config.get('AFRICASTALKING_API_USERNAME'),
                 config.get('AFRICASTALKING_API_KEY'),
-                config.get('AFRICASTALKING_API_SENDER_ID')
+                api_sender_id
             )
 
 

apps/cic-notify/cic_notify/tasks/sms/africastalking.py

@@ -56,7 +56,7 @@ class AfricasTalkingNotifier:
             response = self.api_client.send(message=message, recipients=[recipient])
             logg.debug(f'africastalking response no-sender-id {response}')
 
-        recipients = response.get('Recipients')
+        recipients = response.get('SMSMessageData').get('Recipients')
 
         if len(recipients) != 1:
             status = response.get('SMSMessageData').get('Message')

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.5')
+version = (0, 4, 0, 'alpha.6')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1 @@
-cic_base[full_graph]~=0.1.2a61
+cic_base[full_graph]==0.1.3a3+build.4aa03607

apps/cic-notify/setup.cfg

@@ -28,6 +28,7 @@ packages =
 	cic_notify
 	cic_notify.db
 	cic_notify.db.models
+	cic_notify.ext
 	cic_notify.tasks.sms
 	cic_notify.runnable
 scripts =

apps/cic-notify/test_requirements.txt

@@ -2,4 +2,3 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
-

apps/cic-ussd/.config/app.ini

@@ -5,6 +5,7 @@ LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
+SUPPORT_PHONE_NUMBER=0757628885
 
 [phone_number]
 REGION=KE

apps/cic-ussd/.config/test/app.ini

@@ -5,6 +5,7 @@ LOCALE_PATH=var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#
+SUPPORT_PHONE_NUMBER=0757628885
 
 [ussd]
 MENU_FILE=/usr/local/lib/python3.8/site-packages/cic_ussd/db/ussd_menu.json

apps/cic-ussd/.config/test/integration.ini

@@ -0,0 +1,4 @@
+[test]
+gift_value = 50.00
+server_url = http://localhost:63315/
+token_symbol = GFT

apps/cic-ussd/cic_ussd/db/enum.py

@@ -0,0 +1,9 @@
+# standard import
+from enum import IntEnum
+
+
+class AccountStatus(IntEnum):
+    PENDING = 1
+    ACTIVE = 2
+    LOCKED = 3
+    RESET = 4

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -1,19 +1,13 @@
 # standard imports
-from enum import IntEnum
-
-# third party imports
-from sqlalchemy import Column, Integer, String
 
 # local imports
+from cic_ussd.db.enum import AccountStatus
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.encoder import check_password_hash, create_password_hash
 
-
-class AccountStatus(IntEnum):
-    PENDING = 1
-    ACTIVE = 2
-    LOCKED = 3
-    RESET = 4
+# third party imports
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm.session import Session
 
 
 class Account(SessionBase):
@@ -30,6 +24,21 @@ class Account(SessionBase):
     account_status = Column(Integer)
     preferred_language = Column(String)
 
+    @staticmethod
+    def get_by_phone_number(phone_number: str, session: Session):
+        """Retrieves an account from a phone number.
+        :param phone_number: The E164 format of a phone number.
+        :type phone_number:str
+        :param session:
+        :type session:
+        :return: An account object.
+        :rtype: Account
+        """
+        session = SessionBase.bind_session(session=session)
+        account = session.query(Account).filter_by(phone_number=phone_number).first()
+        SessionBase.release_session(session=session)
+        return account
+
     def __init__(self, blockchain_address, phone_number):
         self.blockchain_address = blockchain_address
         self.phone_number = phone_number

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -238,13 +238,55 @@
             "description": "Menu to display a user's entire profile",
             "display_key": "ussd.kenya.display_user_metadata",
             "name": "display_user_metadata",
-            "parent": "account_management"
+            "parent": "metadata_management"
         },
         "41": {
             "description": "The recipient is not in the system",
             "display_key": "ussd.kenya.exit_invalid_recipient",
             "name": "exit_invalid_recipient",
             "parent": null
+        },
+        "42": {
+            "description": "Pin entry menu for changing name data.",
+            "display_key": "ussd.kenya.name_edit_pin_authorization",
+            "name": "name_edit_pin_authorization",
+            "parent": "metadata_management"
+        },
+        "43": {
+            "description": "Pin entry menu for changing gender data.",
+            "display_key": "ussd.kenya.gender_edit_pin_authorization",
+            "name": "gender_edit_pin_authorization",
+            "parent": "metadata_management"
+        },
+        "44": {
+            "description": "Pin entry menu for changing location data.",
+            "display_key": "ussd.kenya.location_edit_pin_authorization",
+            "name": "location_edit_pin_authorization",
+            "parent": "metadata_management"
+        },
+        "45": {
+            "description": "Pin entry menu for changing products data.",
+            "display_key": "ussd.kenya.products_edit_pin_authorization",
+            "name": "products_edit_pin_authorization",
+            "parent": "metadata_management"
+        },
+        "46": {
+            "description": "Pin confirmation for pin change.",
+            "display_key": "ussd.kenya.new_pin_confirmation",
+            "name": "new_pin_confirmation",
+            "parent": "metadata_management"
+        },
+        "47": {
+            "description": "Year of birth entry menu.",
+            "display_key": "ussd.kenya.enter_date_of_birth",
+            "name": "enter_date_of_birth",
+            "parent": "metadata_management"
+        },
+        "48": {
+            "description": "Pin entry menu for changing year of birth data.",
+            "display_key": "ussd.kenya.dob_edit_pin_authorization",
+            "name": "dob_edit_pin_authorization",
+            "parent": "metadata_management"
         }
 
     }

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -78,28 +78,27 @@ class MetadataRequestsHandler(Metadata):
         :param data: The data to be stored in the metadata server.
         :type data: dict|str
         """
-        data = json.dumps(data).encode('utf-8')
+        data = json.dumps(data)
         result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
         metadata_http_error_handler(result=result)
-        metadata = result.content
+        metadata = result.json()
         self.edit(data=metadata)
 
-    def edit(self, data: bytes):
+    def edit(self, data: Union[Dict, str]):
         """ This function is responsible for editing data in the metadata server corresponding to a unique pointer.
         :param data: The data to be edited in the metadata server.
-        :type data: bytes
+        :type data: dict
         """
         cic_meta_signer = Signer()
         signature = cic_meta_signer.sign_digest(data=data)
         algorithm = cic_meta_signer.get_operational_key().get('algo')
-        decoded_data = data.decode('utf-8')
         formatted_data = {
-            'm': data.decode('utf-8'),
+            'm': json.dumps(data),
             's': {
                 'engine': self.engine,
                 'algo': algorithm,
                 'data': signature,
-                'digest': json.loads(data).get('digest'),
+                'digest': data.get('digest'),
             }
         }
         formatted_data = json.dumps(formatted_data)
@@ -110,19 +109,32 @@ class MetadataRequestsHandler(Metadata):
             decoded_identifier = self.identifier.decode("utf-8")
         except UnicodeDecodeError:
             decoded_identifier = self.identifier.hex()
-        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {decoded_data}.')
+        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
 
     def query(self):
-        """This function is responsible for querying the metadata server for data corresponding to a unique pointer."""
+        """
+        :return:
+        :rtype:
+        """
+        # retrieve the  metadata
         result = make_request(method='GET', url=self.url)
         metadata_http_error_handler(result=result)
-        response_data = result.json()
-        data = json.loads(response_data)
-        if not isinstance(data, dict):
-            raise ValueError(f'Invalid data object: {data}.')
+
+        # json serialize retrieved data
+        result_data = result.json()
+
+        # validate result data format
+        if not isinstance(result_data, dict):
+            raise ValueError(f'Invalid result data object: {result_data}.')
+
         if result.status_code == 200 and self.cic_type == ':cic.person':
+            # validate person metadata
             person = Person()
-            deserialized_person = person.deserialize(person_data=data)
-            data = json.dumps(deserialized_person.serialize())
-            cache_data(self.metadata_pointer, data=data)
+            person_data = person.deserialize(person_data=result_data)
+
+            # format new person data for caching
+            data = json.dumps(person_data.serialize())
+
+            # cache metadata
+            cache_data(key=self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class CustomMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.custom', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PreferencesMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.preferences', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/signer.py

@@ -47,14 +47,13 @@ class Signer:
         logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
         return gpg_keys[0]
 
-    def sign_digest(self, data: bytes):
+    def sign_digest(self, data: dict):
         """
         :param data:
         :type data:
         :return:
         :rtype:
         """
-        data = json.loads(data)
         digest = data['digest']
         key_id = self.get_operational_key().get('keyid')
         signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)

apps/cic-ussd/cic_ussd/operations.py

@@ -6,11 +6,13 @@ import logging
 import celery
 import i18n
 from cic_eth.api.api_task import Api
+from sqlalchemy.orm.session import Session
 from tinydb.table import Document
 from typing import Optional
 
 # local imports
 from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
@@ -22,15 +24,18 @@ from cic_ussd.validator import check_known_user, validate_response_type
 logg = logging.getLogger()
 
 
-def add_tasks_to_tracker(task_uuid):
-    """
-    This function takes tasks spawned over api interfaces and records their creation time for tracking.
+def add_tasks_to_tracker(session, task_uuid: str):
+    """This function takes tasks spawned over api interfaces and records their creation time for tracking.
+    :param session:
+    :type session:
     :param task_uuid: The uuid for an initiated task.
     :type task_uuid: str
     """
+    session = SessionBase.bind_session(session=session)
     task_record = TaskTracker(task_uuid=task_uuid)
-    TaskTracker.session.add(task_record)
-    TaskTracker.session.commit()
+    session.add(task_record)
+    session.flush()
+    SessionBase.release_session(session=session)
 
 
 def define_response_with_content(headers: list, response: str) -> tuple:
@@ -95,6 +100,7 @@ def create_or_update_session(
         service_code: str,
         user_input: str,
         current_menu: str,
+        session,
         session_data: Optional[dict] = None) -> InMemoryUssdSession:
     """
     Handles the creation or updating of session as necessary.
@@ -108,12 +114,15 @@ def create_or_update_session(
     :type user_input: str
     :param current_menu: Menu name that is currently being displayed on the ussd session
     :type current_menu: str
+    :param session:
+    :type session:
     :param session_data: Any additional data that was persisted during the user's interaction with the system.
     :type session_data: dict.
     :return: ussd session object
     :rtype: InMemoryUssdSession
     """
-    existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
+    session = SessionBase.bind_session(session=session)
+    existing_ussd_session = session.query(UssdSession).filter_by(
         external_session_id=external_session_id).first()
 
     if existing_ussd_session:
@@ -132,20 +141,25 @@ def create_or_update_session(
             current_menu=current_menu,
             session_data=session_data
         )
+    SessionBase.release_session(session=session)
     return ussd_session
 
 
-def get_account_status(phone_number) -> str:
+def get_account_status(phone_number, session: Session) -> str:
     """Get the status of a user's account.
     :param phone_number: The phone number to be checked.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: The user account status.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    status = user.get_account_status()
-    Account.session.add(user)
-    Account.session.commit()
+    session = SessionBase.bind_session(session=session)
+    account = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    status = account.get_account_status()
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
 
     return status
 
@@ -165,6 +179,7 @@ def initiate_account_creation_request(chain_str: str,
                                       external_session_id: str,
                                       phone_number: str,
                                       service_code: str,
+                                      session,
                                       user_input: str) -> str:
     """This function issues a task to create a blockchain account on cic-eth. It then creates a record of the ussd
     session corresponding to the creation of the account and returns a response denoting that the user's account is
@@ -177,6 +192,8 @@ def initiate_account_creation_request(chain_str: str,
     :type phone_number: str
     :param service_code: The service code dialed.
     :type service_code: str
+    :param session:
+    :type session:
     :param user_input: The input entered by the user.
     :type user_input: str
     :return: A response denoting that the account is being created.
@@ -190,7 +207,7 @@ def initiate_account_creation_request(chain_str: str,
     creation_task_id = cic_eth_api.create_account().id
 
     # record task initiation time
-    add_tasks_to_tracker(task_uuid=creation_task_id)
+    add_tasks_to_tracker(task_uuid=creation_task_id, session=session)
 
     # cache account creation data
     cache_account_creation_task_id(phone_number=phone_number, task_id=creation_task_id)
@@ -204,6 +221,7 @@ def initiate_account_creation_request(chain_str: str,
         phone=phone_number,
         service_code=service_code,
         current_menu=current_menu.get('name'),
+        session=session,
         user_input=user_input)
 
     # define response to relay to user
@@ -268,12 +286,14 @@ def cache_account_creation_task_id(phone_number: str, task_id: str):
     redis_cache.persist(name=task_id)
 
 
-def process_current_menu(ussd_session: Optional[dict], user: Account, user_input: str) -> Document:
+def process_current_menu(account: Account, session: Session, ussd_session: Optional[dict], user_input: str) -> Document:
     """This function checks user input and returns a corresponding ussd menu
     :param ussd_session: An in db ussd session object.
     :type ussd_session: UssdSession
-    :param user: A user object.
-    :type user: Account
+    :param account: A account object.
+    :type account: Account
+    :param session:
+    :type session:
     :param user_input: The user's input.
     :type user_input: str
     :return: An in memory ussd menu object.
@@ -285,7 +305,13 @@ def process_current_menu(ussd_session: Optional[dict], user: Account, user_input
     else:
         # get current state
         latest_input = get_latest_input(user_input=user_input)
-        current_menu = process_request(ussd_session=ussd_session, user_input=latest_input, user=user)
+        session = SessionBase.bind_session(session=session)
+        current_menu = process_request(
+            account=account,
+            session=session,
+            ussd_session=ussd_session,
+            user_input=latest_input)
+        SessionBase.release_session(session=session)
     return current_menu
 
 
@@ -294,6 +320,7 @@ def process_menu_interaction_requests(chain_str: str,
                                       phone_number: str,
                                       queue: str,
                                       service_code: str,
+                                      session,
                                       user_input: str) -> str:
     """This function handles requests intended for interaction with ussd menu, it checks whether a user matching the
     provided phone number exists and in the absence of which it creates an account for the user.
@@ -308,25 +335,29 @@ def process_menu_interaction_requests(chain_str: str,
     :type queue: str
     :param service_code: The service dialed by the user making the request.
     :type service_code: str
+    :param session:
+    :type session:
     :param user_input: The inputs entered by the user.
     :type user_input: str
     :return: A response based on the request received.
     :rtype: str
     """
     # check whether the user exists
-    if not check_known_user(phone=phone_number):
+    if not check_known_user(phone_number=phone_number, session=session):
         response = initiate_account_creation_request(chain_str=chain_str,
                                                      external_session_id=external_session_id,
                                                      phone_number=phone_number,
                                                      service_code=service_code,
+                                                     session=session,
                                                      user_input=user_input)
 
     else:
-        # get user
-        user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+        # get account
+        session = SessionBase.bind_session(session=session)
+        account = Account.get_by_phone_number(phone_number=phone_number, session=session)
 
         # retrieve and cache user's metadata
-        blockchain_address = user.blockchain_address
+        blockchain_address = account.blockchain_address
         s_query_person_metadata = celery.signature(
             'cic_ussd.tasks.metadata.query_person_metadata',
             [blockchain_address]
@@ -334,24 +365,25 @@ def process_menu_interaction_requests(chain_str: str,
         s_query_person_metadata.apply_async(queue='cic-ussd')
 
         # find any existing ussd session
-        existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
-            external_session_id=external_session_id).first()
+        existing_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first()
 
         # validate user inputs
         if existing_ussd_session:
             current_menu = process_current_menu(
+                account=account,
+                session=session,
                 ussd_session=existing_ussd_session.to_json(),
-                user=user,
                 user_input=user_input
             )
         else:
             current_menu = process_current_menu(
+                account=account,
+                session=session,
                 ussd_session=None,
-                user=user,
                 user_input=user_input
             )
 
-        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session)
 
         if last_ussd_session:
             # create or update the ussd session as appropriate
@@ -361,6 +393,7 @@ def process_menu_interaction_requests(chain_str: str,
                 service_code=service_code,
                 user_input=user_input,
                 current_menu=current_menu.get('name'),
+                session=session,
                 session_data=last_ussd_session.session_data
             )
         else:
@@ -369,15 +402,17 @@ def process_menu_interaction_requests(chain_str: str,
                 phone=phone_number,
                 service_code=service_code,
                 user_input=user_input,
-                current_menu=current_menu.get('name')
+                current_menu=current_menu.get('name'),
+                session=session
             )
 
         # define appropriate response
         response = custom_display_text(
+            account=account,
             display_key=current_menu.get('display_key'),
             menu_name=current_menu.get('name'),
+            session=session,
             ussd_session=ussd_session.to_json(),
-            user=user
         )
 
     # check that the response from the processor is valid
@@ -386,21 +421,26 @@ def process_menu_interaction_requests(chain_str: str,
 
     # persist session to db
     persist_session_to_db_task(external_session_id=external_session_id, queue=queue)
+    SessionBase.release_session(session=session)
 
     return response
 
 
-def reset_pin(phone_number: str) -> str:
+def reset_pin(phone_number: str, session: Session) -> str:
     """Reset account status from Locked to Pending.
     :param phone_number: The phone number belonging to the account to be unlocked.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: The status of the pin reset.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    user.reset_account_pin()
-    Account.session.add(user)
-    Account.session.commit()
+    session = SessionBase.bind_session(session=session)
+    account = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    account.reset_account_pin()
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
 
     response = f'Pin reset for user {phone_number} is successful!'
     return response
@@ -438,11 +478,13 @@ def update_ussd_session(
     return session
 
 
-def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_session: dict):
+def save_to_in_memory_ussd_session_data(queue: str, session: Session, session_data: dict, ussd_session: dict):
     """This function is used to save information to the session data attribute of a ussd session object in the redis
     cache.
     :param queue: The queue on which the celery task should run.
     :type queue: str
+    :param session:
+    :type session:
     :param session_data: A dictionary containing data for a specific ussd session in redis that needs to be saved
     temporarily.
     :type session_data: dict
@@ -473,7 +515,7 @@ def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_ses
         service_code=in_redis_ussd_session.get('service_code'),
         user_input=in_redis_ussd_session.get('user_input'),
         current_menu=in_redis_ussd_session.get('state'),
+        session=session,
         session_data=session_data
     )
     persist_session_to_db_task(external_session_id=external_session_id, queue=queue)
-

apps/cic-ussd/cic_ussd/phone_number.py

@@ -8,6 +8,10 @@ import phonenumbers
 from cic_ussd.db.models.account import Account
 
 
+class E164Format:
+    region = None
+
+
 def process_phone_number(phone_number: str, region: str):
     """This function parses any phone number for the provided region
     :param phone_number: A string with a phone number.
@@ -29,15 +33,5 @@ def process_phone_number(phone_number: str, region: str):
 
     return parsed_phone_number
 
-
-def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
-    """This function queries the database for a user based on the provided phone number.
-    :param phone_number: A valid phone number.
-    :type phone_number: str
-    :return: A user object matching a given phone number
-    :rtype: Account|None
-    """
-    # consider adding region to user's metadata
-    phone_number = process_phone_number(phone_number=phone_number, region='KE')
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    return user
+class Support:
+    phone_number = None

apps/cic-ussd/cic_ussd/processor.py

@@ -1,25 +1,27 @@
 # standard imports
+import datetime
 import logging
 import json
-import re
 from typing import Optional
 
 # third party imports
-import celery
 from sqlalchemy import desc
 from cic_eth.api import Api
+from sqlalchemy.orm.session import Session
 from tinydb.table import Document
 
 # local imports
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
 from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession
-from cic_ussd.error import MetadataNotFoundError, SeppukuError
+from cic_ussd.db.enum import AccountStatus
+from cic_ussd.error import SeppukuError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
-from cic_ussd.phone_number import get_user_by_phone_number
+from cic_ussd.phone_number import Support
 from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.conversions import to_wei, from_wei
@@ -61,47 +63,48 @@ def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()):
             raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
 
 
-def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
-    """
-    This method provides translation for all ussd menu entries that follow the pin authorization pattern.
+def process_pin_authorization(account: Account, display_key: str, **kwargs) -> str:
+    """This method provides translation for all ussd menu entries that follow the pin authorization pattern.
+    :param account: The account in a running USSD session.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user in a running USSD session.
-    :type user: Account
     :param kwargs: Any additional information required by the text values in the internationalization files.
     :type kwargs
     :return: A string value corresponding the ussd menu's text value.
     :rtype: str
     """
     remaining_attempts = 3
-    if user.failed_pin_attempts > 0:
+    if account.failed_pin_attempts > 0:
         return translation_for(
             key=f'{display_key}.retry',
-            preferred_language=user.preferred_language,
-            remaining_attempts=(remaining_attempts - user.failed_pin_attempts)
+            preferred_language=account.preferred_language,
+            remaining_attempts=(remaining_attempts - account.failed_pin_attempts)
         )
     else:
         return translation_for(
             key=f'{display_key}.first',
-            preferred_language=user.preferred_language,
+            preferred_language=account.preferred_language,
             **kwargs
         )
 
 
-def process_exit_insufficient_balance(display_key: str, user: Account, ussd_session: dict):
+def process_exit_insufficient_balance(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes the exit menu letting users their account balance is insufficient to perform a specific
     transaction.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
     :rtype: str
     """
     # get account balance
-    operational_balance = get_cached_operational_balance(blockchain_address=user.blockchain_address)
+    operational_balance = get_cached_operational_balance(blockchain_address=account.blockchain_address)
 
     # compile response data
     user_input = ussd_session.get('user_input').split('*')[-1]
@@ -111,13 +114,13 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     token_symbol = retrieve_token_symbol()
 
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
 
     tx_recipient_information = define_account_tx_metadata(user=recipient)
 
     return translation_for(
         key=display_key,
-        preferred_language=user.preferred_language,
+        preferred_language=account.preferred_language,
         amount=from_wei(transaction_amount),
         token_symbol=token_symbol,
         recipient_information=tx_recipient_information,
@@ -125,12 +128,14 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     )
 
 
-def process_exit_successful_transaction(display_key: str, user: Account, ussd_session: dict):
+def process_exit_successful_transaction(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes the exit menu after a successful initiation for a transfer of tokens.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -139,13 +144,13 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount')))
     token_symbol = retrieve_token_symbol()
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
-    tx_sender_information = define_account_tx_metadata(user=user)
+    tx_sender_information = define_account_tx_metadata(user=account)
 
     return translation_for(
         key=display_key,
-        preferred_language=user.preferred_language,
+        preferred_language=account.preferred_language,
         transaction_amount=from_wei(transaction_amount),
         token_symbol=token_symbol,
         recipient_information=tx_recipient_information,
@@ -153,13 +158,15 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     )
 
 
-def process_transaction_pin_authorization(user: Account, display_key: str, ussd_session: dict):
+def process_transaction_pin_authorization(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes pin authorization where making a transaction is concerned. It constructs a
     pre-transaction response menu that shows the details of the transaction.
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
+    :param session:
+    :type session:
     :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's
     text values.
     :type ussd_session: UssdSession
@@ -168,16 +175,16 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     """
     # compile response data
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
-    tx_sender_information = define_account_tx_metadata(user=user)
+    tx_sender_information = define_account_tx_metadata(user=account)
 
     token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
     return process_pin_authorization(
-        user=user,
+        account=account,
         display_key=display_key,
         recipient_information=tx_recipient_information,
         transaction_amount=from_wei(transaction_amount),
@@ -186,14 +193,12 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     )
 
 
-def process_account_balances(user: Account, display_key: str, ussd_session: dict):
+def process_account_balances(user: Account, display_key: str):
     """
     :param user:
     :type user:
     :param display_key:
     :type display_key:
-    :param ussd_session:
-    :type ussd_session:
     :return:
     :rtype:
     """
@@ -251,12 +256,16 @@ def process_display_user_metadata(user: Account, display_key: str):
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
         cic_type=':cic.person'
     )
-    user_metadata = get_cached_data(key)
-    if user_metadata:
-        user_metadata = json.loads(user_metadata)
+    cached_metadata = get_cached_data(key)
+    if cached_metadata:
+        user_metadata = json.loads(cached_metadata)
         contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard'))
         logg.debug(f'{contact_data}')
         full_name = f'{contact_data.get("given")} {contact_data.get("family")}'
+        date_of_birth = user_metadata.get('date_of_birth')
+        year_of_birth = date_of_birth.get('year')
+        present_year = datetime.datetime.now().year
+        age = present_year - year_of_birth
         gender = user_metadata.get('gender')
         products = ', '.join(user_metadata.get('products'))
         location = user_metadata.get('location').get('area_name')
@@ -265,22 +274,38 @@ def process_display_user_metadata(user: Account, display_key: str):
             key=display_key,
             preferred_language=user.preferred_language,
             full_name=full_name,
+            age=age,
             gender=gender,
             location=location,
             products=products
         )
     else:
-        raise MetadataNotFoundError(f'Expected person metadata but found none in cache for key: {key}')
+        # TODO [Philip]: All these translations could be moved to translation files.
+        logg.warning(f'Expected person metadata but found none in cache for key: {key}')
+
+        absent = ''
+        if user.preferred_language == 'en':
+            absent = 'Not provided'
+        elif user.preferred_language == 'sw':
+            absent = 'Haijawekwa'
+
+        return translation_for(
+            key=display_key,
+            preferred_language=user.preferred_language,
+            full_name=absent,
+            gender=absent,
+            age=absent,
+            location=absent,
+            products=absent
+        )
 
 
-def process_account_statement(user: Account, display_key: str, ussd_session: dict):
+def process_account_statement(user: Account, display_key: str):
     """
     :param user:
     :type user:
     :param display_key:
     :type display_key:
-    :param ussd_session:
-    :type ussd_session:
     :return:
     :rtype:
     """
@@ -382,23 +407,26 @@ def process_start_menu(display_key: str, user: Account):
     )
 
 
-def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
+def retrieve_most_recent_ussd_session(phone_number: str, session: Session) -> UssdSession:
     # get last ussd session based on user phone number
-    last_ussd_session = UssdSession.session\
-        .query(UssdSession)\
+    session = SessionBase.bind_session(session=session)
+    last_ussd_session = session.query(UssdSession)\
         .filter_by(msisdn=phone_number)\
         .order_by(desc(UssdSession.created))\
         .first()
+    SessionBase.release_session(session=session)
     return last_ussd_session
 
 
-def process_request(user_input: str, user: Account, ussd_session: Optional[dict] = None) -> Document:
+def process_request(account: Account, session, user_input: str, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
     provided  session id exists in the database or whether the creation of a new ussd session object is warranted.
     It then returns the appropriate ussd menu text values.
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
+    :param session:
+    :type session:
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :param ussd_session: A JSON serialized in-memory ussd session object
@@ -406,22 +434,20 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
-    # retrieve metadata before any transition
-    key = generate_metadata_pointer(
-        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
-    )
-    person_metadata = get_cached_data(key=key)
 
     if ussd_session:
         if user_input == "0":
             return UssdMenu.parent_menu(menu_name=ussd_session.get('state'))
         else:
-            successive_state = next_state(ussd_session=ussd_session, user=user, user_input=user_input)
+            successive_state = next_state(
+                account=account,
+                session=session,
+                ussd_session=ussd_session,
+                user_input=user_input)
             return UssdMenu.find_by_name(name=successive_state)
     else:
-        if user.has_valid_pin():
-            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+        if account.has_valid_pin():
+            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session)
 
             if last_ussd_session:
                 # get last state
@@ -433,74 +459,108 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                     'exit_invalid_pin',
                     'exit_invalid_new_pin',
                     'exit_pin_mismatch',
-                    'exit_invalid_request'
-                ] and person_metadata is not None:
+                    'exit_invalid_request',
+                    'exit_successful_transaction'
+                ]:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)
         else:
-            if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name:
+            if account.failed_pin_attempts >= 3 and account.get_account_status() == AccountStatus.LOCKED.name:
                 return UssdMenu.find_by_name(name='exit_pin_blocked')
-            elif user.preferred_language is None:
+            elif account.preferred_language is None:
                 return UssdMenu.find_by_name(name='initial_language_selection')
             else:
                 return UssdMenu.find_by_name(name='initial_pin_entry')
 
 
-def next_state(ussd_session: dict, user: Account, user_input: str) -> str:
+def next_state(account: Account, session, ussd_session: dict, user_input: str) -> str:
     """This function navigates the state machine based on the ussd session object and user inputs it receives.
     It checks the user input and provides the successive state in the state machine. It then updates the session's
     state attribute with the new state.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :return: A string value corresponding the successive give a specific state in the state machine.
     """
     state_machine = UssdStateMachine(ussd_session=ussd_session)
-    state_machine.scan_data((user_input, ussd_session, user))
+    state_machine.scan_data((user_input, ussd_session, account, session))
     new_state = state_machine.state
 
     return new_state
 
 
+def process_exit_invalid_menu_option(display_key: str, preferred_language: str):
+    return translation_for(
+        key=display_key,
+        preferred_language=preferred_language,
+        support_phone=Support.phone_number
+    )
+
+
 def custom_display_text(
+        account: Account,
         display_key: str,
         menu_name: str,
-        ussd_session: dict,
-        user: Account) -> str:
+        session: Session,
+        ussd_session: dict) -> str:
     """This function extracts the appropriate session data based on the current menu name. It then inserts them as
     keywords in the i18n function.
+    :param account: The account in a running USSD session.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param menu_name: The name by which a specific menu can be identified.
     :type menu_name: str
-    :param user: The user in a running USSD session.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: A string value corresponding the ussd menu's text value.
     :rtype: str
     """
     if menu_name == 'transaction_pin_authorization':
-        return process_transaction_pin_authorization(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_transaction_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'exit_insufficient_balance':
-        return process_exit_insufficient_balance(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_exit_insufficient_balance(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'exit_successful_transaction':
-        return process_exit_successful_transaction(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_exit_successful_transaction(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'start':
-        return process_start_menu(display_key=display_key, user=user)
+        return process_start_menu(display_key=display_key, user=account)
     elif 'pin_authorization' in menu_name:
-        return process_pin_authorization(display_key=display_key, user=user)
+        return process_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session)
     elif 'enter_current_pin' in menu_name:
-        return process_pin_authorization(display_key=display_key, user=user)
+        return process_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session)
     elif menu_name == 'account_balances':
-        return process_account_balances(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_account_balances(display_key=display_key, user=account)
     elif 'transaction_set' in menu_name:
-        return process_account_statement(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_account_statement(display_key=display_key, user=account)
     elif menu_name == 'display_user_metadata':
-        return process_display_user_metadata(display_key=display_key, user=user)
+        return process_display_user_metadata(display_key=display_key, user=account)
+    elif menu_name == 'exit_invalid_menu_option':
+        return process_exit_invalid_menu_option(display_key=display_key, preferred_language=account.preferred_language)
     else:
-        return translation_for(key=display_key, preferred_language=user.preferred_language)
+        return translation_for(key=display_key, preferred_language=account.preferred_language)

apps/cic-ussd/cic_ussd/requests.py

@@ -8,9 +8,12 @@ from urllib.parse import urlparse, parse_qs
 
 # third-party imports
 from sqlalchemy import desc
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.db.enum import AccountStatus
 from cic_ussd.operations import get_account_status, reset_pin
 from cic_ussd.validator import check_known_user
 
@@ -72,24 +75,26 @@ def get_account_creation_callback_request_data(env: dict) -> tuple:
     return status, task_id, result
 
 
-def process_pin_reset_requests(env: dict, phone_number: str):
+def process_pin_reset_requests(env: dict, phone_number: str, session: Session):
     """This function processes requests that are responsible for the pin reset functionality. It processes GET and PUT
     requests responsible for returning an account's status and
     :param env: A dictionary of values representing data sent on the api.
     :type env: dict
     :param phone_number: The phone of the user whose pin is being reset.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: A response denoting the result of the request to reset the user's pin.
     :rtype: str
     """
-    if not check_known_user(phone=phone_number):
+    if not check_known_user(phone_number=phone_number, session=session):
         return f'No user matching {phone_number} was found.', '404 Not Found'
 
     if get_request_method(env) == 'PUT':
-        return reset_pin(phone_number=phone_number), '200 OK'
+        return reset_pin(phone_number=phone_number, session=session), '200 OK'
 
     if get_request_method(env) == 'GET':
-        status = get_account_status(phone_number=phone_number)
+        status = get_account_status(phone_number=phone_number, session=session)
         response = {
             'status': f'{status}'
         }
@@ -97,16 +102,18 @@ def process_pin_reset_requests(env: dict, phone_number: str):
         return response, '200 OK'
 
 
-def process_locked_accounts_requests(env: dict) -> tuple:
+def process_locked_accounts_requests(env: dict, session: Session) -> tuple:
     """This function authenticates staff requests and returns a serialized JSON formatted list of blockchain addresses
     of accounts for which the PIN has been locked due to too many failed attempts.
     :param env: A dictionary of values representing data sent on the api.
     :type env: dict
+    :param session:
+    :type session:
     :return: A tuple containing a serialized list of blockchain addresses for locked accounts and corresponding message
     for the response.
     :rtype: tuple
     """
-    logg.debug('Authentication requires integration with cic-auth')
+    session = SessionBase.bind_session(session=session)
     response = ''
 
     if get_request_method(env) == 'GET':
@@ -123,12 +130,14 @@ def process_locked_accounts_requests(env: dict) -> tuple:
             else:
                 limit = r[1]
 
-        locked_accounts = Account.session.query(Account.blockchain_address).filter(
+        locked_accounts = session.query(Account.blockchain_address).filter(
             Account.account_status == AccountStatus.LOCKED.value,
             Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all()
 
         # convert lists to scalar blockchain addresses
         locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts]
+
+        SessionBase.release_session(session=session)
         response = json.dumps(locked_accounts)
         return response, '200 OK'
     return response, '405 Play by the rules'

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -36,11 +36,8 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 # set up db
 data_source_name = dsn_from_config(config)
 SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
 
 
-# handle requests from CICADA
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
     :param env: Object containing server and request information
@@ -55,19 +52,24 @@ def application(env, start_response):
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
     headers = [('Content-Type', 'text/plain')]
 
+    # create session for the life time of http request
+    session = SessionBase.create_session()
+
     if get_request_endpoint(env) == '/pin':
         phone_number = get_query_parameters(env=env, query_name='phoneNumber')
         phone_number = quote_plus(phone_number)
-        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
+        response, message = process_pin_reset_requests(env=env, phone_number=phone_number, session=session)
         response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-        SessionBase.session.close()
+        session.commit()
+        session.close()
         start_response(message, headers)
         return [response_bytes]
 
     # handle requests for locked accounts
-    response, message = process_locked_accounts_requests(env=env)
+    response, message = process_locked_accounts_requests(env=env, session=session)
     response_bytes, headers = define_response_with_content(headers=headers, response=response)
     start_response(message, headers)
-    SessionBase.session.close()
+    session.commit()
+    session.close()
     return [response_bytes]