Remove commented code

data-seeding image updates and tweak meta import

See merge request grassrootseconomics/cic-internal-integration!212

.gitignore

@@ -8,3 +8,8 @@ gmon.out
 *.egg-info
 dist/
 build/
+**/*sqlite
+**/.nyc_output
+**/coverage
+**/.venv
+.idea

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -89,7 +89,7 @@ def process_transactions_all_data(session, env):
 
     offset = r[1]
     end = r[2]
-    if r[2] < r[1]:
+    if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,6 +16,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -28,10 +29,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
+from chainsyncer.driver.head import HeadSyncer
-        HeadSyncer,
+from chainsyncer.driver.history import HistorySyncer
-        HistorySyncer,
-        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -113,10 +112,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,12 +1,13 @@
-cic-base~=0.1.2b10
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.3a3
+erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,6 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-web3==5.12.2
+cic_base[full]==0.1.3a3+build.984b5cff
-cic-eth-registry~=0.5.5a3
+sarafu-faucet~=0.0.4a1
-cic-base[full]==0.1.2b8

apps/cic-eth/.coveragerc

@@ -5,3 +5,5 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
+	cic_eth/queue/state.py
+	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -3,33 +3,29 @@
         APP_NAME: cic-eth
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
 
-.cic_eth_changes_target:
+.cic_eth_mr_changes_target:
     rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-      - $CONTEXT/$APP_NAME/**/*
+          - $CONTEXT/$APP_NAME/**/*
       when: always
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .cic_eth_changes_target
+        - .cic_eth_mr_changes_target
         - .py_build_target_test
 
 test-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .cic_eth_changes_target
+        - .cic_eth_mr_changes_target
-        - .cic_eth_changes_target
     stage: test
-    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+    image: $IMAGE_TAG_BASE
     script:
         - cd apps/$APP_NAME/
-        - pytest tests/unit/ 
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
 
 build-push-cic-eth:
     extends:
         - .py_build_push
-        - .cic_eth_variables
-        
-

apps/cic-eth/MANIFEST.in

@@ -0,0 +1,2 @@
+include *requirements.txt
+

apps/cic-eth/admin_requirements.txt

@@ -0,0 +1,5 @@
+SQLAlchemy==1.3.20
+cic-eth-registry~=0.5.6a1
+hexathon~=0.0.1a7
+chainqueue~=0.0.2b5
+eth-erc20==0.0.10a2

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,11 +4,18 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
+from chainlib.connection import RPCConnection
-from chainqueue.query import get_tx
+from chainlib.eth.tx import (
-from chainqueue.state import set_cancel
+        unpack,
+        TxFactory,
+        )
+from chainlib.eth.gas import OverrideGasOracle
+from chainqueue.sql.query import get_tx
+from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
+from hexathon import strip_0x
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -21,13 +28,14 @@ from cic_eth.admin.ctrl import (
         )
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True)
+@celery_app.task(bind=True, base=BaseTask)
-def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
+def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -38,25 +46,29 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
+    chain_spec = ChainSpec.from_dict(chainspec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    session = BaseTask.session_func()
-    tx_brief = get_tx(tx_hash_orig_hex)
+    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
 
-    lock_queue(None, chain_str, address)
+    lock_queue(None, chain_spec.asdict(), address=address)
-    lock_send(None, chain_str, address)
+    lock_send(None, chain_spec.asdict(), address=address)
+
+    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -69,49 +81,57 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
+        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        del(tx_new['hash'])
+        tx_new['gas_price'] += 1
-        del(tx_new['hash_unsigned'])
+        tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
 
-        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
+        logg.debug('tx_new {}'.format(tx_new))
+
+        del(tx_new['hash'])
+        del(tx_new['hash_unsigned'])
+        del(tx_new['hashUnsigned'])
+
+        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
+        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            nonce=tx_new['nonce'],
+            tx_new['nonce'],
-            address=tx_new['from'],
+            tx_hash_hex,
-            tx_hash=tx_hash_hex,
+            tx_signed_raw_hex,
-            signed_tx=tx_signed_raw_hex,
+            )
-                )
         session.add(otx)
-        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(tx_previous_hash_hex, True)
+        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
+        session.commit()
 
     session.close()
 
-    s = create_check_gas_and_send_task(
+    s = create_check_gas_task(
          txs, 
-         chain_str,
+         chain_spec,
          tx_new['from'],
-         tx_new['gas'],
+         gas=tx_new['gas'],
-         tx_hashes, 
+         tx_hashes_hex=tx_hashes, 
-         queue,
+         queue=queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,
@@ -119,7 +139,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/cic_eth/api/__init__.py

@@ -5,4 +5,3 @@
 """
 
 from .api_task import Api
-from .api_admin import AdminApi

apps/cic-eth/cic_eth/api/admin.py

@@ -8,6 +8,7 @@ from chainlib.eth.constant import (
         ZERO_ADDRESS,
         )
 from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.contract import code
@@ -30,13 +31,14 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
+from chainqueue.sql.query import get_tx
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
-from cic_eth.queue.query import get_tx
 
 app = celery.current_app
 
@@ -188,6 +190,7 @@ class AdminApi:
         s_manual = celery.signature(
             'cic_eth.queue.state.set_manual',
             [
+                chain_spec.asdict(),
                 tx_hash_hex,
                 ],
             queue=self.queue,
@@ -206,8 +209,9 @@ class AdminApi:
             s.link(s_gas)
 
         return s_manual.apply_async()
-                        
+                    
-    def check_nonce(self, address):
+
+    def check_nonce(self, chain_spec, address):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -228,13 +232,12 @@ class AdminApi:
             s_get_tx = celery.signature(
                     'cic_eth.queue.query.get_tx',
                     [
-                    chain_spec.asdict(),
+                        chain_spec.asdict(),
                         k,
                         ],
                     queue=self.queue,
                     )
             tx = s_get_tx.apply_async().get()
-            #tx = get_tx(k)
             logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
             if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@@ -242,7 +245,9 @@ class AdminApi:
                 blocking_tx = k
                 blocking_nonce = nonce_otx
             elif nonce_otx - last_nonce > 1:
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from']))
+                logg.debug('tx {}'.format(tx))
+                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
+                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
                 blocking_tx = k
                 blocking_nonce = nonce_otx
                 break
@@ -256,12 +261,13 @@ class AdminApi:
                 'blocking': blocking_nonce,
             },
             'tx': {
-                'blocking': blocking_tx,
+                'blocking': add_0x(blocking_tx),
-                }
             }
+        }
 
 
-    def fix_nonce(self, address, nonce, chain_spec):
+    # TODO: is risky since it does not validate that there is actually a nonce problem?
+    def fix_nonce(self, chain_spec, address, nonce):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -275,15 +281,17 @@ class AdminApi:
         txs = s.apply_async().get()
 
         tx_hash_hex = None
+        session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(k)
+            tx_dict = get_tx(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
+        session.close()
 
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    self.rpc.chain_spec.asdict(),
+                    chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -388,12 +396,13 @@ class AdminApi:
     
         t = s.apply_async()
         tx = t.get()
-  
+
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
+            source_token_declaration = None
             if registry != None:
                 try:
-                    source_token = registry.by_address(tx['source_token'])
+                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
             else:
@@ -406,16 +415,21 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                source_token = t.get()
+                source_token_declaration = t.get()
-                if source_token == None:
+
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
+            if source_token_declaration != None:
+                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
+                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
+                logg.debug('source token set tup {}'.format(source_token))
+
 
 
         destination_token = None
         if tx['destination_token'] != ZERO_ADDRESS:
+            destination_token_declaration = None
             if registry != None:
                 try:
-                    destination_token = registry.by_address(tx['destination_token'])
+                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
             else:
@@ -428,10 +442,10 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                destination_token = t.get()
+                destination_token_declaration = t.get()
-                if destination_token == None:
+            if destination_token_declaration != None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
+                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
-
+                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
@@ -543,13 +557,19 @@ class AdminApi:
                 if role != None:
                     tx['recipient_description'] = role
 
+        erc20_c = ERC20(chain_spec)
         if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol()
+            tx['source_token_symbol'] = source_token.symbol
-            tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call()
+            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['sender_token_balance'] = erc20_c.parse_balance(r)
 
         if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol()
+            tx['destination_token_symbol'] = destination_token.symbol
-            tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call()
+            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
+            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
         tx['network_status'] = 'Not in node' 

apps/cic-eth/cic_eth/api/api_task.py

@@ -8,11 +8,10 @@ import logging
 
 # external imports 
 import celery
-from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 
 # local imports
-from cic_eth.db.enum import LockEnum
+from cic_eth.enum import LockEnum
 
 app = celery.current_app
 
@@ -74,29 +73,156 @@ class Api:
         return s_token.apply_async()
 
 
-    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param to_address: Ethereum address of receipient
+#        :type to_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_str,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    to_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
+#
+#
+#    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    from_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
+
+
+    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
+        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
 
         :param from_address: Ethereum address of sender
         :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of receipient
+        :param to_address: Ethereum address of recipient
         :type to_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
+        :param value: Estimated return from conversion
-        :type  target_return: int
+        :type  value: int
-        :param minimum_return: The least value of destination token return to allow
+        :param token_symbol: ERC20 token symbol of token to send
-        :type minimum_return: int
+        :type token_symbol: str
-        :param from_token_symbol: ERC20 token symbol of token being converted
+        :param spender_address: Ethereum address of recipient
-        :type from_token_symbol: str
+        :type spender_address: str, 0x-hex
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
         :returns: uuid of root task
         :rtype: celery.Task
         """
-        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
-                    [from_token_symbol, to_token_symbol],
+                    [token_symbol],
                     self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     from_address,
@@ -107,102 +233,51 @@ class Api:
                 'cic_eth.eth.nonce.reserve_nonce',
                 [
                     self.chain_spec.asdict(),
+                    from_address,
                     ],
                 queue=self.queue,
                 )
         s_tokens = celery.signature(
                 'cic_eth.eth.erc20.resolve_tokens_by_symbol',
                 [
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
-        s_convert = celery.signature(
+        s_allow = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
+                'cic_eth.eth.erc20.check_allowance',
+                [
+                    from_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_transfer = celery.signature(
+                'cic_eth.eth.erc20.transfer_from',
                 [
                     from_address,
-                    target_return,
-                    minimum_return,
                     to_address,
+                    value,
                     self.chain_spec.asdict(),
+                    spender_address,
                     ],
                 queue=self.queue,
                 )
+        s_tokens.link(s_allow)
         s_nonce.link(s_tokens)
         s_check.link(s_nonce)
         if self.callback_param != None:
-            s_convert.link(self.callback_success)
+            s_transfer.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
+            s_allow.link(s_transfer).on_error(self.callback_error)
         else:
-            s_tokens.link(s_convert)
+            s_allow.link(s_transfer)
 
         t = s_check.apply_async(queue=self.queue)
         return t
 
 
-    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
-        :type  target_return: int
-        :param minimum_return: The least value of destination token return to allow
-        :type minimum_return: int
-        :param from_token_symbol: ERC20 token symbol of token being converted
-        :type from_token_symbol: str
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        raise NotImplementedError('out of service until new DEX migration is done')
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [from_token_symbol, to_token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_convert = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
-                [
-                    from_address,
-                    target_return,
-                    minimum_return,
-                    from_address,
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_convert.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
-        else:
-            s_tokens.link(s_convert)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
-
 
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.

apps/cic-eth/cic_eth/db/enum.py

@@ -1,158 +1 @@
-# standard imports
+from cic_eth.enum import *
-import enum
-
-
-@enum.unique
-class StatusBits(enum.IntEnum):
-    """Individual bit flags that are combined to define the state and legacy of a queued transaction
-
-    """
-    QUEUED = 0x01 # transaction should be sent to network
-    IN_NETWORK = 0x08 # transaction is in network
-   
-    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
-    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
-
-    LOCAL_ERROR = 0x100 # errors that originate internally from the component
-    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
-    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
-    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
-
-    FINAL = 0x1000 # transaction processing has completed
-    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
-    MANUAL = 0x8000 # transaction processing has been manually overridden
-
-
-@enum.unique
-class StatusEnum(enum.IntEnum):
-    """
-
-    - Inactive, not finalized. (<0)
-        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
-        * SENDFAIL: The transaction was not received by the node.
-        * RETRY: The transaction is queued for a new send attempt after previously failing.
-        * READYSEND: The transaction is queued for its first send attempt
-        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
-        * WAITFORGAS: The transaction is on hold pending gas funding.
-    - Active state: (==0)
-        * SENT: The transaction has been sent to the mempool.
-    - Inactive, finalized. (>0)
-        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
-        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
-        * OVERRIDDEN: Transaction has been manually overriden.
-        * REJECTED: The transaction was rejected by the node.
-        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
-        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
-
-    """
-    PENDING = 0
-
-    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
-    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
-    READYSEND = StatusBits.QUEUED
-
-    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
-
-    WAITFORGAS = StatusBits.GAS_ISSUES
-
-    SENT = StatusBits.IN_NETWORK
-    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
-    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
-    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
-
-    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
-    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
-    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
-
-
-@enum.unique
-class LockEnum(enum.IntEnum):
-    """
-    STICKY: When set, reset is not possible
-    CREATE: Disable creation of accounts
-    SEND: Disable sending to network
-    QUEUE: Disable queueing new or modified transactions
-    """
-    STICKY=1
-    INIT=2
-    CREATE=4
-    SEND=8
-    QUEUE=16
-    QUERY=32
-    ALL=int(0xfffffffffffffffe)
-
-
-def status_str(v, bits_only=False):
-    """Render a human-readable string describing the status
-
-    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
-
-    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
-
-    :param v: Status bit field
-    :type v: number
-    :param bits_only: Only render individual bit labels.
-    :type bits_only: bool
-    :returns: Status string
-    :rtype: str
-    """
-    s = ''
-    if not bits_only:
-        try:
-            s = StatusEnum(v).name
-            return s
-        except ValueError:
-            pass
-
-    if v == 0:
-        return 'NONE'
-
-    for i in range(16):
-        b = (1 << i)
-        if (b & 0xffff) & v:
-            n = StatusBits(b).name
-            if len(s) > 0:
-                s += ','
-            s += n
-    if not bits_only:
-        s += '*'
-    return s
-
-
-def all_errors():
-    """Bit mask of all error states
-
-    :returns: Error flags
-    :rtype: number
-    """
-    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
-
-
-def is_error_status(v):
-    """Check if value is an error state
-
-    :param v: Status bit field
-    :type v: number
-    :returns: True if error
-    :rtype: bool
-    """
-    return bool(v & all_errors())
-
-
-def dead():
-    """Bit mask defining whether a transaction is still likely to be processed on the network.
-
-    :returns: Bit mask
-    :rtype: number
-    """
-    return StatusBits.FINAL | StatusBits.OBSOLETE
-
-
-def is_alive(v):
-    """Check if transaction is still likely to be processed on the network.
-
-    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
-
-    :returns: 
-    """
-    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/db/util.py

@@ -1,8 +0,0 @@
-import math
-
-def num_serialize(n):
-    if n == 0:
-        return b'\x00'
-    binlog = math.log2(n)
-    bytelength = int(binlog / 8 + 1)
-    return n.to_bytes(bytelength, 'big')

apps/cic-eth/cic_eth/enum.py

@@ -0,0 +1,158 @@
+# standard imports
+import enum
+
+
+@enum.unique
+class StatusBits(enum.IntEnum):
+    """Individual bit flags that are combined to define the state and legacy of a queued transaction
+
+    """
+    QUEUED = 0x01 # transaction should be sent to network
+    IN_NETWORK = 0x08 # transaction is in network
+   
+    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
+    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
+
+    LOCAL_ERROR = 0x100 # errors that originate internally from the component
+    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
+    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
+    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
+
+    FINAL = 0x1000 # transaction processing has completed
+    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
+    MANUAL = 0x8000 # transaction processing has been manually overridden
+
+
+@enum.unique
+class StatusEnum(enum.IntEnum):
+    """
+
+    - Inactive, not finalized. (<0)
+        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
+        * SENDFAIL: The transaction was not received by the node.
+        * RETRY: The transaction is queued for a new send attempt after previously failing.
+        * READYSEND: The transaction is queued for its first send attempt
+        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
+        * WAITFORGAS: The transaction is on hold pending gas funding.
+    - Active state: (==0)
+        * SENT: The transaction has been sent to the mempool.
+    - Inactive, finalized. (>0)
+        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
+        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
+        * OVERRIDDEN: Transaction has been manually overriden.
+        * REJECTED: The transaction was rejected by the node.
+        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
+        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
+
+    """
+    PENDING = 0
+
+    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
+    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
+    READYSEND = StatusBits.QUEUED
+
+    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
+
+    WAITFORGAS = StatusBits.GAS_ISSUES
+
+    SENT = StatusBits.IN_NETWORK
+    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
+    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
+    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
+
+    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
+    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
+    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
+
+
+@enum.unique
+class LockEnum(enum.IntEnum):
+    """
+    STICKY: When set, reset is not possible
+    CREATE: Disable creation of accounts
+    SEND: Disable sending to network
+    QUEUE: Disable queueing new or modified transactions
+    """
+    STICKY=1
+    INIT=2
+    CREATE=4
+    SEND=8
+    QUEUE=16
+    QUERY=32
+    ALL=int(0xfffffffffffffffe)
+
+
+def status_str(v, bits_only=False):
+    """Render a human-readable string describing the status
+
+    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
+
+    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
+
+    :param v: Status bit field
+    :type v: number
+    :param bits_only: Only render individual bit labels.
+    :type bits_only: bool
+    :returns: Status string
+    :rtype: str
+    """
+    s = ''
+    if not bits_only:
+        try:
+            s = StatusEnum(v).name
+            return s
+        except ValueError:
+            pass
+
+    if v == 0:
+        return 'NONE'
+
+    for i in range(16):
+        b = (1 << i)
+        if (b & 0xffff) & v:
+            n = StatusBits(b).name
+            if len(s) > 0:
+                s += ','
+            s += n
+    if not bits_only:
+        s += '*'
+    return s
+
+
+def all_errors():
+    """Bit mask of all error states
+
+    :returns: Error flags
+    :rtype: number
+    """
+    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
+
+
+def is_error_status(v):
+    """Check if value is an error state
+
+    :param v: Status bit field
+    :type v: number
+    :returns: True if error
+    :rtype: bool
+    """
+    return bool(v & all_errors())
+
+
+def dead():
+    """Bit mask defining whether a transaction is still likely to be processed on the network.
+
+    :returns: Bit mask
+    :rtype: number
+    """
+    return StatusBits.FINAL | StatusBits.OBSOLETE
+
+
+def is_alive(v):
+    """Check if transaction is still likely to be processed on the network.
+
+    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
+
+    :returns: 
+    """
+    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/error.py

@@ -80,3 +80,8 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
+
+
+class YouAreBrokeError(Exception):
+    """Exception raised when a value transfer is attempted without access to sufficient funds
+    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,6 +24,7 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
+        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -71,6 +72,117 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
+@celery_app.task(bind=True)
+def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
+    """Best-effort verification that the allowance for a transfer from spend is sufficient.
+
+    :raises YouAreBrokeError: If allowance is insufficient
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :return: Token list as passed to task
+    :rtype: dict
+    """
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    caller_address = ERC20Token.caller_address 
+    c = ERC20(chain_spec)
+    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
+    r = rpc.do(o)
+    allowance = c.parse_allowance(r)
+    if allowance < value:
+        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
+        logg.error(errstr)
+        raise YouAreBrokeError(errstr)
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
+    """Transfer ERC20 tokens between addresses
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    try:
+        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
+    except FileNotFoundError as e:
+        raise SignerError(e)
+    except ConnectionError as e:
+        raise SignerError(e)
+    
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -232,6 +344,7 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
+            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -279,6 +392,48 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_transfer_from_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_transfer_from
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+
+    tx_data = ERC20.parse_transfer_from_request(tx['data'])
+    spender_address = tx_data[0]
+    recipient_address = tx_data[1]
+    token_value = tx_data[2]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/eth/gas.py

@@ -57,10 +57,12 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+MAXIMUM_FEE_UNITS = 8000000
+
 class MaxGasOracle:
 
     def gas(code=None):
-        return 8000000
+        return MAXIMUM_FEE_UNITS
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -150,7 +152,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,24 +172,30 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    if len(txs) == 0:
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-        for i in range(len(tx_hashes)):
+    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
-            o = get_tx(tx_hashes[i])
+
-            txs.append(o['signed_tx'])
+    addresspass = None
-            if address == None:
+    if len(txs) == 0:
-                address = o['address']
+        addresspass = []
+        for i in range(len(tx_hashes)):
+            o = get_tx(chain_spec_dict, tx_hashes[i])
+            txs.append(o['signed_tx'])
+            logg.debug('sender {}'.format(o))
+            tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
+            if address == None:
+                address = tx['from']
+            elif address != tx['from']:
+                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
+            addresspass.append(address)
 
-    #if not web3.Web3.isChecksumAddress(address):
     if not is_checksum_address(address):
         raise ValueError('invalid address {}'.format(address))
 
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
-    # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     gas_balance = 0
     try:
         o = balance(address)
@@ -198,6 +206,9 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
         conn.disconnect()
         raise EthError('gas_balance call for {}: {}'.format(address, e))
 
+    if gas_required == None:
+        gas_required = MAXIMUM_FEE_UNITS
+
     logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
     session = SessionBase.create_session()
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@@ -268,7 +279,8 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
             queue=queue,
             )
         ready_tasks.append(s)
-    celery.group(ready_tasks)()
+    t = celery.group(ready_tasks)()
+    logg.debug('group {}'.format(t))
 
     return txs
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -21,6 +21,7 @@ from chainqueue.db.models.tx import Otx
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db import SessionBase
@@ -58,6 +59,9 @@ def hashes_to_txs(self, tx_hashes):
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
+    for i in range(len(tx_hashes)):
+        tx_hashes[i] = strip_0x(tx_hashes[i])
+
     queue = self.request.delivery_info['routing_key']
 
     session = SessionBase.create_session()
@@ -148,7 +152,7 @@ def send(self, txs, chain_spec_dict):
 
 @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
 def sync_tx(self, tx_hash_hex, chain_spec_dict):
-    """Force update of network status of a simgle transaction
+    """Force update of network status of a single transaction
 
     :param tx_hash_hex: Transaction hash
     :type tx_hash_hex: str, 0x-hex
@@ -173,12 +177,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
 
     # TODO: apply receipt in tx object to validate and normalize input
     if rcpt != None:
+        rcpt = snake_and_camel(rcpt)
         success = rcpt['status'] == 1
-        logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
+        logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
 
         s = celery.signature(
             'cic_eth.queue.state.set_final',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 rcpt['blockNumber'],
                 rcpt['transactionIndex'],
@@ -186,12 +192,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
                 ],
                 queue=queue,
             )
+    # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
     else:
         logg.debug('sync tx {} mempool'.format(tx_hash_hex))
 
         s = celery.signature(
             'cic_eth.queue.state.set_sent',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 ],
                 queue=queue,

apps/cic-eth/cic_eth/ext/tx.py

@@ -19,7 +19,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
-from chainqueue.query import get_tx_cache
+from chainqueue.sql.query import get_tx_cache
 from eth_erc20 import ERC20
 
 # local imports

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -22,7 +22,6 @@ def init_celery_tasks(
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
-#        'cic_eth.eth.bancor',
         'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
@@ -47,8 +46,8 @@ def celery_config():
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
     rq = tempfile.mkdtemp()
-    logg.debug('celery broker queue {} processed {}'.format(bq, bp))
+    logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
-    logg.debug('celery backend store {}'.format(rq))
+    logg.debug('celery backend session store {}'.format(rq))
     yield {
         'broker_url': 'filesystem://',
         'broker_transport_options': {
@@ -58,12 +57,11 @@ def celery_config():
             },
         'result_backend': 'file://{}'.format(rq),
             }
-    logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq))
+    logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
     shutil.rmtree(bq)
     shutil.rmtree(bp)
     shutil.rmtree(rq)
 
-
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
+root_dir = os.path.dirname(os.path.dirname(script_dir))
-logg = logging.getLogger(__file__)
+logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/cic_eth/pytest/fixtures_contract.py

@@ -0,0 +1,77 @@
+# standard imports
+import os
+
+# external imports
+import pytest
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        TxFactory,
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from hexathon import strip_0x
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+
+
+@pytest.fixture(scope='function')
+def bogus_tx_block(
+    default_chain_spec,
+    eth_rpc,
+    eth_signer,
+    contract_roles,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
+
+    f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
+    bytecode = f.read()
+    f.close()
+
+    c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
+    tx = c.set_code(tx, bytecode)
+    (tx_hash_hex, o) = c.build(tx)
+
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+
+    contract_address = r['contract_address']
+
+    enc = ABIContractEncoder()
+    enc.method('poke')
+    data = enc.get()
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
+    tx = c.set_code(tx, data)
+    (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
+    r = eth_rpc.do(o)
+    tx_signed_raw_hex = strip_0x(o['params'][0])
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    return (block, tx)

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -37,7 +37,8 @@ def init_database(
         database_engine,
         ):
 
-    rootdir = os.path.dirname(os.path.dirname(__file__))
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    rootdir = os.path.dirname(os.path.dirname(script_dir))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/cic_eth/pytest/testdata/Bogus.bin

@@ -0,0 +1 @@
+60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

apps/cic-eth/cic_eth/pytest/testdata/Bogus.sol

@@ -0,0 +1,10 @@
+pragma solidity ^0.8.0;
+
+contract Bogus {
+
+	uint256 public a = 0;
+
+	function poke() public {
+		a++;
+	}
+}

apps/cic-eth/cic_eth/queue/query.py

@@ -5,7 +5,7 @@ import datetime
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.query
+import chainqueue.sql.query
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -28,7 +28,7 @@ celery_app = celery.current_app
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r =  chainqueue.query.get_tx(chain_spec, tx_hash)
+    r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -46,7 +46,7 @@ def get_tx(chain_spec_dict, tx_hash):
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
+    r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
     session.close()
     return r
 
@@ -55,17 +55,17 @@ def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True,
 def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):

apps/cic-eth/cic_eth/queue/state.py

@@ -1,6 +1,6 @@
 # external imports
 from chainlib.chain import ChainSpec
-import chainqueue.state
+import chainqueue.sql.state
 
 # local imports
 import celery
@@ -14,7 +14,7 @@ celery_app = celery.current_app
 def set_sent(chain_spec_dict, tx_hash, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session)
+    r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
     session.close()
     return r
 
@@ -23,7 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
+    r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
     session.close()
     return r
 
@@ -32,7 +32,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
+    r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
     session.close()
     return r
 
@@ -41,7 +41,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 def set_rejected(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -50,7 +50,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 def set_fubar(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -59,7 +59,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 def set_manual(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -68,7 +68,7 @@ def set_manual(chain_spec_dict, tx_hash):
 def set_ready(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -77,7 +77,7 @@ def set_ready(chain_spec_dict, tx_hash):
 def set_reserved(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -86,7 +86,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 def set_waitforgas(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
     session.close()
     return r
     
@@ -95,7 +95,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 def get_state_log(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
+    r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -104,6 +104,6 @@ def get_state_log(chain_spec_dict, tx_hash):
 def obsolete(chain_spec_dict, tx_hash, final):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
+    r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
     session.close()
     return r

apps/cic-eth/cic_eth/queue/tx.py

@@ -15,14 +15,14 @@ from sqlalchemy import tuple_
 from sqlalchemy import func
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.state
+import chainqueue.sql.state
 from chainqueue.db.enum import (
         StatusEnum,
         StatusBits,
         is_alive,
         dead,
         )
-from chainqueue.tx import create
+from chainqueue.sql.tx import create
 from chainqueue.error import NotLocalTxError
 from chainqueue.db.enum import status_str
 

apps/cic-eth/cic_eth/registry.py

@@ -5,29 +5,30 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
 
 logg = logging.getLogger()
 
 
-def connect_token_registry(rpc, chain_spec):
+def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    token_registry_address = registry.by_name('TokenRegistry')
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
     logg.debug('using token registry address {}'.format(token_registry_address))
     lookup = TokenIndexLookup(chain_spec, token_registry_address)
     CICRegistry.add_lookup(lookup)
 
 
-def connect_declarator(rpc, chain_spec, trusted_addresses):
+def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator')
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
     logg.debug('using declarator address {}'.format(declarator_address))
     lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
     CICRegistry.add_lookup(lookup)
 
 
-def connect(rpc, chain_spec, registry_address):
+def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry')
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
     return registry
 

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -12,7 +12,7 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.address import is_checksum_address
 
 # local imports
-from cic_eth.api import AdminApi
+from cic_eth.api.admin import AdminApi
 from cic_eth.db.enum import LockEnum
 
 logging.basicConfig(level=logging.WARNING)

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -21,7 +21,7 @@ from chainqueue.db.enum import (
     StatusBits,
     )
 from chainqueue.error import NotLocalTxError
-from chainqueue.state import set_reserved
+from chainqueue.sql.state import set_reserved
 
 # local imports
 import cic_eth

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,14 +10,15 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.query import get_paused_tx_cache as get_paused_tx
+from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 
 class GasFilter(SyncFilter):
@@ -27,11 +28,11 @@ class GasFilter(SyncFilter):
         self.chain_spec = chain_spec
 
 
-    def filter(self, conn, block, tx, session):
+    def filter(self, conn, block, tx, db_session):
         if tx.value > 0:
             tx_hash_hex = add_0x(tx.hash)
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(session)
+            session = SessionBase.bind_session(db_session)
             q = session.query(TxCache.recipient)
             q = q.join(Otx)
             q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@@ -56,7 +57,7 @@ class GasFilter(SyncFilter):
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
-                s.apply_async()
+                return s.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -50,7 +50,8 @@ class RegistrationFilter(SyncFilter):
                     queue=self.queue,
                     )
                 s_nonce.link(s_gift)
-                s_nonce.apply_async()
+                t = s_nonce.apply_async()
+                return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/straggler.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from chainqueue.state import obsolete_by_cache
+from chainqueue.sql.state import obsolete_by_cache
 
 logg = logging.getLogger()
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
         self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
 
 
-    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
+    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
 
         if tx.payload == None:
             logg.debug('no payload')
@@ -45,16 +45,17 @@ class TransferAuthFilter(SyncFilter):
             return False
 
         recipient = tx.inputs[0]
-        if recipient != self.transfer_request_contract.address():
+        #if recipient != self.transfer_request_contract.address():
+        if recipient != self.transfer_request_contract:
             logg.debug('not our transfer auth contract address {}'.format(recipient))
             return False
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
-           
+          
-        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
+        sender = r[0]
-        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
+        recipient = r[1]
-        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
+        token = r[2]
-        value = abi_decode_single(ABIContractType.UINT256, r[3])
+        value = r[3]
 
         token_data = {
             'address': token,
@@ -64,6 +65,7 @@ class TransferAuthFilter(SyncFilter):
             'cic_eth.eth.nonce.reserve_nonce',
             [
                 [token_data],
+                self.chain_spec.asdict(), 
                 sender,
                 ],
             queue=self.queue,
@@ -80,7 +82,7 @@ class TransferAuthFilter(SyncFilter):
             )
         s_nonce.link(s_approve)
         t = s_nonce.apply_async()
-        return True
+        return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,136 +0,0 @@
-# standard imports
-import os
-import re
-import logging
-import argparse
-import json
-
-# third-party imports
-import web3
-import confini
-import celery
-from json.decoder import JSONDecodeError
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import dsn_from_config
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_eth', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-queue = args.q
-
-re_something = r'^/something/?'
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-def process_something(session, env):
-    r = re.match(re_something, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    #if env.get('CONTENT_TYPE') != 'application/json':
-    #    raise AttributeError('content type')
-
-    #if env.get('REQUEST_METHOD') != 'POST':
-    #    raise AttributeError('method')
-
-    #post_data = json.load(env.get('wsgi.input'))
-    
-    #return ('text/plain', 'foo'.encode('utf-8'),)
-
-
-# uwsgi application
-def application(env, start_response):
-    
-    for k in env.keys():
-        logg.debug('env {} {}'.format(k, env[k]))
-
-    headers = []
-    content = b''
-    err = None
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_something,
-            ]:
-        try:
-            r = handler(session, env)
-        except AttributeError as e:
-            logg.error('handler fail attribute {}'.format(e))
-            err = '400 Impertinent request'
-            break
-        except JSONDecodeError as e:
-            logg.error('handler fail json {}'.format(e))
-            err = '400 Invalid data format'
-            break
-        except KeyError as e:
-            logg.error('handler fail key {}'.format(e))
-            err = '400 Invalid JSON'
-            break
-        except ValueError as e:
-            logg.error('handler fail value {}'.format(e))
-            err = '400 Invalid data'
-            break
-        except RuntimeError as e:
-            logg.error('task fail value {}'.format(e))
-            err = '500 Task failed, sorry I cannot tell you more'
-            break
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    if err != None:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response(err, headers)
-        session.close()
-        return [content]
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,6 +194,7 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
+    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,10 +27,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
+from chainsyncer.driver.head import HeadSyncer
-        HeadSyncer,
+from chainsyncer.driver.history import HistorySyncer
-        HistorySyncer,
-        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,10 +12,8 @@ import confini
 import celery
 
 # local imports
-from cic_eth.api import (
+from cic_eth.api import Api
-        Api,
+from cic_eth.api.admin import AdminApi
-        AdminApi,
-        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

apps/cic-eth/cic_eth/runnable/resend.py

@@ -11,7 +11,7 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
 
 # local imports
-from cic_eth.api.api_admin import AdminApi
+from cic_eth.api.admin import AdminApi
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

apps/cic-eth/cic_eth/runnable/tag.py

@@ -12,7 +12,7 @@ from chainlib.chain import ChainSpec
 from xdg.BaseDirectory import xdg_config_home
 
 # local imports
-from cic_eth.api import AdminApi
+from cic_eth.api.admin import AdminApi
 from cic_eth.db import dsn_from_config
 from cic_eth.db.models.base import SessionBase
 

apps/cic-eth/cic_eth/runnable/view.py

@@ -19,7 +19,7 @@ from chainlib.eth.connection import EthHTTPConnection
 from hexathon import add_0x
 
 # local imports
-from cic_eth.api import AdminApi
+from cic_eth.api.admin import AdminApi
 from cic_eth.db.enum import (
     StatusEnum,
     status_str,

apps/cic-eth/cic_eth/stat.py

@@ -20,7 +20,11 @@ def init_chain_stat(rpc, block_start=0):
     if block_start == 0:
         o = block_latest()
         r = rpc.do(o)
-        block_start = int(r, 16)
+        try:
+            block_start = int(r, 16)
+        except TypeError:
+            block_start = int(r)
+        logg.debug('blockstart {}'.format(block_start))
 
     for i in range(BLOCK_SAMPLES):
         o = block_by_number(block_start-10+i)

apps/cic-eth/cic_eth/task.py

@@ -20,7 +20,8 @@ import liveness.linux
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 celery_app = celery.current_app
 
@@ -118,12 +119,13 @@ def registry():
     return CICRegistry.address
 
 
-@celery_app.task()
+@celery_app.task(bind=True, base=BaseTask)
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
+def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
+    r = registry.by_address(address, sender_address=self.call_address)
+    return r
 
 
 @celery_app.task(throws=(UnknownContractError,))
@@ -131,7 +133,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
+    return registry.by_name(name, sender_address=self.call_address)
 
 
 @celery_app.task()

apps/cic-eth/cic_eth/version.py

@@ -8,9 +8,9 @@ import semver
 
 version = (
         0,
-        11,
+        12,
         0,
-        'beta.14',
+        'alpha.1',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -8,13 +8,14 @@ RUN apt-get update && \
 #RUN python -m venv venv && . venv/bin/activate
 
 ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 RUN /usr/local/bin/python -m pip install --upgrade pip
 RUN pip install semver
 
-# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
 COPY cic-eth/ .
-RUN	pip install $pip_extra_index_url_flag .
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-RUN	pip install $pip_extra_index_url_flag pycryptodome==3.10.1
+    	--extra-index-url $EXTRA_INDEX_URL .
 
 # --- TEST IMAGE ---
 FROM python:3.8.6-slim-buster as test 
@@ -33,8 +34,12 @@ COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
 # COPY --from=compile /usr/src/cic-eth/ .
 # RUN . venv/bin/activate
 
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+
 COPY cic-eth/test_requirements.txt .
-RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
 
 COPY cic-eth . 
 

apps/cic-eth/requirements.txt

@@ -1,24 +1,3 @@
-cic-base~=0.1.2b11
 celery==4.4.7
-crypto-dev-signer~=0.4.14b3
+chainlib~=0.0.5a1
-confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a4
-redis==3.5.3
-alembic==1.4.2
-websockets==8.1
-requests~=2.24.0
-eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a6
-uWSGI==2.0.19.1
 semver==2.13.0
-websocket-client==0.57.0
-moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3a2
-hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a4
-chainqueue~=0.0.2a2
-sarafu-faucet==0.0.3a3
-erc20-faucet==0.2.1a4
-coincurve==15.0.0
-potaahto~=0.0.1a2

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/services_requirements.txt

@@ -0,0 +1,7 @@
+chainsyncer[sql]~=0.0.3a3
+chainqueue~=0.0.2b5
+alembic==1.4.2
+confini~=0.3.6rc4
+redis==3.5.3
+hexathon~=0.0.1a7
+pycryptodome==3.10.1

apps/cic-eth/setup.cfg

@@ -45,16 +45,16 @@ scripts =
 [options.entry_points]
 console_scripts =
 	# daemons
-	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main
+	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main [services]
-	cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main
+	cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main [services]
-	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main
+	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main [services]
-	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main
+	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main [services]
 	# tools	
-	cic-eth-create = cic_eth.runnable.create:main 
+	cic-eth-create = cic_eth.runnable.create:main [tools]
-	cic-eth-inspect = cic_eth.runnable.view:main
+	cic-eth-inspect = cic_eth.runnable.view:main [tools]
-	cic-eth-ctl = cic_eth.runnable.ctrl:main
+	cic-eth-ctl = cic_eth.runnable.ctrl:main [tools]
-	cic-eth-info = cic_eth.runnable.info:main
+	cic-eth-info = cic_eth.runnable.info:main [tools]
 	# TODO: Merge this with ctl when subcmds sorted to submodules
-	cic-eth-tag = cic_eth.runnable.tag:main
+	cic-eth-tag = cic_eth.runnable.tag:main [tools]
-	cic-eth-resend = cic_eth.runnable.resend:main
+	cic-eth-resend = cic_eth.runnable.resend:main [tools]
-	cic-eth-transfer = cic_eth.runnable.transfer:main
+	cic-eth-transfer = cic_eth.runnable.transfer:main [tools]

apps/cic-eth/setup.py

@@ -11,6 +11,41 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
+admin_requirements = []
+f = open('admin_requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    admin_requirements.append(l.rstrip())
+f.close()
+
+
+
+tools_requirements = []
+f = open('tools_requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    tools_requirements.append(l.rstrip())
+f.close()
+
+
+services_requirements = []
+f = open('services_requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    services_requirements.append(l.rstrip())
+f.close()
+
 setup(
-    install_requires=requirements
+    install_requires=requirements,
+    extras_require = {
+        'tools': tools_requirements,
+        'admin_api': admin_requirements,
+        'services': services_requirements,
+        }
         )

apps/cic-eth/test_requirements.txt

@@ -2,5 +2,8 @@ pytest==6.0.1
 pytest-celery==0.0.0a1
 pytest-mock==3.3.1
 pytest-cov==2.10.1
+pytest-redis==2.0.0
+redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
+eth-erc20~=0.0.10a2

apps/cic-eth/tests/check/test_check_db.py

@@ -0,0 +1,8 @@
+# local imports
+from cic_eth.check.db import health
+
+def test_check_health(
+    init_database,
+        ):
+
+    assert health()

apps/cic-eth/tests/check/test_check_gas.py

@@ -0,0 +1,20 @@
+# local imports
+from cic_eth.check.gas import health
+from cic_eth.db.models.role import AccountRole
+
+def test_check_gas(
+    config,
+    init_database,
+    default_chain_spec,
+    eth_rpc,
+    custodial_roles,
+    whoever,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
+    assert health(config=config)
+
+    AccountRole.set('GAS_GIFTER', whoever, session=init_database)
+    init_database.commit()
+    assert not health(config=config)

apps/cic-eth/tests/check/test_check_redis.py

@@ -0,0 +1,16 @@
+# external imports
+import pytest
+
+# local imports
+from cic_eth.check.redis import health
+
+
+def test_check_redis(
+        config,
+        have_redis,
+        ):
+
+    if have_redis != None:
+        pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
+
+    assert health(unit='test', config=config)

apps/cic-eth/tests/check/test_check_signer.py

@@ -0,0 +1,13 @@
+# local imports
+from cic_eth.check.signer import health
+
+
+def test_check_signer(
+        default_chain_spec,
+        config,
+        eth_signer,
+        eth_rpc,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    assert health(config=config)

apps/cic-eth/tests/conftest.py

@@ -2,9 +2,11 @@
 import os
 import sys
 import logging
+import uuid
 
 # external imports
 from eth_erc20 import ERC20
+import redis
 
 # local imports
 from cic_eth.api import Api
@@ -15,10 +17,11 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from tests.fixtures_config import *
+from cic_eth.pytest.fixtures_config import *
-from tests.fixtures_database import *
+from cic_eth.pytest.fixtures_celery import *
-from tests.fixtures_celery import *
+from cic_eth.pytest.fixtures_database import *
-from tests.fixtures_role import *
+from cic_eth.pytest.fixtures_role import *
+from cic_eth.pytest.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -55,3 +58,28 @@ def default_token(
         ):
     BaseTask.default_token_symbol = foo_token_symbol
     BaseTask.default_token_address = foo_token   
+
+
+
+@pytest.fixture(scope='session')
+def have_redis(
+        config,
+        ):
+
+    r = redis.Redis(
+            host = config.get('REDIS_HOST'),
+            port = config.get('REDIS_PORT'),
+            db = config.get('REDIS_DB'),
+            ) 
+    k = str(uuid.uuid4())
+    try:
+        r.set(k, 'foo')
+        r.delete(k)
+    except redis.exceptions.ConnectionError as e:
+        return e
+    except TypeError as e:
+        return e 
+
+    return None
+
+

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -0,0 +1,38 @@
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+from cic_eth.runnable.daemons.filters.callback import CallbackFilter
+from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
+from cic_eth.runnable.daemons.filters.tx import TxFilter
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+# Hit tx mismatch paths on all filters
+def test_filter_bogus(
+        init_database,
+        bogus_tx_block,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        transfer_auth,
+        cic_registry,
+        contract_roles,
+        register_lookups,
+        ):
+
+    fltrs = [
+        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
+        GasFilter(default_chain_spec, queue=None),
+        TxFilter(default_chain_spec, None),
+        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
+        StragglerFilter(default_chain_spec, None),
+        RegistrationFilter(default_chain_spec, queue=None),
+        ]
+      
+    for fltr in fltrs:
+        r = None
+        try:
+            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
+        except:
+            pass
+        assert not r

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -0,0 +1,101 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.sql.state import (
+        set_waitforgas,
+        )
+from hexathon import strip_0x
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.eth.gas import cache_gas_data
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
+    init_database.commit()
+    tx_hash_hex_wait = tx_hash_hex
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['BOB'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    fltr = GasFilter(default_chain_spec, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

apps/cic-eth/tests/filters/test_register_filter.py

@@ -0,0 +1,78 @@
+# external imports
+from eth_accounts_index.registry import AccountRegistry
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import(
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from erc20_faucet import Faucet
+from hexathon import strip_0x
+from chainqueue.sql.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+def test_register_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        account_registry,
+        faucet,
+        register_lookups,
+        contract_roles,
+        agent_roles,
+        cic_registry,
+        init_celery_tasks,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
+
+    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block, rcpt=rcpt)
+    tx.apply_receipt(rcpt)
+
+    fltr = RegistrationFilter(default_chain_spec, queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(gift_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    gift = Faucet.parse_give_to_request(gift_tx['data'])
+    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_straggler_filter.py

@@ -17,8 +17,8 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.tx import create as queue_create
+from chainqueue.sql.tx import create as queue_create
-from chainqueue.state import (
+from chainqueue.sql.state import (
         set_reserved,
         set_ready,
         set_sent,

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -0,0 +1,79 @@
+# external imports
+from erc20_transfer_authorization import TransferAuthorization
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from hexathon import strip_0x
+from chainqueue.sql.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+
+
+def test_filter_transferauth(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        transfer_auth,
+        foo_token,
+        celery_session_worker,
+        register_lookups,
+        init_custodial,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
+    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
+    
+    r = rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(approve_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    c = ERC20(default_chain_spec)
+    approve = c.parse_approve_request(approve_tx['data']) 
+    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -17,13 +17,12 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.tx import create as queue_create
+from chainqueue.sql.tx import create as queue_create
-from chainqueue.state import (
+from chainqueue.sql.state import (
         set_reserved,
         set_ready,
         set_sent,
         )
-
 from hexathon import strip_0x
 
 # local imports
@@ -31,7 +30,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
 from cic_eth.eth.gas import cache_gas_data
 
 
-def test_tx(
+def test_filter_tx(
         default_chain_spec,
         init_database,
         eth_rpc,

apps/cic-eth/tests/task/api/test_admin.py

@@ -9,8 +9,14 @@ from chainlib.eth.tx import (
         unpack,
         TxFormat,
         )
-from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.nonce import (
-from chainlib.eth.gas import Gas
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
 from chainlib.eth.address import to_checksum_address
 from hexathon import (
         strip_0x,
@@ -23,10 +29,18 @@ from chainqueue.db.enum import (
         StatusBits,
         status_str,
         )
-from chainqueue.query import get_tx
+from chainqueue.sql.state import (
+        set_fubar,
+        set_ready,
+        set_reserved,
+        )
+from chainqueue.sql.query import (
+        get_tx,
+        get_nonce_tx_cache,
+        )
 
 # local imports
-from cic_eth.api import AdminApi
+from cic_eth.api.admin import AdminApi
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import InitializationError
@@ -36,150 +50,6 @@ from cic_eth.queue.tx import queue_create
 logg = logging.getLogger()
 
 
-#def test_resend_inplace(
-#    default_chain_spec,
-#    init_database,
-#    init_w3,
-#    celery_session_worker,
-#    ):
-#    
-#    chain_str = str(default_chain_spec)
-#    c = RpcClient(default_chain_spec)
-#    
-#    sigs = []
-#
-#    gas_provider = c.gas_provider()
-#
-#    s_nonce = celery.signature(
-#        'cic_eth.eth.nonce.reserve_nonce',
-#        [
-#            init_w3.eth.accounts[0],
-#            gas_provider,
-#            ],
-#        queue=None,
-#        )
-#    s_refill = celery.signature(
-#        'cic_eth.eth.gas.refill_gas',
-#        [
-#            chain_str,
-#            ],
-#        queue=None,
-#            )
-#    s_nonce.link(s_refill)
-#    t = s_nonce.apply_async()
-#    t.get()
-#    for r in t.collect():
-#        pass
-#    assert t.successful()
-#
-#    q = init_database.query(Otx)
-#    q = q.join(TxCache)
-#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
-#    o = q.first()
-#    tx_raw = o.signed_tx
-#
-#    tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
-#    gas_price_before = tx_dict['gasPrice'] 
-#
-#    s = celery.signature(
-#        'cic_eth.admin.ctrl.lock_send',
-#        [
-#            chain_str,
-#            init_w3.eth.accounts[0],
-#            ],
-#        queue=None,
-#            )
-#    t = s.apply_async()
-#    t.get()
-#    assert t.successful()
-#
-#    api = AdminApi(c, queue=None)
-#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
-#    t.get()
-#    i = 0
-#    tx_hash_new_hex = None
-#    for r in t.collect():
-#        tx_hash_new_hex = r[1]
-#    assert t.successful()
-#  
-#    tx_raw_new = get_tx(tx_hash_new_hex) 
-#    logg.debug('get {}'.format(tx_raw_new))
-#    tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
-#    assert tx_hash_new_hex != tx_dict['hash']
-#    assert tx_dict_new['gasPrice'] > gas_price_before
-#
-#    tx_dict_after = get_tx(tx_dict['hash'])
-#
-#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
-#    assert tx_dict_after['status'] & StatusBits.MANUAL
-
-
-#def test_check_fix_nonce(
-#    default_chain_spec,
-#    init_database,
-#    init_eth_account_roles,
-#    init_w3,
-#    eth_empty_accounts,
-#    celery_session_worker,
-#    ):
-#
-#    chain_str = str(default_chain_spec)
-#    
-#    sigs = []
-#    for i in range(5):
-#        s = celery.signature(
-#            'cic_eth.eth.gas.refill_gas',
-#            [
-#                eth_empty_accounts[i],
-#                chain_str,
-#                ],
-#            queue=None,
-#                )
-#        sigs.append(s)
-#
-#    t = celery.group(sigs)()
-#    txs = t.get()
-#    assert t.successful()
-#
-#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
-#    c = RpcClient(default_chain_spec)
-#    api = AdminApi(c, queue=None)
-#    address = init_eth_account_roles['eth_account_gas_provider']
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['network'] == 0
-#    assert nonce_spec['nonce']['queue'] == 4
-#    assert nonce_spec['nonce']['blocking'] == None
-#
-#    s_set = celery.signature(
-#            'cic_eth.queue.tx.set_rejected',
-#            [
-#                tx_hash.hex(),
-#                ],
-#            queue=None,
-#            )
-#    t = s_set.apply_async()
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['blocking'] == 2
-#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
-#
-#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#    for tx in txs[3:]:
-#        tx_hash = web3.Web3.keccak(hexstr=tx)
-#        tx_dict = get_tx(tx_hash.hex())
-#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
-#
-#
-
-
 def test_have_account(
     default_chain_spec,
     custodial_roles,
@@ -243,28 +113,6 @@ def test_tag_account(
     assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
-#def test_ready(
-#    init_database,
-#    agent_roles,
-#    eth_rpc,
-#    ):
-#
-#    api = AdminApi(eth_rpc)
-#   
-#    with pytest.raises(InitializationError):
-#        api.ready()
-#
-#    bogus_account = os.urandom(20)
-#    bogus_account_hex = '0x' + bogus_account.hex()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
-#    with pytest.raises(KeyError):
-#        api.ready()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
-#    api.ready()
-
-
 def test_tx(
     default_chain_spec,
     cic_registry,
@@ -286,3 +134,168 @@ def test_tx(
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
     tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
     logg.warning('code missing to verify tx contents {}'.format(tx))
+
+
+def test_check_nonce_gap(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        if i == 3:
+            j = 1
+            nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
+
+        queue_create(
+                default_chain_spec,
+                i+j,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
+
+
+def test_check_nonce_localfail(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    set_ready(default_chain_spec, tx_hashes[4], session=init_database)
+    set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
+    set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[4]
+
+
+def test_fix_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        init_celery_tasks,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
+    r = t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+    
+    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
+    ks = txs.keys()
+    assert len(ks) == 2
+    for k in ks:
+        hsh = add_0x(k)
+        otx = Otx.load(hsh, session=init_database)
+        init_database.refresh(otx)
+        logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
+        if add_0x(k) == tx_hashes[3]:
+            assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+        else:
+            assert otx.status == 1

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -0,0 +1,373 @@
+# standard imports
+import logging
+import io
+import json
+
+# external imports
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        nonce,
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.tx import (
+        raw,
+        receipt,
+        TxFormat,
+        Tx,
+        )
+from chainlib.eth.block import block_latest
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        set_sent,
+        set_ready,
+        )
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.query import get_nonce_tx_cache
+from eth_erc20 import ERC20
+from cic_eth_registry import CICRegistry
+
+# local imports
+from cic_eth.api.admin import AdminApi
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.eth.erc20 import cache_transfer_data
+
+logg = logging.getLogger()
+
+
+def test_admin_api_tx(
+        default_chain_spec,
+        init_database,
+        init_celery_tasks,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        custodial_roles,
+        celery_session_worker,
+        foo_token,
+        address_declarator,
+        cic_registry,
+        register_tokens,
+        register_lookups,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
+
+    o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
+    r = eth_rpc.do(o)
+    gifter_nonce = int(r, 16)
+
+    #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            gifter_nonce, # will only work if agent starts at 0
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_transfer_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    # lookup by transaction hash, without registry
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
+    logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by RLP transaction, without registry
+    tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, with registry
+    registry = CICRegistry(default_chain_spec, eth_rpc)
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, using writer
+    buf = io.StringIO()
+    api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex 
+
+
+def test_admin_api_account(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes_alice = []
+    txs_alice = []
+
+    for i in range(3):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_alice.append(tx_hash_hex)
+        txs_alice.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
+    tx_hashes_bob = []
+    txs_bob = []
+
+    for i in range(2):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                13+i,
+                agent_roles['BOB'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_bob.append(tx_hash_hex)
+        txs_bob.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'])
+    assert len(r) == 5
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
+    assert len(r) == 2
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
+    assert len(r) == 3
+
+
+def test_admin_api_account_writer(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    buf = io.StringIO()
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
+
+    # TODO: improve eval
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex
+
+
+def test_registry(
+        eth_rpc,
+        cic_registry,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.registry()
+    r = t.get_leaf()
+    assert r == cic_registry
+
+
+def test_proxy_do(
+        default_chain_spec,
+        eth_rpc,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.proxy_do(default_chain_spec, o)
+    rr = t.get_leaf()
+
+    assert r == rr
+
+
+def test_resend_inplace(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
+    r = t.get_leaf()
+    assert t.successful()
+
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 2
+
+
+
+@pytest.mark.xfail()
+def test_resend_clone(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
+    r = t.get_leaf()
+    assert t.successful()
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    otx = Otx.load(txs[0], session=init_database)
+    assert otx.status == 0

apps/cic-eth/tests/task/api/test_app.py

@@ -8,11 +8,20 @@ import pytest
 import celery
 from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
+from eth_accounts_index import AccountsIndex
+from chainlib.eth.tx import (
+        transaction,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        )
 
 # local imports
 from cic_eth.api import Api
+from cic_eth.queue.query import get_tx
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 def test_account_api(
@@ -29,6 +38,47 @@ def test_account_api(
     assert t.successful()
 
 
+def test_account_api_register(
+        default_chain_spec,
+        init_database,
+        account_registry,
+        faucet,
+        custodial_roles,
+        cic_registry,
+        register_lookups,
+        eth_rpc,
+        celery_session_worker,
+        ):
+    api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
+    t = api.create_account('')
+    register_tx_hash = t.get_leaf()
+    assert t.successful()
+
+    set_reserved(default_chain_spec, register_tx_hash, session=init_database)
+
+    tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                [tx['signed_tx']],
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = transaction(register_tx_hash)
+    tx_src = eth_rpc.do(o)
+
+    c = AccountsIndex(default_chain_spec)
+    address = c.parse_add_request(tx_src['data'])
+    o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    assert c.parse_have(r)
+
+
 def test_transfer_api(
         default_chain_spec,
         eth_rpc,
@@ -37,16 +87,15 @@ def test_transfer_api(
         custodial_roles,
         agent_roles,
         cic_registry,
-        register_tokens,
+        token_registry,
         register_lookups,
         celery_session_worker,
+        register_tokens,
+        foo_token_symbol,
         ):
 
-    #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
-    foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
-
     api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol)
+    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) 
     t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -0,0 +1,19 @@
+# local imports
+from cic_eth.api.api_task import Api
+from cic_eth.task import BaseTask
+
+def test_default_token(
+        default_chain_spec,
+        foo_token,
+        default_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None)     
+    t = api.default_token()
+    r = t.get_leaf()
+    assert r['address'] == foo_token

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from tests.fixtures_celery import *
+from cic_eth.pytest.fixtures_celery import *

apps/cic-eth/tests/task/test_task_account.py

@@ -156,6 +156,7 @@ def test_gift(
     eth_signer,
     init_celery_tasks,
     cic_registry,
+    register_lookups,
     celery_session_worker,
     ):
 

apps/cic-eth/tests/task/test_task_admin.py

@@ -0,0 +1,88 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.sql.tx import (
+        create as queue_create,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import TxFormat
+from chainqueue.sql.query import get_nonce_tx_cache
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from hexathon import add_0x
+
+# local imports
+from cic_eth.admin.nonce import shift_nonce
+from cic_eth.eth.gas import cache_gas_data
+
+logg = logging.getLogger()
+
+
+def test_shift_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.admin.nonce.shift_nonce',
+            [
+                default_chain_spec.asdict(),
+                tx_hashes[3],
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+
+    for i in range(42+3, 42+10):
+        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
+        for k in txs.keys():
+            hsh = add_0x(k)
+            otx = Otx.load(hsh, session=init_database)
+            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
+            if add_0x(k) == tx_hashes[i-42]:
+                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+            else:
+                assert otx.status == 1

apps/cic-eth/tests/task/test_task_erc20.py

@@ -13,6 +13,7 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
+from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -167,3 +168,101 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
+
+
+def test_erc20_transfer_from_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    s_nonce = celery.signature(
+        'cic_eth.eth.nonce.reserve_nonce',
+        [
+            [token_object],
+            default_chain_spec.asdict(),
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            ],
+        queue=None,
+            )
+    s_transfer = celery.signature(
+            'cic_eth.eth.erc20.transfer_from',
+            [
+                custodial_roles['FOO_TOKEN_GIFTER'],
+                agent_roles['BOB'],
+                transfer_value,
+                default_chain_spec.asdict(),
+                agent_roles['ALICE'],
+                ],
+            queue=None,
+            )
+    s_nonce.link(s_transfer)
+    t = s_nonce.apply_async()
+    r = t.get_leaf()
+
+    logg.debug('result {}'.format(r))
+
+
+def test_erc20_allowance_check_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+        'symbol': 'FOO',
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    s_check = celery.signature(
+        'cic_eth.eth.erc20.check_allowance',
+        [
+            [token_object],
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            transfer_value,
+            default_chain_spec.asdict(),
+            agent_roles['ALICE']
+            ],
+        queue=None,
+            )
+    t = s_check.apply_async()
+    with pytest.raises(YouAreBrokeError):
+        t.get() 
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = s_check.apply_async()
+    t.get() 
+    assert t.successful()

apps/cic-eth/tests/task/test_task_gas.py

@@ -0,0 +1,286 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        unpack,
+        TxFormat,
+        )
+from chainlib.eth.constant import (
+        MINIMUM_FEE_UNITS,
+        MINIMUM_FEE_PRICE,
+        )
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.query import get_tx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.state import (
+        set_ready,
+        set_reserved,
+        set_sent,
+        )
+from chainqueue.db.models.otx import Otx
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.error import OutOfGasError
+
+logg = logging.getLogger()
+
+
+def test_task_check_gas_ok(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                8000000,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+def test_task_check_gas_insufficient(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(whoever, 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                None,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    try:
+        r = t.get_leaf()
+    except OutOfGasError:
+        pass
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+
+def test_task_check_gas_low(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
+    nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
+    r = eth_rpc.do(o)
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) 
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                ],
+                [],
+                None,
+                None,
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+@pytest.mark.parametrize(
+        '_gas_price,_gas_factor',
+        [
+        (None, 1.1),
+        (MINIMUM_FEE_PRICE * 1.1, 0.9),
+        (None, 1.3),
+        ]
+        )
+def test_task_resend_explicit(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        _gas_price,
+        _gas_factor,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+
+    init_database.commit()
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    s = celery.signature(
+            'cic_eth.eth.gas.resend_with_higher_gas',
+            [
+                tx_hash_hex,
+                default_chain_spec.asdict(),
+                _gas_price,
+                _gas_factor,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(Otx)
+    q = q.filter(Otx.tx_hash==strip_0x(r))
+    otx = q.first()
+    if otx == None:
+        raise NotLocalTxError(r)
+
+    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
+    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
+    assert tx_after['gasPrice'] > tx_before['gasPrice']
+

apps/cic-eth/tests/task/test_task_tx.py

@@ -4,16 +4,27 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.gas import Gas
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
         transaction,
         receipt,
+        raw,
         )
 from hexathon import strip_0x
 from chainqueue.db.models.otx import Otx
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        set_sent,
+        )
+from chainqueue.db.enum import StatusBits
 
 # local imports
 from cic_eth.queue.tx import register_tx
@@ -60,15 +71,6 @@ def test_tx_send(
 
 
 def test_sync_tx(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-    pass
-
-
-def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,
         eth_rpc,
@@ -77,31 +79,48 @@ def test_resend_with_higher_gas(
         celery_session_worker,
         ):
 
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
+
-    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    o = raw(tx_signed_raw_hex)
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
 
     s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
+            'cic_eth.eth.tx.sync_tx',
             [
                 tx_hash_hex,
                 default_chain_spec.asdict(),
                 ],
-            queue=None,
+            queue=None
             )
     t = s.apply_async()
     r = t.get_leaf()
+    assert t.successful() 
 
-    q = init_database.query(Otx)
+    init_database.commit()
-    q = q.filter(Otx.tx_hash==strip_0x(r))
-    otx = q.first()
-    if otx == None:
-        raise NotLocalTxError(r)
-
-    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
-    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
-    assert tx_after['gasPrice'] > tx_before['gasPrice']
 
+    o = Otx.load(tx_hash_hex, session=init_database)
+    assert o.status & StatusBits.FINAL == StatusBits.FINAL

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -0,0 +1,170 @@
+# standard imports
+import os
+import logging
+
+# external imports
+import pytest
+import celery
+from chainqueue.sql.tx import create as queue_create
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        receipt,
+        )
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        )
+
+logg = logging.getLogger()
+
+
+def test_hashes_to_txs(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+
+    #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    init_database.commit()
+
+    bogus_one = add_0x(os.urandom(32).hex())
+    bogus_two = add_0x(os.urandom(32).hex())
+
+    yarrgs = [
+                bogus_one,
+                tx_hash_hex_two,
+                bogus_two,
+                tx_hash_hex_one,
+            ]   
+    s = celery.signature(
+        'cic_eth.eth.tx.hashes_to_txs',
+        [
+            yarrgs,
+                ],
+        queue=None,
+        )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    assert len(r) == 2
+
+    logg.debug('r {}'.format(r))
+    txs = [
+        tx_signed_raw_hex_two,
+        tx_signed_raw_hex_one,
+            ]
+    for tx in r:
+        txs.remove(add_0x(tx))
+    assert len(txs) == 0
+
+
+
+def test_double_send(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+    set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
+    init_database.commit()
+
+    yarrgs = [
+        tx_signed_raw_hex_one,
+        tx_signed_raw_hex_two,
+            ]
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                yarrgs,
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = receipt(tx_hash_hex_one)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    o = receipt(tx_hash_hex_two)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+
+

apps/cic-eth/tests/unit/ext/test_address.py

@@ -19,6 +19,7 @@ def test_translate(
         agent_roles,
         cic_registry,
         init_celery_tasks,
+        register_lookups,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)

apps/cic-eth/tests/unit/test_registry_connect.py

@@ -0,0 +1,22 @@
+# local imports
+from cic_eth.registry import *
+
+def test_registry_connect(
+        eth_rpc,
+        default_chain_spec,
+        address_declarator,
+        token_registry,
+        contract_roles,
+        purge_lookups,
+        registry,
+        agent_roles,
+        ):
+
+    r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+

apps/cic-eth/tests/unit/test_stat.py

@@ -0,0 +1,18 @@
+# standard imports
+import datetime
+
+# local imports
+from cic_eth.stat import init_chain_stat
+
+
+def test_chain_stat(
+        eth_rpc,
+        init_eth_tester,
+        ):
+
+    now = int(datetime.datetime.now().timestamp()) + 1
+    for i in range(11):
+        init_eth_tester.time_travel(now + (i * 2))
+    
+    s = init_chain_stat(eth_rpc, block_start=0)
+    assert s.block_average() == 2

apps/cic-eth/tools_requirements.txt

@@ -0,0 +1,8 @@
+crypto-dev-signer~=0.4.14b6
+chainqueue~=0.0.2b5
+confini~=0.3.6rc4
+cic-eth-registry~=0.5.6a1
+redis==3.5.3
+hexathon~=0.0.1a7
+pycryptodome==3.10.1
+pyxdg==0.27

apps/cic-meta/.gitignore

@@ -3,3 +3,5 @@ dist
 dist-web
 dist-server
 scratch
+coverage
+.nyc_output

apps/cic-meta/.gitlab-ci.yml

@@ -3,17 +3,38 @@
     variables:
         APP_NAME: cic-meta
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
 .cic_meta_changes_target:
     rules:
-        - changes:
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-            - $CONTEXT/$APP_NAME/*
+#        - changes:
+#            - $CONTEXT/$APP_NAME/*
+        - when: always
 
-build-mr-cic-meta:
+cic-meta-build-mr:
+  stage: build
+  extends:
+    - .cic_meta_variables
+    - .cic_meta_changes_target
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
+
+test-mr-cic-meta:
     extends:
-        - .cic_meta_changes_target
-        - .py_build_merge_request
         - .cic_meta_variables
+        - .cic_meta_changes_target
+    stage: test
+    image: $IMAGE_TAG 
+    script:
+        - cd /tmp/src/cic-meta
+        - npm install --dev 
+        - npm run test
+        - npm run test:coverage
+    needs: ["cic-meta-build-mr"]
 
 build-push-cic-meta:
     extends:

apps/cic-meta/docker/Dockerfile

@@ -4,29 +4,28 @@ WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
-COPY cic-meta/package.json \
+# required to build the cic-client-meta module
-     ./
-
 COPY cic-meta/src/ src/
-COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
 
+# copy the dependencies
+COPY cic-meta/package.json .
+COPY cic-meta/tsconfig.json . 
+COPY cic-meta/webpack.config.js .
+     
 RUN npm install
 
-# see exports_dir gpg.ini
+COPY cic-meta/tests/ tests/
 COPY cic-meta/tests/*.asc /root/pgp/
-RUN alias tsc=node_modules/typescript/bin/tsc 
-
 
+# copy runtime configs
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-# COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
+# db migrations
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+RUN alias tsc=node_modules/typescript/bin/tsc 
-#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
-
 COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/package.json

@@ -11,6 +11,7 @@
 	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
+		"test:coverage": "nyc mocha tests/*.ts --timeout 3000 --check-coverage=true",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
@@ -34,7 +35,9 @@
 	"devDependencies": {
 		"@types/mocha": "^8.0.3",
 		"mocha": "^8.2.0",
+		"nock": "^13.1.0",
 		"node-localstorage": "^2.1.6",
+		"nyc": "^15.1.0",
 		"ts-node": "^9.0.0",
 		"typescript": "^4.0.5",
 		"webpack": "^5.4.0",
@@ -50,5 +53,26 @@
 	"license": "GPL-3.0-or-later",
 	"engines": {
 		"node": ">=14.16.1"
+	},
+	"nyc": {
+		"include": [
+			"src/**/*.ts"
+		],
+		"extension": [
+			".ts"
+		],
+		"require": [
+			"ts-node/register"
+		],
+		"reporter": [
+			"text",
+			"html"
+		],
+		"sourceMap": true,
+		"instrument": true,
+		"branches": ">80",
+		"lines": ">80",
+		"functions": ">80",
+		"statements": ">80"
 	}
 }

apps/cic-meta/scripts/server/handlers.ts

@@ -31,7 +31,7 @@ function handleNoMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('mesage', e);
+				console.error('message', e);
 				doh(e);
 			});
 		})
@@ -46,7 +46,7 @@ function handleServerMergePost(data, db, digest, keystore, signer) {
 			let e = undefined;
 			let s = undefined;
 			if (v === undefined) {
-				s = new Syncable(digest, data);
+				s = new Syncable(digest, o);
 				s.onwrap = (e) => {
 					whohoo(e.toJSON());
 				};

apps/cic-meta/scripts/server/server.ts

@@ -204,7 +204,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty onctent', data);
+			console.error('empty content', data);
 			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/cic-meta/src/custom.ts

@@ -9,7 +9,7 @@ class Custom extends Syncable implements Addressable {
         super('', v);
         Custom.toKey(name).then((cid) => {
             this.id = cid;
-            this.value = v;
+            this.name = name;
         });
     }
 

apps/cic-meta/src/meta.ts

@@ -100,13 +100,15 @@ class Meta {
             identifier = await User.toKey(name);
         } else if (type === 'phone') {
             identifier = await Phone.toKey(name);
-        } else {
+        } else if (type === 'custom') {
             identifier = await Custom.toKey(name);
+        } else {
+            identifier = await Custom.toKey(name, type);
         }
         return identifier;
     }
 
-    private wrap(syncable: Syncable): Promise<Envelope> {
+    wrap(syncable: Syncable): Promise<Envelope> {
         return new Promise<Envelope>(async (resolve, reject) => {
             syncable.setSigner(this.signer);
             syncable.onwrap = async (env) => {

apps/cic-meta/tests/custom.ts

@@ -0,0 +1,49 @@
+import * as assert from 'assert';
+import {Custom} from "../src";
+
+const testName = 'areas';
+const testObject = {
+    area: ['Nairobi', 'Mombasa', 'Kilifi']
+}
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+describe('custom', () => {
+
+    context('with predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName, testObject);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, testObject);
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    context('without predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, {});
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    describe('#toKey()', () => {
+        context('without a custom identifier', () => {
+            it('should generate a key from the custom name', async () => {
+                assert.strictEqual(await Custom.toKey(testName), testNameKey);
+            });
+        });
+
+        context('with a custom identifier', () => {
+            it('should generate a key from the custom name with a custom identifier', async () => {
+                assert.strictEqual(await Custom.toKey(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/meta.ts

@@ -0,0 +1,176 @@
+import * as assert from 'assert';
+import * as fs from 'fs';
+const nock = require('nock');
+import {Meta} from "../src";
+import {getResponse, metaData, networkErrorResponse, notFoundResponse, putResponse} from "./response";
+import {Syncable} from "@cicnet/crdt-meta";
+
+const metaUrl = 'https://meta.dev.grassrootseconomics.net';
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+const testName = 'areas'
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+function readFile(filename) {
+    if(!fs.existsSync(filename)) {
+        console.error(`File ${filename} not found`);
+        return;
+    }
+    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
+}
+
+const privateKey = readFile('./privatekeys.asc');
+
+describe('meta', () => {
+    beforeEach(() => {
+        nock(metaUrl)
+            .get(`/${testAddressKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testPhoneKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .get(`/${testIdentifier}`)
+            .replyWithError(networkErrorResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddressKey}`)
+            .reply(200, putResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .post('/post')
+            .reply(500);
+    });
+
+    describe('#get()', () => {
+        it('should fetch data from the meta service', async () => {
+            const account = await Meta.get(testAddressKey, metaUrl);
+            assert.strictEqual(account.toJSON(account), getResponse.payload);
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testAddress, metaUrl);
+                assert.strictEqual(account, `404: Not Found`);
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testIdentifier, metaUrl);
+                assert.strictEqual(account, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+            });
+        });
+    })
+
+    describe('#set()', () => {
+        context('object data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testAddressKey, metaData);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('string data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testPhoneKey, testAddress);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testIdentifier, metaData);
+                    assert.strictEqual(response, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+                }
+            });
+        });
+    });
+
+    describe('#updateMeta()', () => {
+        it('should update data in the meta server', async () => {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.updateMeta(syncable, testAddressKey);
+                assert.strictEqual(response, putResponse);
+            }
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', () => {
+                const syncable = new Syncable(testAddress, metaData);
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.updateMeta(syncable, testAddress);
+                    assert.strictEqual(response, notFoundResponse);
+                }
+            });
+        });
+    });
+
+    describe('#wrap()', () => {
+        it('should sign a syncable object', function () {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.wrap(syncable);
+                assert.strictEqual(response.toJSON(), getResponse);
+            }
+        });
+    })
+
+    describe('#getIdentifier()', () => {
+        context('without type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName), testNameKey);
+            });
+        });
+
+        context('with user type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testAddress, 'user'), testAddressKey);
+            });
+        });
+
+        context('with phone type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testPhone, 'phone'), testPhoneKey);
+            });
+        });
+
+        context('with custom type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, 'custom'), testNameKey);
+            });
+        });
+
+        context('with unrecognised type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/phone.ts

@@ -0,0 +1,24 @@
+import * as assert from 'assert';
+import {Phone} from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+
+describe('phone', () => {
+
+    it('should create a phone object', () => {
+        const phone = new Phone(testAddress, testPhone);
+        setTimeout(() => {
+            assert.strictEqual(phone.address, testAddress);
+            assert.strictEqual(phone.m.data.msisdn, testPhone);
+            assert.strictEqual(phone.key(), testPhoneKey)
+        }, 0);
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the phone number', async () => {
+            assert.strictEqual(await Phone.toKey(testPhone), testPhoneKey);
+        });
+    });
+});

apps/cic-meta/tests/user.ts

@@ -0,0 +1,54 @@
+import * as assert from 'assert';
+
+import { User } from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testUser = {
+    user: {
+        firstName: 'Test',
+        lastName: 'User'
+    }
+}
+
+describe('user', () => {
+
+    context('without predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, '');
+                assert.strictEqual(user.m.data.user.lastName, '');
+            }, 0);
+        });
+    });
+
+    context('with predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress, testUser);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+                assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+            }, 0);
+        });
+    });
+
+    describe('#setName()', () => {
+        it('should set user\'s names to metadata', () => {
+            const user = new User(testAddress);
+            user.setName(testUser.user.firstName, testUser.user.lastName);
+            assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+            assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+        });
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the user\'s address', async () => {
+            assert.strictEqual(await User.toKey(testAddress), testAddressKey);
+        });
+    });
+});

apps/cic-meta/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "baseUrl": ".",
     "outDir": "./dist.browser",
-    "target": "es5",
+    "target": "es2015",
     "module": "commonjs",
     "moduleResolution": "node",
     "lib": ["es2016", "dom", "es5"],

apps/cic-notify/cic_notify/api.py

@@ -26,7 +26,7 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
         for q in qs[host]:
             if re.match(re_q, q['name']):
                 host_queues.append((host, q['name'],))
-   
+
     task_prefix_len = len(task_prefix)
     queue_tasks = []
     for (host, queue) in host_queues:
@@ -35,17 +35,18 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
             for task in tasks:
                 if len(task) >= task_prefix_len and task[:task_prefix_len] == task_prefix:
                     queue_tasks.append((queue, task,))
-    
+
     return queue_tasks
 
 
 class Api:
     # TODO: Implement callback strategy
-    def __init__(self, queue='cic-notify'):
+    def __init__(self, queue=None):
         """
         :param queue: The queue on which to execute notification tasks
         :type queue: str
         """
+        self.queue = queue
         self.sms_tasks = get_sms_queue_tasks(app)
         logg.debug('sms tasks {}'.format(self.sms_tasks))
 
@@ -61,13 +62,19 @@ class Api:
         """
         signatures = []
         for q in self.sms_tasks:
+
+            if not self.queue:
+                queue = q[0]
+            else:
+                queue = self.queue
+
             signature = celery.signature(
                     q[1],
                     [
                         message,
                         recipient,
                         ],
-                    queue=q[0],
+                    queue=queue,
                     )
             signatures.append(signature)
 

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -87,10 +87,18 @@ for key in config.store.keys():
         module = importlib.import_module(config.store[key])
         if key == 'TASKS_AFRICASTALKING':
             africastalking_notifier = module.AfricasTalkingNotifier
+
+            api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID')
+            logg.debug(f'SENDER ID VALUE IS: {api_sender_id}')
+
+            if not api_sender_id:
+                api_sender_id = None
+                logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}')
+
             africastalking_notifier.initialize(
                 config.get('AFRICASTALKING_API_USERNAME'),
                 config.get('AFRICASTALKING_API_KEY'),
-                config.get('AFRICASTALKING_API_SENDER_ID')
+                api_sender_id
             )
 
 

apps/cic-notify/cic_notify/tasks/sms/africastalking.py

@@ -56,7 +56,7 @@ class AfricasTalkingNotifier:
             response = self.api_client.send(message=message, recipients=[recipient])
             logg.debug(f'africastalking response no-sender-id {response}')
 
-        recipients = response.get('Recipients')
+        recipients = response.get('SMSMessageData').get('Recipients')
 
         if len(recipients) != 1:
             status = response.get('SMSMessageData').get('Message')