Uncomment gas tests

Unit tests for cic-eth

See merge request grassrootseconomics/cic-internal-integration!164

.gitlab-ci.yml

@@ -6,6 +6,7 @@ include:
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/cic_cache/cache.py

@@ -1,22 +1,28 @@
 # standard imports
 import logging
+import datetime
 
-# third-party imports
+# external imports
 import moolb
 
 # local imports
-from cic_cache.db import list_transactions_mined
-from cic_cache.db import list_transactions_account_mined
+from cic_cache.db.list import (
+        list_transactions_mined,
+        list_transactions_account_mined,
+        list_transactions_mined_with_data,
+        )
 
 logg = logging.getLogger()
 
 
-class BloomCache:
+class Cache:
 
     def __init__(self, session):
         self.session = session
 
 
+class BloomCache(Cache):
+
     @staticmethod
     def __get_filter_size(n):
         n = 8192 * 8
@@ -87,3 +93,44 @@ class BloomCache:
             f_blocktx.add(block + tx)
             logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
+
+
+class DataCache(Cache):
+
+    def load_transactions_with_data(self, offset, end):
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
+        tx_cache = []
+        highest_block = -1;
+        lowest_block = -1;
+        date_is_str = None # stick this in startup
+        for r in rows:
+            if highest_block == -1:
+                highest_block = r['block_number']
+            lowest_block = r['block_number']
+            tx_type = 'unknown'
+
+            if r['value'] != None:
+                tx_type = '{}.{}'.format(r['domain'], r['value'])
+
+            if date_is_str == None:
+                date_is_str = type(r['date_block']).__name__ == 'str'
+
+            o = {
+                'block_number': r['block_number'],
+                'tx_hash': r['tx_hash'],
+                'date_block': r['date_block'],
+                'sender': r['sender'],
+                'recipient': r['recipient'],
+                'from_value': int(r['from_value']),
+                'to_value': int(r['to_value']),
+                'source_token': r['source_token'],
+                'destination_token': r['destination_token'],
+                'success': r['success'],
+                'tx_type': tx_type,
+            }
+
+            if date_is_str:
+                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
+
+            tx_cache.append(o)
+        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/db/list.py

@@ -28,6 +28,26 @@ def list_transactions_mined(
     return r
 
 
+def list_transactions_mined_with_data(
+        session,
+        offset,
+        end,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param offset: Offset in data set to return transactions from
+    :type offset: int
+    :param limit: Max number of transactions to retrieve
+    :type limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+
+    r = session.execute(s)
+    return r
+
+
 def list_transactions_account_mined(
         session,
         address,

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -0,0 +1,110 @@
+# standard imports
+import logging
+import json
+import re
+import base64
+
+# local imports
+from cic_cache.cache import (
+        BloomCache,
+        DataCache,
+    )
+
+logg = logging.getLogger(__name__)
+
+re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+
+DEFAULT_LIMIT = 100
+
+
+def process_transactions_account_bloom(session, env):
+    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    address = r[1]
+    if r[2] == None:
+        address = '0x' + address
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 2:
+        offset = r[3]
+    limit = 0
+    if r.lastindex > 3:
+        limit = r[4]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_bloom(session, env):
+    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 0:
+        offset = r[1]
+    limit = 0
+    if r.lastindex > 1:
+        limit = r[2]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_data(session, env):
+    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
+    if not r:
+        return None
+    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
+        return None
+
+    offset = r[1]
+    end = r[2]
+    if r[2] < r[1]:
+        raise ValueError('cart before the horse, dude')
+
+    c = DataCache(session)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
+
+    for r in tx_cache:
+        r['date_block'] = r['date_block'].timestamp()
+
+    o = {
+        'low': lowest_block,
+        'high': highest_block,
+        'data': tx_cache,
+    }
+
+    
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,18 +1,20 @@
 # standard imports
 import os
-import re
 import logging
 import argparse
-import json
 import base64
 
-# third-party imports
+# external imports
 import confini
 
 # local imports
-from cic_cache import BloomCache
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
+from cic_cache.runnable.daemons.query import (
+        process_transactions_account_bloom,
+        process_transactions_all_bloom,
+        process_transactions_all_data,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -44,72 +46,6 @@ logg.debug('config:\n{}'.format(config))
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-
-DEFAULT_LIMIT = 100
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    address = r[1]
-    if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
 
 # uwsgi application
 def application(env, start_response):
@@ -119,10 +55,16 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
+            process_transactions_all_data,
             process_transactions_all_bloom,
             process_transactions_account_bloom,
             ]:
-        r = handler(session, env)
+        r = None
+        try:
+            r = handler(session, env)
+        except ValueError as e:
+            start_response('400 {}'.format(str(e)))
+            return []
         if r != None:
             (mime_type, content) = r
             break

apps/cic-cache/config/test/syncer.ini

@@ -0,0 +1,2 @@
+[syncer]
+loop_interval = 1

apps/cic-cache/tests/conftest.py

@@ -88,3 +88,16 @@ def txs(
             tx_hash_first,
             tx_hash_second,
             ]
+
+
+@pytest.fixture(scope='function')
+def tag_txs(
+        init_database,
+        txs,
+        ):
+
+    db.add_tag(init_database, 'taag', domain='test')
+    init_database.commit()
+
+    db.tag_transaction(init_database, txs[1], 'taag', domain='test')
+

apps/cic-cache/tests/test_api.py

@@ -0,0 +1,31 @@
+# standard imports
+import json
+
+# external imports
+import pytest
+
+# local imports
+from cic_cache.runnable.daemons.query import process_transactions_all_data
+
+
+def test_api_all_data(
+        init_database,
+        txs,
+        ):
+
+    env = {
+        'PATH_INFO': '/txa/410000/420000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    j = process_transactions_all_data(init_database, env)
+    o = json.loads(j[1])
+    
+    assert len(o['data']) == 2
+
+    env = {
+        'PATH_INFO': '/txa/420000/410000',
+        'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+   
+    with pytest.raises(ValueError):
+        j = process_transactions_all_data(init_database, env)

apps/cic-cache/tests/test_cache.py

@@ -9,6 +9,7 @@ import pytest
 
 # local imports
 from cic_cache import BloomCache
+from cic_cache.cache import DataCache
 
 logg = logging.getLogger()
 
@@ -33,3 +34,23 @@ def test_cache(
 
     assert b[0] == list_defaults['block'] - 1
 
+
+def test_cache_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        tag_txs,
+        ):
+
+    session = init_database
+
+    c = DataCache(session)
+    b = c.load_transactions_with_data(410000, 420000)
+
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == txs[1]
+    assert b[2][1]['tx_type'] == 'unknown'
+    assert b[2][0]['tx_type'] == 'test.taag'
+    

apps/cic-eth/.coveragerc

@@ -5,3 +5,5 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
+	cic_eth/queue/state.py
+	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -5,18 +5,29 @@
 
 .cic_eth_changes_target:
     rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      #changes:
+          #- $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
-        - .cic_eth_changes_target
-        - .py_build_merge_request
         - .cic_eth_variables
+        - .cic_eth_changes_target
+        - .py_build_target_test
+
+test-mr-cic-eth:
+    extends:
+        - .cic_eth_variables
+        - .cic_eth_changes_target
+    stage: test
+    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
+    script:
+        - cd apps/$APP_NAME/
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
-        
-

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,11 +4,18 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.eth.tx import unpack
-from chainqueue.query import get_tx
-from chainqueue.state import set_cancel
+from chainlib.connection import RPCConnection
+from chainlib.eth.tx import (
+        unpack,
+        TxFactory,
+        )
+from chainlib.eth.gas import OverrideGasOracle
+from chainqueue.sql.query import get_tx
+from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
+from hexathon import strip_0x
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -21,13 +28,14 @@ from cic_eth.admin.ctrl import (
         )
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True)
-def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
+@celery_app.task(bind=True, base=BaseTask)
+def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -38,25 +46,29 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
+    chain_spec = ChainSpec.from_dict(chainspec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_brief = get_tx(tx_hash_orig_hex)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
+    session = BaseTask.session_func()
+    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
 
-    lock_queue(None, chain_str, address)
-    lock_send(None, chain_str, address)
+    lock_queue(None, chain_spec.asdict(), address=address)
+    lock_send(None, chain_spec.asdict(), address=address)
+
+    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -69,49 +81,57 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
+        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        del(tx_new['hash'])
-        del(tx_new['hash_unsigned'])
+        tx_new['gas_price'] += 1
+        tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
 
-        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
+        logg.debug('tx_new {}'.format(tx_new))
+
+        del(tx_new['hash'])
+        del(tx_new['hash_unsigned'])
+        del(tx_new['hashUnsigned'])
+
+        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
+        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            nonce=tx_new['nonce'],
-            address=tx_new['from'],
-            tx_hash=tx_hash_hex,
-            signed_tx=tx_signed_raw_hex,
-                )
+            tx_new['nonce'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            )
         session.add(otx)
-        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(tx_previous_hash_hex, True)
+        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
+        session.commit()
 
     session.close()
 
-    s = create_check_gas_and_send_task(
+    s = create_check_gas_task(
          txs, 
-         chain_str,
+         chain_spec,
          tx_new['from'],
-         tx_new['gas'],
-         tx_hashes, 
-         queue,
+         gas=tx_new['gas'],
+         tx_hashes_hex=tx_hashes, 
+         queue=queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,
@@ -119,7 +139,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_str,
+            chain_spec.asdict(),
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/cic_eth/admin/token.py

@@ -16,4 +16,6 @@ def default_token(self):
     return {
             'symbol': self.default_token_symbol,
             'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
             }

apps/cic-eth/cic_eth/api/api_admin.py

@@ -8,6 +8,7 @@ from chainlib.eth.constant import (
         ZERO_ADDRESS,
         )
 from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.contract import code
@@ -30,13 +31,14 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
+from chainqueue.query import get_tx
+from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
-from cic_eth.queue.query import get_tx
 
 app = celery.current_app
 
@@ -188,6 +190,7 @@ class AdminApi:
         s_manual = celery.signature(
             'cic_eth.queue.state.set_manual',
             [
+                chain_spec.asdict(),
                 tx_hash_hex,
                 ],
             queue=self.queue,
@@ -206,8 +209,9 @@ class AdminApi:
             s.link(s_gas)
 
         return s_manual.apply_async()
-                        
-    def check_nonce(self, address):
+                    
+
+    def check_nonce(self, chain_spec, address):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -228,13 +232,12 @@ class AdminApi:
             s_get_tx = celery.signature(
                     'cic_eth.queue.query.get_tx',
                     [
-                    chain_spec.asdict(),
+                        chain_spec.asdict(),
                         k,
                         ],
                     queue=self.queue,
                     )
             tx = s_get_tx.apply_async().get()
-            #tx = get_tx(k)
             logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
             if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@@ -242,7 +245,9 @@ class AdminApi:
                 blocking_tx = k
                 blocking_nonce = nonce_otx
             elif nonce_otx - last_nonce > 1:
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from']))
+                logg.debug('tx {}'.format(tx))
+                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
+                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
                 blocking_tx = k
                 blocking_nonce = nonce_otx
                 break
@@ -256,12 +261,13 @@ class AdminApi:
                 'blocking': blocking_nonce,
             },
             'tx': {
-                'blocking': blocking_tx,
-                }
+                'blocking': add_0x(blocking_tx),
             }
+        }
 
 
-    def fix_nonce(self, address, nonce, chain_spec):
+    # TODO: is risky since it does not validate that there is actually a nonce problem?
+    def fix_nonce(self, chain_spec, address, nonce):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -275,15 +281,17 @@ class AdminApi:
         txs = s.apply_async().get()
 
         tx_hash_hex = None
+        session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(k)
+            tx_dict = get_tx(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
+        session.close()
 
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    self.rpc.chain_spec.asdict(),
+                    chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -388,12 +396,13 @@ class AdminApi:
     
         t = s.apply_async()
         tx = t.get()
-  
+
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
+            source_token_declaration = None
             if registry != None:
                 try:
-                    source_token = registry.by_address(tx['source_token'])
+                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
             else:
@@ -406,16 +415,21 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                source_token = t.get()
-                if source_token == None:
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
+                source_token_declaration = t.get()
+
+            if source_token_declaration != None:
+                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
+                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
+                logg.debug('source token set tup {}'.format(source_token))
+
 
 
         destination_token = None
         if tx['destination_token'] != ZERO_ADDRESS:
+            destination_token_declaration = None
             if registry != None:
                 try:
-                    destination_token = registry.by_address(tx['destination_token'])
+                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
                 except UnknownContractError:
                     logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
             else:
@@ -428,10 +442,10 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                destination_token = t.get()
-                if destination_token == None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
-
+                destination_token_declaration = t.get()
+            if destination_token_declaration != None:
+                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
+                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
@@ -543,13 +557,19 @@ class AdminApi:
                 if role != None:
                     tx['recipient_description'] = role
 
+        erc20_c = ERC20(chain_spec)
         if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol()
-            tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call()
+            tx['source_token_symbol'] = source_token.symbol
+            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
 
         if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol()
-            tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call()
+            tx['destination_token_symbol'] = destination_token.symbol
+            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
+            r = self.rpc.do(o)
+            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
+            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
         tx['network_status'] = 'Not in node' 

apps/cic-eth/cic_eth/api/api_task.py

@@ -74,134 +74,134 @@ class Api:
         return s_token.apply_async()
 
 
-    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of receipient
-        :type to_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
-        :type  target_return: int
-        :param minimum_return: The least value of destination token return to allow
-        :type minimum_return: int
-        :param from_token_symbol: ERC20 token symbol of token being converted
-        :type from_token_symbol: str
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        raise NotImplementedError('out of service until new DEX migration is done')
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [from_token_symbol, to_token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        s_convert = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
-                [
-                    from_address,
-                    target_return,
-                    minimum_return,
-                    to_address,
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_convert.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
-        else:
-            s_tokens.link(s_convert)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
-
-
-    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param target_return: Estimated return from conversion
-        :type  target_return: int
-        :param minimum_return: The least value of destination token return to allow
-        :type minimum_return: int
-        :param from_token_symbol: ERC20 token symbol of token being converted
-        :type from_token_symbol: str
-        :param to_token_symbol: ERC20 token symbol of token to receive
-        :type to_token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        raise NotImplementedError('out of service until new DEX migration is done')
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [from_token_symbol, to_token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_convert = celery.signature(
-                'cic_eth.eth.bancor.convert_with_default_reserve',
-                [
-                    from_address,
-                    target_return,
-                    minimum_return,
-                    from_address,
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_convert.link(self.callback_success)
-            s_tokens.link(s_convert).on_error(self.callback_error)
-        else:
-            s_tokens.link(s_convert)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
+#    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param to_address: Ethereum address of receipient
+#        :type to_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_str,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    to_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
+#
+#
+#    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
+#
+#        :param from_address: Ethereum address of sender
+#        :type from_address: str, 0x-hex
+#        :param target_return: Estimated return from conversion
+#        :type  target_return: int
+#        :param minimum_return: The least value of destination token return to allow
+#        :type minimum_return: int
+#        :param from_token_symbol: ERC20 token symbol of token being converted
+#        :type from_token_symbol: str
+#        :param to_token_symbol: ERC20 token symbol of token to receive
+#        :type to_token_symbol: str
+#        :returns: uuid of root task
+#        :rtype: celery.Task
+#        """
+#        raise NotImplementedError('out of service until new DEX migration is done')
+#        s_check = celery.signature(
+#                'cic_eth.admin.ctrl.check_lock',
+#                [
+#                    [from_token_symbol, to_token_symbol],
+#                    self.chain_spec.asdict(),
+#                    LockEnum.QUEUE,
+#                    from_address,
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce = celery.signature(
+#                'cic_eth.eth.nonce.reserve_nonce',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_tokens = celery.signature(
+#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+#                [
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_convert = celery.signature(
+#                'cic_eth.eth.bancor.convert_with_default_reserve',
+#                [
+#                    from_address,
+#                    target_return,
+#                    minimum_return,
+#                    from_address,
+#                    self.chain_spec.asdict(),
+#                    ],
+#                queue=self.queue,
+#                )
+#        s_nonce.link(s_tokens)
+#        s_check.link(s_nonce)
+#        if self.callback_param != None:
+#            s_convert.link(self.callback_success)
+#            s_tokens.link(s_convert).on_error(self.callback_error)
+#        else:
+#            s_tokens.link(s_convert)
+#
+#        t = s_check.apply_async(queue=self.queue)
+#        return t
 
 
     def transfer(self, from_address, to_address, value, token_symbol):

apps/cic-eth/cic_eth/db/util.py

@@ -1,8 +0,0 @@
-import math
-
-def num_serialize(n):
-    if n == 0:
-        return b'\x00'
-    binlog = math.log2(n)
-    bytelength = int(binlog / 8 + 1)
-    return n.to_bytes(bytelength, 'big')

apps/cic-eth/cic_eth/eth/gas.py

@@ -57,10 +57,12 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+MAXIMUM_FEE_UNITS = 8000000
+
 class MaxGasOracle:
 
     def gas(code=None):
-        return 8000000
+        return MAXIMUM_FEE_UNITS
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -150,7 +152,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -170,24 +172,30 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    if len(txs) == 0:
-        for i in range(len(tx_hashes)):
-            o = get_tx(tx_hashes[i])
-            txs.append(o['signed_tx'])
-            if address == None:
-                address = o['address']
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
+
+    addresspass = None
+    if len(txs) == 0:
+        addresspass = []
+        for i in range(len(tx_hashes)):
+            o = get_tx(chain_spec_dict, tx_hashes[i])
+            txs.append(o['signed_tx'])
+            logg.debug('sender {}'.format(o))
+            tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
+            if address == None:
+                address = tx['from']
+            elif address != tx['from']:
+                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
+            addresspass.append(address)
 
-    #if not web3.Web3.isChecksumAddress(address):
     if not is_checksum_address(address):
         raise ValueError('invalid address {}'.format(address))
 
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
-    # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     gas_balance = 0
     try:
         o = balance(address)
@@ -198,6 +206,9 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
         conn.disconnect()
         raise EthError('gas_balance call for {}: {}'.format(address, e))
 
+    if gas_required == None:
+        gas_required = MAXIMUM_FEE_UNITS
+
     logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
     session = SessionBase.create_session()
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@@ -268,7 +279,8 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
             queue=queue,
             )
         ready_tasks.append(s)
-    celery.group(ready_tasks)()
+    t = celery.group(ready_tasks)()
+    logg.debug('group {}'.format(t))
 
     return txs
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -21,6 +21,7 @@ from chainqueue.db.models.tx import Otx
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db import SessionBase
@@ -58,6 +59,9 @@ def hashes_to_txs(self, tx_hashes):
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
+    for i in range(len(tx_hashes)):
+        tx_hashes[i] = strip_0x(tx_hashes[i])
+
     queue = self.request.delivery_info['routing_key']
 
     session = SessionBase.create_session()
@@ -148,7 +152,7 @@ def send(self, txs, chain_spec_dict):
 
 @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
 def sync_tx(self, tx_hash_hex, chain_spec_dict):
-    """Force update of network status of a simgle transaction
+    """Force update of network status of a single transaction
 
     :param tx_hash_hex: Transaction hash
     :type tx_hash_hex: str, 0x-hex
@@ -173,12 +177,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
 
     # TODO: apply receipt in tx object to validate and normalize input
     if rcpt != None:
+        rcpt = snake_and_camel(rcpt)
         success = rcpt['status'] == 1
-        logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
+        logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
 
         s = celery.signature(
             'cic_eth.queue.state.set_final',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 rcpt['blockNumber'],
                 rcpt['transactionIndex'],
@@ -186,12 +192,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
                 ],
                 queue=queue,
             )
+    # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
     else:
         logg.debug('sync tx {} mempool'.format(tx_hash_hex))
 
         s = celery.signature(
             'cic_eth.queue.state.set_sent',
             [
+                chain_spec_dict,
                 tx_hash_hex,
                 ],
                 queue=queue,

apps/cic-eth/cic_eth/queue/query.py

@@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r =  chainqueue.query.get_tx(chain_spec, tx_hash)
+    r =  chainqueue.query.get_tx(chain_spec, tx_hash, session=session)
     session.close()
     return r
 

apps/cic-eth/cic_eth/registry.py

@@ -5,29 +5,30 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
 
 logg = logging.getLogger()
 
 
-def connect_token_registry(rpc, chain_spec):
+def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    token_registry_address = registry.by_name('TokenRegistry')
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
     logg.debug('using token registry address {}'.format(token_registry_address))
     lookup = TokenIndexLookup(chain_spec, token_registry_address)
     CICRegistry.add_lookup(lookup)
 
 
-def connect_declarator(rpc, chain_spec, trusted_addresses):
+def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
     registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator')
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
     logg.debug('using declarator address {}'.format(declarator_address))
     lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
     CICRegistry.add_lookup(lookup)
 
 
-def connect(rpc, chain_spec, registry_address):
+def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry')
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
     return registry
 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -72,7 +72,9 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
+        c = Faucet(self.chain_spec)
+
+        o = c.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -17,7 +17,8 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 
 class GasFilter(SyncFilter):
@@ -27,11 +28,11 @@ class GasFilter(SyncFilter):
         self.chain_spec = chain_spec
 
 
-    def filter(self, conn, block, tx, session):
+    def filter(self, conn, block, tx, db_session):
         if tx.value > 0:
             tx_hash_hex = add_0x(tx.hash)
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(session)
+            session = SessionBase.bind_session(db_session)
             q = session.query(TxCache.recipient)
             q = q.join(Otx)
             q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@@ -56,7 +57,7 @@ class GasFilter(SyncFilter):
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
-                s.apply_async()
+                return s.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -50,7 +50,8 @@ class RegistrationFilter(SyncFilter):
                     queue=self.queue,
                     )
                 s_nonce.link(s_gift)
-                s_nonce.apply_async()
+                t = s_nonce.apply_async()
+                return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
         self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
 
 
-    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
+    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
 
         if tx.payload == None:
             logg.debug('no payload')
@@ -45,16 +45,17 @@ class TransferAuthFilter(SyncFilter):
             return False
 
         recipient = tx.inputs[0]
-        if recipient != self.transfer_request_contract.address():
+        #if recipient != self.transfer_request_contract.address():
+        if recipient != self.transfer_request_contract:
             logg.debug('not our transfer auth contract address {}'.format(recipient))
             return False
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
-           
-        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
-        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
-        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
-        value = abi_decode_single(ABIContractType.UINT256, r[3])
+          
+        sender = r[0]
+        recipient = r[1]
+        token = r[2]
+        value = r[3]
 
         token_data = {
             'address': token,
@@ -64,6 +65,7 @@ class TransferAuthFilter(SyncFilter):
             'cic_eth.eth.nonce.reserve_nonce',
             [
                 [token_data],
+                self.chain_spec.asdict(), 
                 sender,
                 ],
             queue=self.queue,
@@ -80,7 +82,7 @@ class TransferAuthFilter(SyncFilter):
             )
         s_nonce.link(s_approve)
         t = s_nonce.apply_async()
-        return True
+        return t
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -22,6 +22,7 @@ from chainlib.eth.connection import (
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
+from cic_eth_registry.erc20 import ERC20Token
 import liveness.linux
 
 
@@ -207,6 +208,11 @@ def main():
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
+    default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
+    default_token.load(rpc)
+    BaseTask.default_token_decimals = default_token.decimals
+    BaseTask.default_token_name = default_token.name
+
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    

apps/cic-eth/cic_eth/runnable/info.py

@@ -67,6 +67,8 @@ def main():
     token_info = t.get()
     print('Default token symbol: {}'.format(token_info['symbol']))
     print('Default token address: {}'.format(token_info['address']))
+    logg.debug('Default token name: {}'.format(token_info['name']))
+    logg.debug('Default token decimals: {}'.format(token_info['decimals']))
 
     
 if __name__ == '__main__':

apps/cic-eth/cic_eth/stat.py

@@ -20,7 +20,11 @@ def init_chain_stat(rpc, block_start=0):
     if block_start == 0:
         o = block_latest()
         r = rpc.do(o)
-        block_start = int(r, 16)
+        try:
+            block_start = int(r, 16)
+        except TypeError:
+            block_start = int(r)
+        logg.debug('blockstart {}'.format(block_start))
 
     for i in range(BLOCK_SAMPLES):
         o = block_by_number(block_start-10+i)

apps/cic-eth/cic_eth/task.py

@@ -20,7 +20,8 @@ import liveness.linux
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 celery_app = celery.current_app
 
@@ -33,6 +34,8 @@ class BaseTask(celery.Task):
     create_gas_oracle = RPCGasOracle
     default_token_address = None
     default_token_symbol = None
+    default_token_name = None
+    default_token_decimals = None
     run_dir = '/run'
 
     def create_session(self):
@@ -116,12 +119,13 @@ def registry():
     return CICRegistry.address
 
 
-@celery_app.task()
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
+@celery_app.task(bind=True, base=BaseTask)
+def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
+    r = registry.by_address(address, sender_address=self.call_address)
+    return r
 
 
 @celery_app.task(throws=(UnknownContractError,))
@@ -129,7 +133,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
+    return registry.by_name(name, sender_address=self.call_address)
 
 
 @celery_app.task()

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.13',
+        'beta.14',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,48 +1,62 @@
-# FROM grassrootseconomics:cic
-
-#FROM python:3.8.6-alpine
-FROM python:3.8.6-slim-buster
-
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+FROM python:3.8.6-slim-buster as compile
 
 WORKDIR /usr/src/cic-eth
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
-
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file:  ${root_requirement_file}"
-#COPY $root_requirement_file .
-#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
-RUN /usr/local/bin/python -m pip install --upgrade pip
-#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
-#	cd cic-base && \
-#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
-#	python merge_requirements.py | tee merged_requirements.txt 
-#RUN cd cic-base && \
-#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+#RUN python -m venv venv && . venv/bin/activate
 
-COPY cic-eth/scripts/ scripts/
-COPY cic-eth/setup.cfg cic-eth/setup.py ./
-COPY cic-eth/cic_eth/ cic_eth/
-# Copy app specific requirements
-COPY cic-eth/requirements.txt .
-COPY cic-eth/test_requirements.txt .
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+RUN /usr/local/bin/python -m pip install --upgrade pip
+RUN pip install semver
+
+# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
+COPY cic-eth/ .
 RUN	pip install $pip_extra_index_url_flag .
 
+# --- TEST IMAGE ---
+FROM python:3.8.6-slim-buster as test 
+
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+WORKDIR /usr/src/cic-eth
+
+RUN /usr/local/bin/python -m pip install --upgrade pip
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+# TODO we could use venv inside container to isolate the system and app deps further
+# COPY --from=compile /usr/src/cic-eth/ .
+# RUN . venv/bin/activate
+
+COPY cic-eth/test_requirements.txt .
+RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
+
+COPY cic-eth . 
+
+ENV PYTHONPATH .
+
+ENTRYPOINT ["pytest"]
+
+# --- RUNTIME ---
+FROM python:3.8.6-slim-buster as runtime 
+
+RUN apt-get update && \
+	apt install -y gnupg libpq-dev procps
+
+WORKDIR /usr/src/cic-eth
+
+COPY --from=compile /usr/local/bin/ /usr/local/bin/
+COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
-COPY cic-eth/tests/ tests/
 
+COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -51,3 +65,4 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
+

apps/cic-eth/requirements.txt

@@ -1,24 +1,25 @@
-cic-base~=0.1.2b11
+cic-base~=0.1.2b14
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.5a7
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
 eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a6
+erc20-transfer-authorization~=0.3.1a7
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.1a11
-chainlib~=0.0.3a2
+chainlib~=0.0.3rc2
 hexathon~=0.0.1a7
 chainsyncer[sql]~=0.0.2a4
-chainqueue~=0.0.2a2
+chainqueue~=0.0.2b1
 sarafu-faucet==0.0.3a3
 erc20-faucet==0.2.1a4
 coincurve==15.0.0
 potaahto~=0.0.1a2
+pycryptodome==3.10.1

apps/cic-eth/setup.py

@@ -11,17 +11,6 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
 setup(
-    install_requires=requirements,
-    tests_require=test_requirements,
+    install_requires=requirements
         )

apps/cic-eth/test_requirements.txt

@@ -4,4 +4,3 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/check/test_check_db.py

@@ -0,0 +1,8 @@
+# local imports
+from cic_eth.check.db import health
+
+def test_check_health(
+    init_database,
+        ):
+
+    assert health()

apps/cic-eth/tests/check/test_check_gas.py

@@ -0,0 +1,20 @@
+# local imports
+from cic_eth.check.gas import health
+from cic_eth.db.models.role import AccountRole
+
+def test_check_gas(
+    config,
+    init_database,
+    default_chain_spec,
+    eth_rpc,
+    custodial_roles,
+    whoever,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
+    assert health(config=config)
+
+    AccountRole.set('GAS_GIFTER', whoever, session=init_database)
+    init_database.commit()
+    assert not health(config=config)

apps/cic-eth/tests/check/test_check_redis.py

@@ -0,0 +1,16 @@
+# external imports
+import pytest
+
+# local imports
+from cic_eth.check.redis import health
+
+
+def test_check_redis(
+        config,
+        have_redis,
+        ):
+
+    if have_redis != None:
+        pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
+
+    assert health(unit='test', config=config)

apps/cic-eth/tests/check/test_check_signer.py

@@ -0,0 +1,13 @@
+# local imports
+from cic_eth.check.signer import health
+
+
+def test_check_signer(
+        default_chain_spec,
+        config,
+        eth_signer,
+        eth_rpc,
+        ):
+
+    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    assert health(config=config)

apps/cic-eth/tests/conftest.py

@@ -2,9 +2,11 @@
 import os
 import sys
 import logging
+import uuid
 
 # external imports
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
+import redis
 
 # local imports
 from cic_eth.api import Api
@@ -19,6 +21,7 @@ from tests.fixtures_config import *
 from tests.fixtures_database import *
 from tests.fixtures_celery import *
 from tests.fixtures_role import *
+from tests.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -55,3 +58,28 @@ def default_token(
         ):
     BaseTask.default_token_symbol = foo_token_symbol
     BaseTask.default_token_address = foo_token   
+
+
+
+@pytest.fixture(scope='session')
+def have_redis(
+        config,
+        ):
+
+    r = redis.Redis(
+            host = config.get('REDIS_HOST'),
+            port = config.get('REDIS_PORT'),
+            db = config.get('REDIS_DB'),
+            ) 
+    k = str(uuid.uuid4())
+    try:
+        r.set(k, 'foo')
+        r.delete(k)
+    except redis.exceptions.ConnectionError as e:
+        return e
+    except TypeError as e:
+        return e 
+
+    return None
+
+

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -14,9 +14,9 @@ from chainlib.eth.tx import (
         Tx,
         )
 from chainlib.eth.block import Block
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from sarafu_faucet import MinterFaucet
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
 from potaahto.symbols import snake_and_camel
 from hexathon import add_0x
 
@@ -26,7 +26,6 @@ from cic_eth.runnable.daemons.filters.callback import CallbackFilter
 logg = logging.getLogger()
 
 
-@pytest.mark.skip()
 def test_transfer_tx(
         default_chain_spec,
         init_database,
@@ -66,7 +65,6 @@ def test_transfer_tx(
     assert transfer_type == 'transfer'
 
 
-@pytest.mark.skip()
 def test_transfer_from_tx(
         default_chain_spec,
         init_database,

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -0,0 +1,38 @@
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+from cic_eth.runnable.daemons.filters.callback import CallbackFilter
+from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
+from cic_eth.runnable.daemons.filters.tx import TxFilter
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+# Hit tx mismatch paths on all filters
+def test_filter_bogus(
+        init_database,
+        bogus_tx_block,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        transfer_auth,
+        cic_registry,
+        contract_roles,
+        register_lookups,
+        ):
+
+    fltrs = [
+        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
+        GasFilter(default_chain_spec, queue=None),
+        TxFilter(default_chain_spec, None),
+        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
+        StragglerFilter(default_chain_spec, None),
+        RegistrationFilter(default_chain_spec, queue=None),
+        ]
+      
+    for fltr in fltrs:
+        r = None
+        try:
+            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
+        except:
+            pass
+        assert not r

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -0,0 +1,101 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.tx import create as queue_create
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.state import (
+        set_waitforgas,
+        )
+from hexathon import strip_0x
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.eth.gas import cache_gas_data
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
+    init_database.commit()
+    tx_hash_hex_wait = tx_hash_hex
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['BOB'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    fltr = GasFilter(default_chain_spec, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

apps/cic-eth/tests/filters/test_register_filter.py

@@ -0,0 +1,78 @@
+# external imports
+from eth_accounts_index.registry import AccountRegistry
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import(
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from erc20_faucet import Faucet
+from hexathon import strip_0x
+from chainqueue.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+
+
+def test_register_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        account_registry,
+        faucet,
+        register_lookups,
+        contract_roles,
+        agent_roles,
+        cic_registry,
+        init_celery_tasks,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
+
+    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block, rcpt=rcpt)
+    tx.apply_receipt(rcpt)
+
+    fltr = RegistrationFilter(default_chain_spec, queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(gift_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    gift = Faucet.parse_give_to_request(gift_tx['data'])
+    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -0,0 +1,79 @@
+# external imports
+from erc20_transfer_authorization import TransferAuthorization
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from hexathon import strip_0x
+from chainqueue.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+
+
+def test_filter_transferauth(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        transfer_auth,
+        foo_token,
+        celery_session_worker,
+        register_lookups,
+        init_custodial,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
+    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
+    
+    r = rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(approve_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    c = ERC20(default_chain_spec)
+    approve = c.parse_approve_request(approve_tx['data']) 
+    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -23,7 +23,6 @@ from chainqueue.state import (
         set_ready,
         set_sent,
         )
-
 from hexathon import strip_0x
 
 # local imports
@@ -31,7 +30,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
 from cic_eth.eth.gas import cache_gas_data
 
 
-def test_tx(
+def test_filter_tx(
         default_chain_spec,
         init_database,
         eth_rpc,

apps/cic-eth/tests/fixtures_celery.py

@@ -22,7 +22,6 @@ def init_celery_tasks(
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
-#        'cic_eth.eth.bancor',
         'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
@@ -47,8 +46,8 @@ def celery_config():
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
     rq = tempfile.mkdtemp()
-    logg.debug('celery broker queue {} processed {}'.format(bq, bp))
-    logg.debug('celery backend store {}'.format(rq))
+    logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
+    logg.debug('celery backend session store {}'.format(rq))
     yield {
         'broker_url': 'filesystem://',
         'broker_transport_options': {
@@ -58,12 +57,11 @@ def celery_config():
             },
         'result_backend': 'file://{}'.format(rq),
             }
-    logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq))
+    logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
     shutil.rmtree(bq)
     shutil.rmtree(bp)
     shutil.rmtree(rq)
 
-
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {

apps/cic-eth/tests/fixtures_contract.py

@@ -0,0 +1,77 @@
+# standard imports
+import os
+
+# external imports
+import pytest
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        TxFactory,
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from hexathon import strip_0x
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.dirname(script_dir)
+
+
+@pytest.fixture(scope='function')
+def bogus_tx_block(
+    default_chain_spec,
+    eth_rpc,
+    eth_signer,
+    contract_roles,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
+
+    f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
+    bytecode = f.read()
+    f.close()
+
+    c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
+    tx = c.set_code(tx, bytecode)
+    (tx_hash_hex, o) = c.build(tx)
+
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+
+    contract_address = r['contract_address']
+
+    enc = ABIContractEncoder()
+    enc.method('poke')
+    data = enc.get()
+    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
+    tx = c.set_code(tx, data)
+    (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
+    r = eth_rpc.do(o)
+    tx_signed_raw_hex = strip_0x(o['params'][0])
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    return (block, tx)

apps/cic-eth/tests/task/api/test_admin.py

@@ -9,8 +9,14 @@ from chainlib.eth.tx import (
         unpack,
         TxFormat,
         )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import Gas
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
 from chainlib.eth.address import to_checksum_address
 from hexathon import (
         strip_0x,
@@ -23,7 +29,15 @@ from chainqueue.db.enum import (
         StatusBits,
         status_str,
         )
-from chainqueue.query import get_tx
+from chainqueue.state import (
+        set_fubar,
+        set_ready,
+        set_reserved,
+        )
+from chainqueue.query import (
+        get_tx,
+        get_nonce_tx_cache,
+        )
 
 # local imports
 from cic_eth.api import AdminApi
@@ -36,150 +50,6 @@ from cic_eth.queue.tx import queue_create
 logg = logging.getLogger()
 
 
-#def test_resend_inplace(
-#    default_chain_spec,
-#    init_database,
-#    init_w3,
-#    celery_session_worker,
-#    ):
-#    
-#    chain_str = str(default_chain_spec)
-#    c = RpcClient(default_chain_spec)
-#    
-#    sigs = []
-#
-#    gas_provider = c.gas_provider()
-#
-#    s_nonce = celery.signature(
-#        'cic_eth.eth.nonce.reserve_nonce',
-#        [
-#            init_w3.eth.accounts[0],
-#            gas_provider,
-#            ],
-#        queue=None,
-#        )
-#    s_refill = celery.signature(
-#        'cic_eth.eth.gas.refill_gas',
-#        [
-#            chain_str,
-#            ],
-#        queue=None,
-#            )
-#    s_nonce.link(s_refill)
-#    t = s_nonce.apply_async()
-#    t.get()
-#    for r in t.collect():
-#        pass
-#    assert t.successful()
-#
-#    q = init_database.query(Otx)
-#    q = q.join(TxCache)
-#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
-#    o = q.first()
-#    tx_raw = o.signed_tx
-#
-#    tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
-#    gas_price_before = tx_dict['gasPrice'] 
-#
-#    s = celery.signature(
-#        'cic_eth.admin.ctrl.lock_send',
-#        [
-#            chain_str,
-#            init_w3.eth.accounts[0],
-#            ],
-#        queue=None,
-#            )
-#    t = s.apply_async()
-#    t.get()
-#    assert t.successful()
-#
-#    api = AdminApi(c, queue=None)
-#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
-#    t.get()
-#    i = 0
-#    tx_hash_new_hex = None
-#    for r in t.collect():
-#        tx_hash_new_hex = r[1]
-#    assert t.successful()
-#  
-#    tx_raw_new = get_tx(tx_hash_new_hex) 
-#    logg.debug('get {}'.format(tx_raw_new))
-#    tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
-#    assert tx_hash_new_hex != tx_dict['hash']
-#    assert tx_dict_new['gasPrice'] > gas_price_before
-#
-#    tx_dict_after = get_tx(tx_dict['hash'])
-#
-#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
-#    assert tx_dict_after['status'] & StatusBits.MANUAL
-
-
-#def test_check_fix_nonce(
-#    default_chain_spec,
-#    init_database,
-#    init_eth_account_roles,
-#    init_w3,
-#    eth_empty_accounts,
-#    celery_session_worker,
-#    ):
-#
-#    chain_str = str(default_chain_spec)
-#    
-#    sigs = []
-#    for i in range(5):
-#        s = celery.signature(
-#            'cic_eth.eth.gas.refill_gas',
-#            [
-#                eth_empty_accounts[i],
-#                chain_str,
-#                ],
-#            queue=None,
-#                )
-#        sigs.append(s)
-#
-#    t = celery.group(sigs)()
-#    txs = t.get()
-#    assert t.successful()
-#
-#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
-#    c = RpcClient(default_chain_spec)
-#    api = AdminApi(c, queue=None)
-#    address = init_eth_account_roles['eth_account_gas_provider']
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['network'] == 0
-#    assert nonce_spec['nonce']['queue'] == 4
-#    assert nonce_spec['nonce']['blocking'] == None
-#
-#    s_set = celery.signature(
-#            'cic_eth.queue.tx.set_rejected',
-#            [
-#                tx_hash.hex(),
-#                ],
-#            queue=None,
-#            )
-#    t = s_set.apply_async()
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#
-#    nonce_spec = api.check_nonce(address)
-#    assert nonce_spec['nonce']['blocking'] == 2
-#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
-#
-#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
-#    t.get()
-#    t.collect()
-#    assert t.successful()
-#
-#    for tx in txs[3:]:
-#        tx_hash = web3.Web3.keccak(hexstr=tx)
-#        tx_dict = get_tx(tx_hash.hex())
-#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
-#
-#
-
-
 def test_have_account(
     default_chain_spec,
     custodial_roles,
@@ -243,28 +113,6 @@ def test_tag_account(
     assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
-#def test_ready(
-#    init_database,
-#    agent_roles,
-#    eth_rpc,
-#    ):
-#
-#    api = AdminApi(eth_rpc)
-#   
-#    with pytest.raises(InitializationError):
-#        api.ready()
-#
-#    bogus_account = os.urandom(20)
-#    bogus_account_hex = '0x' + bogus_account.hex()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
-#    with pytest.raises(KeyError):
-#        api.ready()
-#
-#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
-#    api.ready()
-
-
 def test_tx(
     default_chain_spec,
     cic_registry,
@@ -286,3 +134,168 @@ def test_tx(
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
     tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
     logg.warning('code missing to verify tx contents {}'.format(tx))
+
+
+def test_check_nonce_gap(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        if i == 3:
+            j = 1
+            nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
+
+        queue_create(
+                default_chain_spec,
+                i+j,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
+
+
+def test_check_nonce_localfail(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    # NOTE: this only works as long as agents roles start at nonce 0
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    j = 0
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    set_ready(default_chain_spec, tx_hashes[4], session=init_database)
+    set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
+    set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
+
+    assert r['nonce']['blocking'] == 4
+    assert r['tx']['blocking'] == tx_hashes[4]
+
+
+def test_fix_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        init_celery_tasks,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+        queue_create(
+                default_chain_spec,
+                i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
+    t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
+    r = t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+    
+    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
+    ks = txs.keys()
+    assert len(ks) == 2
+    for k in ks:
+        hsh = add_0x(k)
+        otx = Otx.load(hsh, session=init_database)
+        init_database.refresh(otx)
+        logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
+        if add_0x(k) == tx_hashes[3]:
+            assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+        else:
+            assert otx.status == 1

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -0,0 +1,373 @@
+# standard imports
+import logging
+import io
+import json
+
+# external imports
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        nonce,
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainqueue.tx import create as queue_create
+from chainlib.eth.tx import (
+        raw,
+        receipt,
+        TxFormat,
+        Tx,
+        )
+from chainlib.eth.block import block_latest
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainqueue.state import (
+        set_reserved,
+        set_sent,
+        set_ready,
+        )
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from chainqueue.query import get_nonce_tx_cache
+from eth_erc20 import ERC20
+from cic_eth_registry import CICRegistry
+
+# local imports
+from cic_eth.api.api_admin import AdminApi
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.eth.erc20 import cache_transfer_data
+
+logg = logging.getLogger()
+
+
+def test_admin_api_tx(
+        default_chain_spec,
+        init_database,
+        init_celery_tasks,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        custodial_roles,
+        celery_session_worker,
+        foo_token,
+        address_declarator,
+        cic_registry,
+        register_tokens,
+        register_lookups,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
+
+    o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
+    r = eth_rpc.do(o)
+    gifter_nonce = int(r, 16)
+
+    #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            gifter_nonce, # will only work if agent starts at 0
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_transfer_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    # lookup by transaction hash, without registry
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
+    logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by RLP transaction, without registry
+    tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, with registry
+    registry = CICRegistry(default_chain_spec, eth_rpc)
+    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
+    assert tx['tx_hash'] == tx_hash_hex 
+
+    # lookup by transaction hash, using writer
+    buf = io.StringIO()
+    api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex 
+
+
+def test_admin_api_account(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes_alice = []
+    txs_alice = []
+
+    for i in range(3):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_alice.append(tx_hash_hex)
+        txs_alice.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
+    tx_hashes_bob = []
+    txs_bob = []
+
+    for i in range(2):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                13+i,
+                agent_roles['BOB'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes_bob.append(tx_hash_hex)
+        txs_bob.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'])
+    assert len(r) == 5
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
+    assert len(r) == 2
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
+    assert len(r) == 3
+
+
+def test_admin_api_account_writer(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    buf = io.StringIO()
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
+
+    # TODO: improve eval
+    tx = json.loads(buf.getvalue())
+    assert tx['tx_hash'] == tx_hash_hex
+
+
+def test_registry(
+        eth_rpc,
+        cic_registry,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.registry()
+    r = t.get_leaf()
+    assert r == cic_registry
+
+
+def test_proxy_do(
+        default_chain_spec,
+        eth_rpc,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.proxy_do(default_chain_spec, o)
+    rr = t.get_leaf()
+
+    assert r == rr
+
+
+def test_resend_inplace(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
+    r = t.get_leaf()
+    assert t.successful()
+
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 2
+
+
+
+@pytest.mark.xfail()
+def test_resend_clone(
+        init_database,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    init_database.commit()
+
+    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
+    r = t.get_leaf()
+    assert t.successful()
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    assert len(txs) == 1
+
+    otx = Otx.load(txs[0], session=init_database)
+    assert otx.status == 0

apps/cic-eth/tests/task/api/test_app.py

@@ -8,11 +8,20 @@ import pytest
 import celery
 from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
+from eth_accounts_index import AccountsIndex
+from chainlib.eth.tx import (
+        transaction,
+        )
+from chainqueue.state import (
+        set_reserved,
+        )
 
 # local imports
 from cic_eth.api import Api
+from cic_eth.queue.query import get_tx
 
-logg = logging.getLogger(__name__)
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 def test_account_api(
@@ -29,6 +38,47 @@ def test_account_api(
     assert t.successful()
 
 
+def test_account_api_register(
+        default_chain_spec,
+        init_database,
+        account_registry,
+        faucet,
+        custodial_roles,
+        cic_registry,
+        register_lookups,
+        eth_rpc,
+        celery_session_worker,
+        ):
+    api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
+    t = api.create_account('')
+    register_tx_hash = t.get_leaf()
+    assert t.successful()
+
+    set_reserved(default_chain_spec, register_tx_hash, session=init_database)
+
+    tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                [tx['signed_tx']],
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = transaction(register_tx_hash)
+    tx_src = eth_rpc.do(o)
+
+    c = AccountsIndex(default_chain_spec)
+    address = c.parse_add_request(tx_src['data'])
+    o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    assert c.parse_have(r)
+
+
 def test_transfer_api(
         default_chain_spec,
         eth_rpc,
@@ -37,16 +87,15 @@ def test_transfer_api(
         custodial_roles,
         agent_roles,
         cic_registry,
-        register_tokens,
+        token_registry,
         register_lookups,
         celery_session_worker,
+        register_tokens,
+        foo_token_symbol,
         ):
 
-    #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
-    foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
-
     api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol)
+    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) 
     t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -0,0 +1,19 @@
+# local imports
+from cic_eth.api.api_task import Api
+from cic_eth.task import BaseTask
+
+def test_default_token(
+        default_chain_spec,
+        foo_token,
+        default_token,
+        token_registry,
+        register_tokens,
+        register_lookups,
+        cic_registry,
+        celery_session_worker,
+        ):
+
+    api = Api(str(default_chain_spec), queue=None)     
+    t = api.default_token()
+    r = t.get_leaf()
+    assert r['address'] == foo_token

apps/cic-eth/tests/task/api/test_list.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.tx import receipt
 
 # local imports

apps/cic-eth/tests/task/test_task_account.py

@@ -9,7 +9,7 @@ import celery
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import receipt
-from eth_accounts_index import AccountRegistry
+from eth_accounts_index.registry import AccountRegistry
 from hexathon import strip_0x
 from chainqueue.db.enum import StatusEnum
 from chainqueue.db.models.otx import Otx
@@ -156,6 +156,7 @@ def test_gift(
     eth_signer,
     init_celery_tasks,
     cic_registry,
+    register_lookups,
     celery_session_worker,
     ):
 

apps/cic-eth/tests/task/test_task_admin.py

@@ -0,0 +1,88 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.tx import (
+        create as queue_create,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import TxFormat
+from chainqueue.query import get_nonce_tx_cache
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from hexathon import add_0x
+
+# local imports
+from cic_eth.admin.nonce import shift_nonce
+from cic_eth.eth.gas import cache_gas_data
+
+logg = logging.getLogger()
+
+
+def test_shift_nonce(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
+
+    tx_hashes = []
+    txs = []
+
+    for i in range(10):
+        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+        queue_create(
+                default_chain_spec,
+                42+i,
+                agent_roles['ALICE'],
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                session=init_database,
+                )
+        cache_gas_data(
+                tx_hash_hex,
+                tx_signed_raw_hex,
+                default_chain_spec.asdict(),
+                )
+        tx_hashes.append(tx_hash_hex)
+        txs.append(tx_signed_raw_hex)
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.admin.nonce.shift_nonce',
+            [
+                default_chain_spec.asdict(),
+                tx_hashes[3],
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+
+    for i in range(42+3, 42+10):
+        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
+        for k in txs.keys():
+            hsh = add_0x(k)
+            otx = Otx.load(hsh, session=init_database)
+            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
+            if add_0x(k) == tx_hashes[i-42]:
+                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+            else:
+                assert otx.status == 1

apps/cic-eth/tests/task/test_task_erc20.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,

apps/cic-eth/tests/task/test_task_gas.py

@@ -0,0 +1,286 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        unpack,
+        TxFormat,
+        )
+from chainlib.eth.constant import (
+        MINIMUM_FEE_UNITS,
+        MINIMUM_FEE_PRICE,
+        )
+from chainqueue.tx import create as queue_create
+from chainqueue.query import get_tx
+from chainqueue.db.enum import StatusBits
+from chainqueue.state import (
+        set_ready,
+        set_reserved,
+        set_sent,
+        )
+from chainqueue.db.models.otx import Otx
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.error import OutOfGasError
+
+logg = logging.getLogger()
+
+
+def test_task_check_gas_ok(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                8000000,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+def test_task_check_gas_insufficient(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(whoever, 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                [],
+                None,
+                None,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    try:
+        r = t.get_leaf()
+    except OutOfGasError:
+        pass
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+
+def test_task_check_gas_low(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        init_database,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
+    nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
+    r = eth_rpc.do(o)
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) 
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    init_database.commit()
+
+    s = celery.signature(
+            'cic_eth.eth.gas.check_gas',
+            [
+                [
+                    tx_hash_hex,
+                    ],
+                default_chain_spec.asdict(),
+                ],
+                [],
+                None,
+                None,
+            queue=None
+            )
+    t = s.apply_async()
+    t.get_leaf()
+    assert t.successful()
+
+    init_database.commit()
+
+    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
+    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
+
+
+@pytest.mark.parametrize(
+        '_gas_price,_gas_factor',
+        [
+        (None, 1.1),
+        (MINIMUM_FEE_PRICE * 1.1, 0.9),
+        (None, 1.3),
+        ]
+        )
+def test_task_resend_explicit(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        custodial_roles,
+        celery_session_worker,
+        _gas_price,
+        _gas_factor,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            0,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+
+    init_database.commit()
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    s = celery.signature(
+            'cic_eth.eth.gas.resend_with_higher_gas',
+            [
+                tx_hash_hex,
+                default_chain_spec.asdict(),
+                _gas_price,
+                _gas_factor,
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(Otx)
+    q = q.filter(Otx.tx_hash==strip_0x(r))
+    otx = q.first()
+    if otx == None:
+        raise NotLocalTxError(r)
+
+    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
+    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
+    assert tx_after['gasPrice'] > tx_before['gasPrice']
+

apps/cic-eth/tests/task/test_task_tx.py

@@ -4,16 +4,27 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.gas import Gas
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
         transaction,
         receipt,
+        raw,
         )
 from hexathon import strip_0x
 from chainqueue.db.models.otx import Otx
+from chainqueue.tx import create as queue_create
+from chainqueue.state import (
+        set_reserved,
+        set_ready,
+        set_sent,
+        )
+from chainqueue.db.enum import StatusBits
 
 # local imports
 from cic_eth.queue.tx import register_tx
@@ -60,15 +71,6 @@ def test_tx_send(
 
 
 def test_sync_tx(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-    pass
-
-
-def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,
         eth_rpc,
@@ -77,31 +79,48 @@ def test_resend_with_higher_gas(
         celery_session_worker,
         ):
 
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
-    register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
-    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
-    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    o = raw(tx_signed_raw_hex)
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
 
     s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
+            'cic_eth.eth.tx.sync_tx',
             [
                 tx_hash_hex,
                 default_chain_spec.asdict(),
                 ],
-            queue=None,
+            queue=None
             )
     t = s.apply_async()
     r = t.get_leaf()
+    assert t.successful() 
 
-    q = init_database.query(Otx)
-    q = q.filter(Otx.tx_hash==strip_0x(r))
-    otx = q.first()
-    if otx == None:
-        raise NotLocalTxError(r)
-
-    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
-    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
-    assert tx_after['gasPrice'] > tx_before['gasPrice']
+    init_database.commit()
 
+    o = Otx.load(tx_hash_hex, session=init_database)
+    assert o.status & StatusBits.FINAL == StatusBits.FINAL

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -0,0 +1,170 @@
+# standard imports
+import os
+import logging
+
+# external imports
+import pytest
+import celery
+from chainqueue.tx import create as queue_create
+from chainlib.eth.nonce import (
+        RPCNonceOracle,
+        OverrideNonceOracle,
+        )
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        Gas,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        receipt,
+        )
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainqueue.state import (
+        set_reserved,
+        set_ready,
+        )
+
+logg = logging.getLogger()
+
+
+def test_hashes_to_txs(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+
+    #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    init_database.commit()
+
+    bogus_one = add_0x(os.urandom(32).hex())
+    bogus_two = add_0x(os.urandom(32).hex())
+
+    yarrgs = [
+                bogus_one,
+                tx_hash_hex_two,
+                bogus_two,
+                tx_hash_hex_one,
+            ]   
+    s = celery.signature(
+        'cic_eth.eth.tx.hashes_to_txs',
+        [
+            yarrgs,
+                ],
+        queue=None,
+        )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+    assert len(r) == 2
+
+    logg.debug('r {}'.format(r))
+    txs = [
+        tx_signed_raw_hex_two,
+        tx_signed_raw_hex_one,
+            ]
+    for tx in r:
+        txs.remove(add_0x(tx))
+    assert len(txs) == 0
+
+
+
+def test_double_send(
+        init_database,
+        default_chain_spec,
+        agent_roles,            
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex_one,
+            tx_signed_raw_hex_one,
+            session=init_database,
+            )
+    set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['ALICE'],
+            tx_hash_hex_two,
+            tx_signed_raw_hex_two,
+            session=init_database,
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
+    init_database.commit()
+
+    yarrgs = [
+        tx_signed_raw_hex_one,
+        tx_signed_raw_hex_two,
+            ]
+    s = celery.signature(
+            'cic_eth.eth.tx.send',
+            [
+                yarrgs,
+                default_chain_spec.asdict(),
+                ],
+            queue=None
+            )
+    t = s.apply_async()
+    r = t.get_leaf()
+    assert t.successful()
+
+    o = receipt(tx_hash_hex_one)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    o = receipt(tx_hash_hex_two)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+
+

apps/cic-eth/tests/testdata/Bogus.bin

@@ -0,0 +1 @@
+60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

apps/cic-eth/tests/testdata/Bogus.sol

@@ -0,0 +1,10 @@
+pragma solidity ^0.8.0;
+
+contract Bogus {
+
+	uint256 public a = 0;
+
+	function poke() public {
+		a++;
+	}
+}

apps/cic-eth/tests/unit/ext/test_address.py

@@ -3,7 +3,7 @@ from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
         )
-from eth_address_declarator import AddressDeclarator
+from eth_address_declarator import Declarator
 from hexathon import add_0x
 
 # local imports
@@ -19,11 +19,12 @@ def test_translate(
         agent_roles,
         cic_registry,
         init_celery_tasks,
+        register_lookups,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
 
-    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    c = Declarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
 
     description = 'alice'.encode('utf-8').ljust(32, b'\x00').hex()
     (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], add_0x(description))

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -8,7 +8,7 @@ from chainlib.eth.tx import (
         count,
         receipt,
         )
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 
 # local imports

apps/cic-eth/tests/unit/test_registry_connect.py

@@ -0,0 +1,22 @@
+# local imports
+from cic_eth.registry import *
+
+def test_registry_connect(
+        eth_rpc,
+        default_chain_spec,
+        address_declarator,
+        token_registry,
+        contract_roles,
+        purge_lookups,
+        registry,
+        agent_roles,
+        ):
+
+    r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])
+

apps/cic-eth/tests/unit/test_stat.py

@@ -0,0 +1,18 @@
+# standard imports
+import datetime
+
+# local imports
+from cic_eth.stat import init_chain_stat
+
+
+def test_chain_stat(
+        eth_rpc,
+        init_eth_tester,
+        ):
+
+    now = int(datetime.datetime.now().timestamp()) + 1
+    for i in range(11):
+        init_eth_tester.time_travel(now + (i * 2))
+    
+    s = init_chain_stat(eth_rpc, block_start=0)
+    assert s.block_average() == 2

apps/cic-meta/bin/get.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+const colors = require('colors');
+const {Meta} = require("../dist");
+
+let { argv } = require('yargs')
+    .usage('Usage: $0 -m http://localhost:63380 -n publickeys')
+    .example(
+        '$0 -m http://localhost:63380 -n publickeys',
+        'Fetches the public keys blob from the meta server'
+    )
+    .option('m', {
+        alias: 'metaurl',
+        describe: 'The URL for the meta service',
+        demandOption: 'The meta url is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('n', {
+        alias: 'name',
+        describe: 'The name of the resource to be fetched from the meta service',
+        demandOption: 'The name of the resource is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('t', {
+        alias: 'type',
+        describe: 'The type of resource to be fetched from the meta service\n' +
+            'Options: `user`, `phone` and `custom`\n' +
+            'Defaults to `custom`',
+        type: 'string',
+        nargs: 1,
+    })
+    .epilog('Grassroots Economics (c) 2021')
+    .wrap(null);
+
+const metaUrl = argv.m;
+const resourceName = argv.n;
+let type = argv.t;
+if (type === undefined) {
+    type = 'custom'
+}
+
+(async () => {
+    const identifier = await Meta.getIdentifier(resourceName, type);
+    console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
+    const metaResponse = await Meta.get(identifier, metaUrl);
+    if (typeof metaResponse !== "object") {
+        console.error(colors.red('Metadata get failed!'));
+    }
+    console.log(colors.green(metaResponse));
+})();

apps/cic-meta/bin/set.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const colors = require('colors');
+const {Meta} = require("../dist");
+
+let { argv } = require('yargs')
+    .usage('Usage: $0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc')
+    .example(
+        '$0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc',
+        'Updates the public keys blob to the meta server'
+    )
+    .option('m', {
+        alias: 'metaurl',
+        describe: 'The URL for the meta service',
+        demandOption: 'The meta url is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('k', {
+        alias: 'privatekey',
+        describe: 'The PGP private key blob file used to sign the changes to the meta service',
+        demandOption: 'The private key file is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('n', {
+        alias: 'name',
+        describe: 'The name of the resource to be set or updated to the meta service',
+        demandOption: 'The name of the resource is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('r', {
+        alias: 'resource',
+        describe: 'The resource file to be set or updated to the meta service',
+        demandOption: 'The resource file is required',
+        type: 'string',
+        nargs: 1,
+    })
+    .option('t', {
+        alias: 'type',
+        describe: 'The type of resource to be set or updated to the meta service\n' +
+            'Options: `user`, `phone` and `custom`\n' +
+            'Defaults to `custom`',
+        type: 'string',
+        nargs: 1,
+    })
+    .epilog('Grassroots Economics (c) 2021')
+    .wrap(null);
+
+const metaUrl = argv.m;
+const privateKeyFile = argv.k;
+const resourceName = argv.n;
+const resourceFile = argv.r;
+let type = argv.t;
+if (type === undefined) {
+    type = 'custom'
+}
+
+const privateKey = readFile(privateKeyFile);
+const resource = readFile(resourceFile);
+
+(async () => {
+    if (privateKey && resource) {
+        const identifier = await Meta.getIdentifier(resourceName, type);
+        console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
+        const meta = new Meta(metaUrl, privateKey);
+        meta.onload = async (status) => {
+            const response = await meta.set(identifier, resource)
+            console.log(colors.green(response));
+        }
+    }
+})();
+
+function readFile(filename) {
+    if(!fs.existsSync(filename)) {
+        console.log(colors.red(`File ${filename} not found`));
+        return;
+    }
+    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
+}

apps/cic-meta/package.json

@@ -1,9 +1,14 @@
 {
-	"name": "cic-client-meta",
-	"version": "0.0.7-alpha.8",
+	"name": "@cicnet/cic-client-meta",
+	"version": "0.0.11",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
+	"bin": {
+		"meta-set": "bin/set.js",
+		"meta-get": "bin/get.js"
+	},
+	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
@@ -11,12 +16,14 @@
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
 		"prepare": "npm run build && npm run build-server",
-                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
+		"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts",
+		"publish": "npm publish --access public"
 	},
 	"dependencies": {
+		"@cicnet/crdt-meta": "^0.0.10",
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
-		"crdt-meta": "0.0.8",
+		"colors": "^1.4.0",
 		"ethereumjs-wallet": "^1.0.1",
 		"ini": "^1.3.8",
 		"openpgp": "^4.10.8",

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-import { Config } from 'crdt-meta';
+import { Config } from '@cicnet/crdt-meta';
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {

apps/cic-meta/scripts/server/handlers.ts

@@ -1,7 +1,7 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
 
-import { Envelope, Syncable } from 'crdt-meta';
+import { Envelope, Syncable } from '@cicnet/crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {

apps/cic-meta/scripts/server/server.ts

@@ -3,7 +3,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import * as handlers from './handlers';
-import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
+import { PGPKeyStore, PGPSigner, Config } from '@cicnet/crdt-meta';
+import { SqliteAdapter, PostgresAdapter } from '../../src/db';
 
 import { standardArgs } from './args';
 

apps/cic-meta/src/custom.ts

@@ -0,0 +1,27 @@
+import {Addressable, mergeKey, Syncable} from "@cicnet/crdt-meta";
+
+class Custom extends Syncable implements Addressable {
+
+    name:	string
+    value:	Object
+
+    constructor(name:string, v:Object={}) {
+        super('', v);
+        Custom.toKey(name).then((cid) => {
+            this.id = cid;
+            this.value = v;
+        });
+    }
+
+    public static async toKey(item:string, identifier: string = ':cic.custom') {
+        return await mergeKey(Buffer.from(item), Buffer.from(identifier));
+    }
+
+    public key(): string {
+        return this.id;
+    }
+}
+
+export {
+    Custom,
+}

apps/cic-meta/src/db.ts

@@ -0,0 +1,90 @@
+import * as pg from 'pg';
+import * as sqlite from 'sqlite3';
+
+type DbConfig = {
+	name:		string
+	host:		string
+	port:		number
+	user:		string
+	password:	string	
+}
+
+interface DbAdapter {
+	query:	(s:string, callback:(e:any, rs:any) => void) => void
+	close: 	() => void
+}
+
+const re_creatematch = /^(CREATE)/i
+const re_getmatch = /^(SELECT)/i;
+const re_setmatch = /^(INSERT|UPDATE)/i;
+
+class SqliteAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig, callback?:(any) => void) {
+		this.db = new sqlite.Database(dbConfig.name); //, callback);
+	}
+
+	public query(s:string, callback:(e:any, rs?:any) => void): void {
+		const local_callback = (e, rs) => {
+			let r = undefined;
+			if (rs !== undefined) {
+				r = {
+					rowCount: rs.length,
+					rows: rs,
+				}
+			}
+			callback(e, r);
+		};
+		if (s.match(re_getmatch)) {
+			this.db.all(s, local_callback);
+		} else if (s.match(re_setmatch)) {
+			this.db.run(s, local_callback);
+		} else if (s.match(re_creatematch)) {
+			this.db.run(s, callback);
+		} else {
+			throw 'unhandled query';
+		}
+	}
+
+	public close() {
+		this.db.close();
+	}
+}
+
+class PostgresAdapter implements DbAdapter {
+
+	db:		any
+
+	constructor(dbConfig:DbConfig) {
+		let o = dbConfig;
+		o['database'] = o.name;
+		this.db = new pg.Pool(o);
+		return this.db;
+	}
+
+	public query(s:string, callback:(e:any, rs:any) => void): void {
+		this.db.query(s, (e, rs) => {
+			let r = {
+				length: rs.rowCount,
+			}
+			rs.length = rs.rowCount;
+			if (e === undefined) {
+				e = null;
+			}
+			console.debug(e, rs);
+			callback(e, rs);
+		});
+	}
+
+	public close() {
+		this.db.end();
+	}
+}
+
+export {
+	DbConfig,
+	SqliteAdapter,
+	PostgresAdapter,
+}

apps/cic-meta/src/index.ts

@@ -1,2 +1,4 @@
 export { User } from './user';
 export { Phone } from './phone';
+export { Custom } from './custom';
+export { Meta } from './meta';

apps/cic-meta/src/meta.ts

@@ -0,0 +1,126 @@
+import {ArgPair, Envelope, Syncable, MutablePgpKeyStore, PGPSigner} from "@cicnet/crdt-meta";
+import {User} from "./user";
+import {Phone} from "./phone";
+import {Custom} from "./custom";
+const fetch = require("node-fetch");
+
+const headers =  {
+    'Content-Type': 'application/json;charset=utf-8',
+    'x-cic-automerge': 'client'
+};
+const options = {
+    headers: headers,
+};
+
+class Meta {
+    keystore: MutablePgpKeyStore = new MutablePgpKeyStore();
+    signer: PGPSigner = new PGPSigner(this.keystore);
+    metaUrl: string;
+    private privateKey: string;
+    onload: (status: boolean) => void;
+
+    constructor(metaUrl: string, privateKey: any) {
+        this.metaUrl = metaUrl;
+        this.privateKey = privateKey;
+        this.keystore.loadKeyring().then(() => {
+            this.keystore.importPrivateKey(privateKey).then(() => this.onload(true));
+        });
+    }
+
+    async set(identifier: string, data: Object): Promise<any> {
+        let syncable: Syncable;
+        const response = await Meta.get(identifier, this.metaUrl);
+        if (response === `Request to ${this.metaUrl}/${identifier} failed. Connection error.`) {
+            return response;
+        } else if (typeof response !== "object" || typeof data !== "object") {
+            syncable = new Syncable(identifier, data);
+            const res = await this.updateMeta(syncable, identifier);
+            return `${res.status}: ${res.statusText}`;
+        } else {
+            syncable = await Meta.get(identifier, this.metaUrl);
+            let update: Array<ArgPair> = [];
+            for (const prop in data) {
+                update.push(new ArgPair(prop, data[prop]));
+            }
+            syncable.update(update, 'client-branch');
+            const res = await this.updateMeta(syncable, identifier);
+            return `${res.status}: ${res.statusText}`;
+        }
+    }
+
+    async updateMeta(syncable: Syncable, identifier: string): Promise<any> {
+        const envelope: Envelope = await this.wrap(syncable);
+        const reqBody: string = envelope.toJSON();
+        const putOptions = {
+            method: 'PUT',
+            headers: headers,
+            body: reqBody
+        };
+        return await fetch(`${this.metaUrl}/${identifier}`, putOptions).then(async response => {
+            if (response.ok) {
+                return Promise.resolve({
+                    status: response.status,
+                    statusText: response.statusText + ', Metadata updated successfully!'
+                });
+            } else {
+                return Promise.reject({
+                    status: response.status,
+                    statusText: response.statusText
+                });
+            }
+        });
+    }
+
+    static async get(identifier: string, metaUrl: string): Promise<any> {
+        const response = await fetch(`${metaUrl}/${identifier}`, options).then(response => {
+            if (response.ok) {
+                return (response.json());
+            } else {
+                return Promise.reject({
+                    status: response.status,
+                    statusText: response.statusText
+                });
+            }
+        }).catch(error => {
+            if (error.code === 'ECONNREFUSED') {
+                return `Request to ${metaUrl}/${identifier} failed. Connection error.`
+            }
+            return `${error.status}: ${error.statusText}`;
+        });
+        if (typeof response !== "object") {
+            return response;
+        }
+        return Envelope.fromJSON(JSON.stringify(response)).unwrap();
+    }
+
+    static async getIdentifier(name: string, type: string = 'custom'): Promise<string> {
+        let identifier: string;
+        type = type.toLowerCase();
+        if (type === 'user') {
+            identifier = await User.toKey(name);
+        } else if (type === 'phone') {
+            identifier = await Phone.toKey(name);
+        } else {
+            identifier = await Custom.toKey(name);
+        }
+        return identifier;
+    }
+
+    private wrap(syncable: Syncable): Promise<Envelope> {
+        return new Promise<Envelope>(async (resolve, reject) => {
+            syncable.setSigner(this.signer);
+            syncable.onwrap = async (env) => {
+                if (env === undefined) {
+                    reject();
+                    return;
+                }
+                resolve(env);
+            };
+            syncable.sign();
+        });
+    }
+}
+
+export {
+    Meta,
+}

apps/cic-meta/src/phone.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, mergeKey } from 'crdt-meta';
+import { Syncable, Addressable, mergeKey } from '@cicnet/crdt-meta';
 
 class Phone extends Syncable implements Addressable {
 

apps/cic-meta/src/user.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
+import { Syncable, Addressable, toAddressKey } from '@cicnet/crdt-meta';
 
 const keySalt = new TextEncoder().encode(':cic.person');
 class User extends Syncable implements Addressable {

apps/cic-meta/tests/server.ts

@@ -4,7 +4,8 @@ import pgp = require('openpgp');
 import sqlite = require('sqlite3');
 
 import * as handlers from '../scripts/server/handlers';
-import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
+import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer } from '@cicnet/crdt-meta';
+import { SqliteAdapter } from '../src/db';
 
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

apps/cic-meta/tsconfig.json

@@ -19,6 +19,7 @@
   "include": [
     "src/**/*",
     "scripts/server/*",
-    "index.ts"
+    "index.ts",
+    "bin"
   ]
 }

apps/cic-notify/.config/africastalking.ini

@@ -1,4 +1,4 @@
 [AFRICASTALKING]
-api_username = foo
-api_key = bar
-api_sender_id = baz
+api_username =
+api_key =
+api_sender_id =

apps/cic-notify/cic_notify/error.py

@@ -9,3 +9,7 @@ class AlreadyInitializedError(Exception):
 class PleaseCommitFirstError(Exception):
     """Raised when there exists uncommitted changes in the code while trying to build out the package."""
     pass
+
+
+class NotificationSendError(Exception):
+    """Raised when a notification failed to due to some error as per the service responsible for dispatching the notification."""

apps/cic-notify/cic_notify/ext/enums.py

@@ -0,0 +1,19 @@
+# standard imports
+from enum import IntEnum
+
+
+class AfricasTalkingStatusCodes(IntEnum):
+    PROCESSED = 100
+    SENT = 101
+    QUEUED = 102
+    RISK_HOLD = 401
+    INVALID_SENDER_ID = 402
+    INVALID_PHONE_NUMBER = 403
+    UNSUPPORTED_NUMBER_TYPE = 404
+    INSUFFICIENT_BALANCE = 405
+    USER_IN_BLACKLIST = 406
+    COULD_NOT_ROUTE = 407
+    INTERNAL_SERVER_ERROR = 500
+    GATEWAY_ERROR = 501
+    REJECTED_BY_GATEWAY = 502
+

apps/cic-notify/cic_notify/tasks/sms/africastalking.py

@@ -6,7 +6,8 @@ import celery
 import africastalking
 
 # local imports
-from cic_notify.error import NotInitializedError, AlreadyInitializedError
+from cic_notify.error import NotInitializedError, AlreadyInitializedError, NotificationSendError
+from cic_notify.ext.enums import AfricasTalkingStatusCodes
 
 logg = logging.getLogger()
 celery_app = celery.current_app
@@ -50,10 +51,27 @@ class AfricasTalkingNotifier:
         if self.sender_id:
             response = self.api_client.send(message=message, recipients=[recipient], sender_id=self.sender_id)
             logg.debug(f'Africastalking response sender-id {response}')
+
         else:
             response = self.api_client.send(message=message, recipients=[recipient])
             logg.debug(f'africastalking response no-sender-id {response}')
 
+        recipients = response.get('Recipients')
+
+        if len(recipients) != 1:
+            status = response.get('SMSMessageData').get('Message')
+            raise NotificationSendError(f'Unexpected number of recipients: {len(recipients)}. Status: {status}')
+
+        status_code = recipients[0].get('statusCode')
+        status = recipients[0].get('status')
+
+        if status_code not in [
+            AfricasTalkingStatusCodes.PROCESSED.value,
+            AfricasTalkingStatusCodes.SENT.value,
+            AfricasTalkingStatusCodes.QUEUED.value
+        ]:
+            raise NotificationSendError(f'Sending notification failed due to: {status}')
+
 
 @celery_app.task
 def send(message, recipient):

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.4')
+version = (0, 4, 0, 'alpha.5')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -102,7 +102,7 @@ class MetadataRequestsHandler(Metadata):
                 'digest': json.loads(data).get('digest'),
             }
         }
-        formatted_data = json.dumps(formatted_data).encode('utf-8')
+        formatted_data = json.dumps(formatted_data)
         result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
         logg.info(f'signed metadata submission status: {result.status_code}.')
         metadata_http_error_handler(result=result)
@@ -116,11 +116,13 @@ class MetadataRequestsHandler(Metadata):
         """This function is responsible for querying the metadata server for data corresponding to a unique pointer."""
         result = make_request(method='GET', url=self.url)
         metadata_http_error_handler(result=result)
-        response_data = result.content
-        data = json.loads(response_data.decode('utf-8'))
+        response_data = result.json()
+        data = json.loads(response_data)
+        if not isinstance(data, dict):
+            raise ValueError(f'Invalid data object: {data}.')
         if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=json.loads(data))
+            deserialized_person = person.deserialize(person_data=data)
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/operations.py

@@ -325,6 +325,14 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
+        # retrieve and cache user's metadata
+        blockchain_address = user.blockchain_address
+        s_query_person_metadata = celery.signature(
+            'cic_ussd.tasks.metadata.query_person_metadata',
+            [blockchain_address]
+        )
+        s_query_person_metadata.apply_async(queue='cic-ussd')
+
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -371,13 +371,6 @@ def process_start_menu(display_key: str, user: Account):
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
-    # retrieve and cache account's metadata
-    s_query_person_metadata = celery.signature(
-        'cic_ussd.tasks.metadata.query_person_metadata',
-        [blockchain_address]
-    )
-    s_query_person_metadata.apply_async(queue='cic-ussd')
-
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 

apps/cic-ussd/var/lib/locale/sms.en.yml

@@ -1,7 +1,7 @@
 en:
   account_successfully_created: |-
-    Hello, you have been registered on Sarafu Network! Your balance is %{balance} %{token_symbol}. To use dial *483*46#. For help 0757628885.
+    You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
   received_tokens: |-
     Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
   terms: |-
-    By using the service, you agree to the terms and conditions at https://www.grassrootseconomics.org/terms-and-conditions.
+    By using the service, you agree to the terms and conditions at http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/sms.sw.yml

@@ -1,7 +1,7 @@
 sw:
   account_successfully_created: |-
-    Habari, umesajiliwa kwa huduma ya sarafu! Salio lako ni %{token_symbol} %{balance}. Kutumia bonyeza *483*46#. Kwa Usaidizi 0757628885.
+    Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
   received_tokens: |-
     Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
   terms: |-
-    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo https://www.grassrootseconomics.org/terms-and-conditions.
+    Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -1,29 +1,30 @@
 en:
   kenya:
     initial_language_selection: |-
-      CON Welcome to Sarafu
+      CON Welcome to Sarafu Network
       1. English
       2. Kiswahili
       3. Help
     initial_pin_entry: |-
-      CON Please enter a PIN to manage your account.
+      CON Please enter a new four number PIN for your account. 
       0. Back
     initial_pin_confirmation: |-
-      CON Enter your PIN again
+      CON Enter your four number PIN again
       0. Back
     enter_given_name: |-
       CON Enter first name
       0. Back
     enter_family_name: |-
-      CON Enter last name
+      CON Enter family name
       0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
       2. Female
+      3. Other
       0. Back
     enter_location: |-
-      CON Enter location
+      CON Enter your location
       0. Back
     enter_products: |-
       CON Please enter a product or service you offer
@@ -83,34 +84,34 @@ en:
         Please enter your PIN to confirm.
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     display_metadata_pin_authorization:
       first: |-
-        CON Please enter your PIN.
+        CON Please enter your PIN
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances_pin_authorization:
       first: |-
-        CON Please enter your PIN to view balances.
+        CON Please enter your PIN to view balances
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_statement_pin_authorization:
       first: |-
-        CON Please enter your PIN to view statement.
+        CON Please enter your PIN to view statement
         0. Back
       retry: |-
-        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
     account_balances: |-
       CON Your balances are as follows:
         balance: %{operational_balance} %{token_symbol}
-        taxes: %{tax} %{token_symbol}
-        bonsuses: %{bonus} %{token_symbol}
+        fees: %{tax} %{token_symbol}
+        rewards: %{bonus} %{token_symbol}
       0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
@@ -140,9 +141,9 @@ en:
     exit_pin_blocked: |-
       END Your PIN has been blocked. For help, please call %{support_phone}.
     exit_invalid_pin: |-
-      END The PIN you have entered is Invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
     exit_invalid_new_pin: |-
-      END The PIN you have entered is Invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
+      END The PIN you have entered is invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
     exit_pin_mismatch: |-
       END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
     exit_invalid_recipient: |-
@@ -169,4 +170,4 @@ en:
       00. Back
       99. Exit
     account_creation_prompt: |-
-      Your account is being created. You will receive an SMS when your account is ready.
+      Your account is being created. You will receive an SMS when your account is ready.

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -1,12 +1,12 @@
 sw:
   kenya:
     initial_language_selection: |-
-      CON Welcome to Sarafu
+      CON Karibu Sarafu Network
       1. English
       2. Kiswahili
       3. Help
     initial_pin_entry: |-
-      CON Tafadhali weka PIN ili kudhibiti akaunti yako.
+      CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako
       0. Nyuma
     initial_pin_confirmation: |-
       CON Weka PIN yako tena
@@ -21,12 +21,13 @@ sw:
       CON Weka jinsia yako
       1. Mwanaume
       2. Mwanamke
+      3. Nyngine 
       0. Nyuma
     enter_location: |-
       CON Weka eneo lako
       0. Nyuma
     enter_products: |-
-      CON Tafadhali weka bidhaa ama huduma unauza
+      CON Weka bidhaa ama huduma unauza
       0. Nyuma
     start: |-
       CON Salio %{account_balance} %{account_token_name}
@@ -155,7 +156,7 @@ sw:
       99. Ondoka
     exit_insufficient_balance: |-
       CON Malipo ya %{amount} %{token_symbol} kwa %{recipient_information} halijakamilika kwa sababu salio lako haitoshi.
-      Akaunti yako ya Sarafu-Network ina salio ifuatayo: %{token_balance}
+      Akaunti yako ya Sarafu ina salio ifuatayo: %{token_balance}
       00. Nyuma
       99. Ondoka
     invalid_service_code: |-
@@ -169,4 +170,4 @@ sw:
       00. Nyuma
       99. Ondoka
     account_creation_prompt: |-
-      Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.
+      Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.

apps/contract-migration/docker/Dockerfile

@@ -87,7 +87,6 @@ COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
-COPY contract-migration/scripts scripts/ 
 
 # A shared output dir for environment configs
 RUN mkdir -p /tmp/cic/config

apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py

@@ -0,0 +1,76 @@
+# external imports
+from chainlib.jsonrpc import JSONRPCException
+from eth_erc20 import ERC20
+from eth_accounts_index import AccountsIndex
+from eth_token_index import TokenUniqueSymbolIndex
+
+class ERC20Token:
+
+    def __init__(self, chain_spec, address, conn):
+        self.__address = address
+
+        c = ERC20(chain_spec)
+        o = c.symbol(address)
+        r = conn.do(o)
+        self.__symbol = c.parse_symbol(r)
+
+        o = c.decimals(address)
+        r = conn.do(o)
+        self.__decimals = c.parse_decimals(r)
+
+
+    def symbol(self):
+        return self.__symbol
+
+
+    def decimals(self):
+        return self.__decimals
+
+
+class IndexCache:
+
+    def __init__(self, chain_spec, address):
+        self.address = address
+        self.chain_spec = chain_spec
+
+    
+    def parse(self, r):
+        return r
+
+
+    def get(self, conn):
+        entries = []
+        i = 0
+        while True:
+            o = self.o.entry(self.address, i)
+            try:
+                r = conn.do(o)
+                entries.append(self.parse(r, conn))
+            except JSONRPCException:
+                return entries
+            i += 1
+
+
+class AccountRegistryCache(IndexCache):
+    
+    def __init__(self, chain_spec, address):
+        super(AccountRegistryCache, self).__init__(chain_spec, address)
+        self.o = AccountsIndex(chain_spec)
+        self.get_accounts = self.get
+
+
+    def parse(self, r, conn):
+        return self.o.parse_account(r)
+
+    
+class TokenRegistryCache(IndexCache):
+
+    def __init__(self, chain_spec, address):
+        super(TokenRegistryCache, self).__init__(chain_spec, address)
+        self.o = TokenUniqueSymbolIndex(chain_spec)
+        self.get_tokens = self.get
+
+
+    def parse(self, r, conn):
+        token_address = self.o.parse_entry(r)
+        return ERC20Token(self.chain_spec, token_address, conn)

apps/contract-migration/scripts/cic_eth/traffic/traffic.py

@@ -1,101 +0,0 @@
-# standard imports
-import os
-import logging
-import re
-import sys
-import json
-
-# external imports
-import redis
-import celery
-from chainsyncer.backend import MemBackend
-from chainsyncer.driver import HeadSyncer
-from chainlib.eth.connection import HTTPConnection
-from chainlib.eth.gas import DefaultGasOracle
-from chainlib.eth.nonce import DefaultNonceOracle
-from chainlib.eth.block import block_latest
-from hexathon import strip_0x
-
-# local imports
-import common
-from cmd.traffic import (
-        TrafficItem,
-        TrafficRouter,
-        TrafficProvisioner,
-        TrafficSyncHandler,
-        )
-from cmd.traffic import add_args as add_traffic_args
-
-
-# common basics
-script_dir = os.path.realpath(os.path.dirname(__file__))
-logg = common.log.create()
-argparser = common.argparse.create(script_dir, common.argparse.full_template)
-argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
-args = common.argparse.parse(argparser, logg)
-config = common.config.create(args.c, args, args.env_prefix)
-
-# map custom args to local config entries
-batchsize = args.batch_size
-if batchsize < 1:
-    batchsize = 1
-logg.info('batch size {}'.format(batchsize))
-config.add(batchsize, '_BATCH_SIZE', True)
-
-config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
-config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
-
-config.add(args.y, '_KEYSTORE_FILE', True)
-
-config.add(args.q, '_CELERY_QUEUE', True)
-
-common.config.log(config)
-
-
-def main():
-    # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
-
-    # connect to celery
-    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-    # set up registry
-    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
-    registry = common.registry.init_legacy(config, w3)
-
-    # Connect to blockchain with chainlib
-    conn = HTTPConnection(config.get('ETH_PROVIDER'))
-    gas_oracle = DefaultGasOracle(conn)
-    nonce_oracle = DefaultNonceOracle(signer_address, conn)
-
-    # Set up magic traffic handler
-    traffic_router = TrafficRouter()
-    traffic_router.apply_import_dict(config.all(), config)
-    handler = TrafficSyncHandler(config, traffic_router)
-
-    # Set up syncer
-    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
-    o = block_latest()
-    r = conn.do(o)
-    block_offset = int(strip_0x(r), 16) + 1
-    syncer_backend.set(block_offset, 0)
-
-    # Set up provisioner for common task input data
-    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    TrafficProvisioner.default_aux = {
-            'chain_spec': config.get('CIC_CHAIN_SPEC'),
-            'registry': registry,
-            'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
-            'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
-            'redis_db': config.get('REDIS_DB'),
-            'api_queue': config.get('_CELERY_QUEUE'),
-            }
-
-    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
-    syncer.add_filter(handler)
-    syncer.loop(1, conn)
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/scripts/package.json

@@ -1,7 +0,0 @@
-{
-  "dependencies": {
-    "cic-client-meta": "^0.0.7-alpha.8",
-    "crdt-meta": "0.0.8",
-    "vcard-parser": "^1.0.0"
-  }
-}

apps/data-seeding/.gitlab-ci.yml

@@ -0,0 +1,21 @@
+.data_seeding_variables:
+    variables:
+        APP_NAME: data-seeding
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.data_seeding_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
+
+build-mr-data-seeding:
+    extends:
+        - .data_seeding_changes_target
+        - .py_build_merge_request
+        - .data_seeding_variables
+
+build-push-data-seeding:
+    extends:
+        - .py_build_push
+        - .data_seeding_variables
+

apps/data-seeding/README.md

@@ -89,7 +89,12 @@ After this step is run, you can find top-level ethereum addresses (like the cic
 
 
 #### Custodial provisions
-
+response_data = send_ussd_request(address, self.data_dir)
+        state = response_data[:3]
+        out = response_data[4:]
+        m = '{} {}'.format(state, out[:7])
+        if m != 'CON Welcome':
+            raise VerifierError(response_data, 'ussd')
 This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
 
 `RUN_MASK=2 docker-compose up contract-migration`
@@ -104,8 +109,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-ussd-tasker
-* cic-ussd-server
+* cic-user-tasker
+* cic-user-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,6 +174,26 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
+
+##### Importing pins and ussd data (optional)
+Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
+
+In one terminal run:
+
+`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
+
+This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the old system.
+This csv file will be stored in the pins export dir defined as the positional argument.
+
+Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
+
+- To import the pins:
+
+`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
+
+- To import ussd data:
+`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
+
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/data-seeding/cic_eth/import_users.py

@@ -171,6 +171,7 @@ if __name__ == '__main__':
                 logg.error('load error for {}: {}'.format(y, e))
                 continue
             f.close()
+            logg.debug('deserializing {} {}'.format(filepath, o))
             u = Person.deserialize(o)
 
             new_address = register_eth(i, u)