Merge remote-tracking branch 'origin/master' into lash/migration-verify-faucet

Bump cic-ussd to apply bumped cic-eth dep

See merge request grassrootseconomics/cic-internal-integration!75

apps/cic-cache/.config/database.ini

@@ -1,5 +1,5 @@
 [database]
-NAME=cic-eth
+NAME=cic_cache
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/cic_cache/db/migrations/default/alembic.ini

@@ -36,7 +36,7 @@ script_location = .
 # output_encoding = utf-8
 
 #sqlalchemy.url = driver://user:pass@localhost/dbname
-sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-cache
+sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic_cache
 
 
 [post_write_hooks]

apps/cic-cache/cic_cache/db/migrations/default/versions/63b629f14a85_.py

@@ -29,8 +29,8 @@ def upgrade():
             sa.Column('source_token', sa.String(42), nullable=False),
             sa.Column('destination_token', sa.String(42), nullable=False),
             sa.Column('success', sa.Boolean, nullable=False),
-            sa.Column('from_value', sa.BIGINT(), nullable=False),
-            sa.Column('to_value', sa.BIGINT(), nullable=False),
+            sa.Column('from_value', sa.NUMERIC(), nullable=False),
+            sa.Column('to_value', sa.NUMERIC(), nullable=False),
             sa.Column('date_block', sa.DateTime, nullable=False),
             )
     op.create_table(

apps/cic-cache/cic_cache/runnable/tracker.py

@@ -312,7 +312,7 @@ class Tracker:
         session.close()
 
 (provider, w3) = web3_constructor()
-trust = config.get('CIC_TRUST_ADDRESS', []).split(",")
+trust = config.get('CIC_TRUST_ADDRESS', "").split(",")
 chain_spec = args.i
 
 try:

apps/cic-cache/config/database.ini

@@ -1,5 +1,5 @@
 [database]
-NAME=cic-eth
+NAME=cic_cache
 USER=postgres
 PASSWORD=
 HOST=localhost

apps/cic-cache/db/initdb_files/create_all_db.sql

@@ -1,5 +0,0 @@
-CREATE DATABASE "cic-cache";
-CREATE DATABASE "cic-eth";
-CREATE DATABASE "cic-notify";
-CREATE DATABASE "cic-meta";
-CREATE DATABASE "cic-signer";

apps/cic-cache/docker/Dockerfile

@@ -17,8 +17,7 @@ RUN apt-get update && \
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-COPY $root_requirement_file .
-RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44
 
 COPY cic-cache/requirements.txt ./
 COPY cic-cache/setup.cfg \

apps/cic-cache/setup.cfg

@@ -25,6 +25,7 @@ licence_files =
 python_requires = >= 3.6
 packages = 
 	cic_cache
+	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
 	cic_cache.runnable
@@ -33,5 +34,6 @@ scripts =
 
 [options.entry_points]
 console_scripts =
-	cic-cache-tracker = cic_cache.runnable.tracker:main
-	cic-cache-server = cic_cache.runnable.server:main
+	cic-cache-trackerd = cic_cache.runnable.tracker:main
+	cic-cache-serverd = cic_cache.runnable.server:main
+	cic-cache-taskerd = cic_cache.runnable.tasker:main

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,18 +4,23 @@ import logging
 
 # third-party imports
 import celery
-from cic_registry import zero_address
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.chain import ChainSpec
 
 # local imports
 from cic_eth.db.enum import LockEnum
+from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        )
 from cic_eth.error import LockedError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-@celery_app.task()
-def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -27,13 +32,14 @@ def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
     logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
 
 
-@celery_app.task()
-def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -45,13 +51,14 @@ def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, flags, address=address)
     logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
 
 
-@celery_app.task()
-def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -61,13 +68,14 @@ def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task()
-def unlock_send(chained_input, chain_str, address=zero_address):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -77,13 +85,14 @@ def unlock_send(chained_input, chain_str, address=zero_address):
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task()
-def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -93,13 +102,14 @@ def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task()
-def unlock_queue(chained_input, chain_str, address=zero_address):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -109,17 +119,23 @@ def unlock_queue(chained_input, chain_str, address=zero_address):
     :returns: New lock state for address
     :rtype: number
     """
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task()
-def check_lock(chained_input, chain_str, lock_flags, address=None):
-    r = Lock.check(chain_str, lock_flags, address=zero_address)
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
+    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    session = SessionBase.create_session()
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
     if address != None:
-        r |= Lock.check(chain_str, lock_flags, address=address)
+        r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:
         logg.debug('lock check {} has match {} for {}'.format(lock_flags, r, address))
+        session.close()
         raise LockedError(r)
+    session.flush()
+    session.close()
     return chained_input

apps/cic-eth/cic_eth/admin/debug.py

@@ -1,12 +1,25 @@
+# standard imports
 import datetime
 
+# external imports
 import celery
 
+# local imports
+from cic_eth.db.models.debug import Debug
+from cic_eth.db.models.base import SessionBase
+from cic_eth.task import CriticalSQLAlchemyTask
+
 celery_app = celery.current_app
 
 
-@celery_app.task()
-def out_tmp(tag, txt):
-    f = open('/tmp/err.{}.txt'.format(tag), "w")
-    f.write(txt)
-    f.close()
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def alert(chained_input, tag, txt):
+    session = SessionBase.create_session()
+
+    o = Debug(tag, txt)
+    session.add(o)
+    session.commit()
+
+    session.close()
+    
+    return chained_input

apps/cic-eth/cic_eth/admin/nonce.py

@@ -1,25 +1,30 @@
 # standard imports
 import logging
 
-# third-party imports
+# external imports
 import celery
-from cic_registry.chain import ChainSpec
+from chainlib.chain import ChainSpec
+from chainlib.eth.tx import unpack
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.otx import Otx
 from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import Nonce
-from cic_eth.admin.ctrl import lock_send
-from cic_eth.admin.ctrl import unlock_send
-from cic_eth.admin.ctrl import lock_queue
-from cic_eth.admin.ctrl import unlock_queue
-from cic_eth.queue.tx import get_tx
-from cic_eth.queue.tx import set_cancel
+from cic_eth.admin.ctrl import (
+        lock_send,
+        unlock_send,
+        lock_queue,
+        unlock_queue,
+        )
+from cic_eth.queue.tx import (
+        get_tx,
+        set_cancel,
+        )
 from cic_eth.queue.tx import create as queue_create
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import sign_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
+from cic_eth.eth.gas import (
+        create_check_gas_task,
+    )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -46,7 +51,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     chain_spec = ChainSpec.from_chain_str(chain_str)
     tx_brief = get_tx(tx_hash_orig_hex)
     tx_raw = bytes.fromhex(tx_brief['signed_tx'][2:])
-    tx = unpack_signed_raw_tx(tx_raw, chain_spec.chain_id())
+    tx = unpack(tx_raw, chain_spec.chain_id())
     nonce = tx_brief['nonce']
     address = tx['from']
 
@@ -67,7 +72,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     txs = []
     for otx in otxs:
         tx_raw = bytes.fromhex(otx.signed_tx[2:])
-        tx_new = unpack_signed_raw_tx(tx_raw, chain_spec.chain_id())
+        tx_new = unpack(tx_raw, chain_spec.chain_id())
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']

apps/cic-eth/cic_eth/api/api_admin.py

@@ -1,14 +1,27 @@
 # standard imports
 import logging
+import sys
 
-# third-party imports
+# external imports
 import celery
-import web3
-from cic_registry import zero_address
-from cic_registry import zero_content
-from cic_registry import CICRegistry
-from crypto_dev_signer.eth.web3ext import Web3 as Web3Ext
-from cic_registry.error import UnknownContractError
+from chainlib.eth.constant import (
+        ZERO_ADDRESS,
+        )
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.error import UnknownContractError
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.contract import code
+from chainlib.eth.tx import (
+        transaction,
+        receipt,
+        unpack,
+        )
+from chainlib.hash import keccak256_hex_to_hex
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainlib.eth.gas import balance
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -18,19 +31,22 @@ from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.enum import (
         StatusEnum,
+        StatusBits,
         is_alive,
+        is_error_status,
+        status_str,
     )
 from cic_eth.error import InitializationError
 from cic_eth.db.error import TxStateChangeError
-from cic_eth.eth.rpc import RpcClient
 from cic_eth.queue.tx import get_tx
-from cic_eth.eth.util import unpack_signed_raw_tx
 
 app = celery.current_app
 
 #logg = logging.getLogger(__file__)
 logg = logging.getLogger()
 
+local_fail = StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.UNKNOWN_ERROR
+
 
 class AdminApi:
     """Provides an interface to view and manipulate existing transaction tasks and system runtime settings.
@@ -40,19 +56,20 @@ class AdminApi:
     :param queue: Name of worker queue to submit tasks to
     :type queue: str
     """
-    def __init__(self, rpc_client, queue='cic-eth'):
-        self.rpc_client = rpc_client
-        self.w3 = rpc_client.w3
+    def __init__(self, rpc, queue='cic-eth', call_address=ZERO_ADDRESS):
+        self.rpc = rpc
         self.queue = queue
+        self.call_address = call_address
 
 
     def unlock(self, chain_spec, address, flags=None):
         s_unlock = celery.signature(
             'cic_eth.admin.ctrl.unlock',
             [
-                str(chain_spec),
-                flags,
+                None,
+                chain_spec.asdict(),
                 address,
+                flags,
                 ],
             queue=self.queue,
             )
@@ -63,9 +80,10 @@ class AdminApi:
         s_lock = celery.signature(
             'cic_eth.admin.ctrl.lock',
             [
-                str(chain_spec),
-                flags,
+                None,
+                chain_spec.asdict(),
                 address,
+                flags,
                 ],
             queue=self.queue,
             )
@@ -78,10 +96,10 @@ class AdminApi:
             [],
             queue=self.queue,
             )
-        return s_lock.apply_async().get()
+        return s_lock.apply_async()
 
 
-    def tag_account(self, tag, address_hex):
+    def tag_account(self, tag, address_hex, chain_spec):
         """Persistently associate an address with a plaintext tag.
 
         Some tags are known by the system and is used to resolve addresses to use for certain transactions. 
@@ -92,26 +110,28 @@ class AdminApi:
         :type address_hex: str, 0x-hex
         :raises ValueError: Invalid checksum address
         """
-        if not web3.Web3.isChecksumAddress(address_hex):
-            raise ValueError('invalid address')
-        session = SessionBase.create_session()
-        role = AccountRole.set(tag, address_hex) 
-        session.add(role)
-        session.commit()
-        session.close()
+        s_tag = celery.signature(
+            'cic_eth.eth.account.set_role',
+            [
+                tag,
+                address_hex,
+                chain_spec.asdict(),
+                ],
+            queue=self.queue,
+            )
+        return s_tag.apply_async()
 
 
-    def have_account(self, address_hex, chain_str):
+    def have_account(self, address_hex, chain_spec):
         s_have = celery.signature(
             'cic_eth.eth.account.have',
             [
                 address_hex,
-                chain_str,
+                chain_spec.asdict(),
                 ],
             queue=self.queue,
             )
-        t = s_have.apply_async()
-        return t.get()
+        return s_have.apply_async()
 
 
     def resend(self, tx_hash_hex, chain_str, in_place=True, unlock=False):
@@ -180,6 +200,7 @@ class AdminApi:
         blocking_tx = None
         blocking_nonce = None
         nonce_otx = 0
+        last_nonce = -1
         for k in txs.keys():
             s_get_tx = celery.signature(
                     'cic_eth.queue.tx.get_tx',
@@ -190,18 +211,25 @@ class AdminApi:
                     )
             tx = s_get_tx.apply_async().get()
             #tx = get_tx(k)
-            logg.debug('checking nonce {}'.format(tx['nonce']))
-            if tx['status'] in [StatusEnum.REJECTED, StatusEnum.FUBAR]:
-                blocking_tx = k
-                blocking_nonce = tx['nonce']
+            logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
+            if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
+                logg.info('permanently errored {} nonce {} status {}'.format(k, nonce_otx, status_str(tx['status'])))
+                blocking_tx = k
+                blocking_nonce = nonce_otx
+            elif nonce_otx - last_nonce > 1:
+                logg.error('nonce gap; {} followed {}'.format(nonce_otx, last_nonce))
+                blocking_tx = k
+                blocking_nonce = nonce_otx
+                break
+            last_nonce = nonce_otx
 
         #nonce_cache = Nonce.get(address)
-        nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
+        #nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
         
         return {
             'nonce': {
-                'network': nonce_w3,
+                #'network': nonce_cache,
                 'queue': nonce_otx,
                 #'cache': nonce_cache,
                 'blocking': blocking_nonce,
@@ -212,7 +240,7 @@ class AdminApi:
             }
 
 
-    def fix_nonce(self, address, nonce):
+    def fix_nonce(self, address, nonce, chain_spec):
         s = celery.signature(
                 'cic_eth.queue.tx.get_account_tx',
                 [
@@ -233,7 +261,7 @@ class AdminApi:
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    str(self.rpc_client.chain_spec),
+                    self.rpc.chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -241,30 +269,29 @@ class AdminApi:
         return s_nonce.apply_async()
 
 
-    # TODO: this is a stub, complete all checks
-    def ready(self):
-        """Checks whether all required initializations have been performed.
-
-        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
-        :raises KeyError: An address provided for initialization is not known by the keystore.
-        """
-        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
-        if addr == zero_address:
-            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
-
-        self.w3.eth.sign(addr, text='666f6f')
+#    # TODO: this is a stub, complete all checks
+#    def ready(self):
+#        """Checks whether all required initializations have been performed.
+#
+#        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
+#        :raises KeyError: An address provided for initialization is not known by the keystore.
+#        """
+#        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
+#        if addr == ZERO_ADDRESS:
+#            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
+#
+#        self.w3.eth.sign(addr, text='666f6f')
 
 
-    def account(self, chain_spec, address, cols=['tx_hash', 'sender', 'recipient', 'nonce', 'block', 'tx_index', 'status', 'network_status', 'date_created'], include_sender=True, include_recipient=True):
+    def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
         """Lists locally originated transactions for the given Ethereum address.
 
         Performs a synchronous call to the Celery task responsible for performing the query.
 
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
-        :param cols: Data columns to include
-        :type cols: list of str
         """
+        last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.tx.get_account_tx',
                 [
@@ -276,33 +303,45 @@ class AdminApi:
 
         tx_dict_list = []
         for tx_hash in txs.keys():
+            errors = []
             s = celery.signature(
                     'cic_eth.queue.tx.get_tx_cache',
                     [tx_hash],
                     queue=self.queue,
                     )
             tx_dict = s.apply_async().get()
-            if tx_dict['sender'] == address and not include_sender:
-                logg.debug('skipping sender tx {}'.format(tx_dict['tx_hash']))
-                continue
+            if tx_dict['sender'] == address:
+                if tx_dict['nonce'] - last_nonce > 1:
+                    logg.error('nonce gap; {} followed {} for tx {}'.format(tx_dict['nonce'], last_nonce, tx_dict['hash']))
+                    errors.append('nonce')
+                elif tx_dict['nonce'] == last_nonce:
+                    logg.warning('nonce {} duplicate in tx {}'.format(tx_dict['nonce'], tx_dict['hash']))
+                last_nonce = tx_dict['nonce']
+                if not include_sender:
+                    logg.debug('skipping sender tx {}'.format(tx_dict['tx_hash']))
+                    continue
             elif tx_dict['recipient'] == address and not include_recipient:
                 logg.debug('skipping recipient tx {}'.format(tx_dict['tx_hash']))
                 continue
 
-            logg.debug(tx_dict)
             o = {
                 'nonce': tx_dict['nonce'], 
                 'tx_hash': tx_dict['tx_hash'],
                 'status': tx_dict['status'],
                 'date_updated': tx_dict['date_updated'],
+                'errors': errors,
                     }
-            tx_dict_list.append(o)
+            if renderer != None:
+                r = renderer(o)
+                w.write(r + '\n')
+            else:
+                tx_dict_list.append(o)
 
         return tx_dict_list
 
 
     # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    def tx(self, chain_spec, tx_hash=None, tx_raw=None):
+    def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
         """Output local and network details about a given transaction with local origin.
 
         If the transaction hash is given, the raw trasnaction data will be retrieved from the local transaction queue backend. Otherwise the raw transaction data must be provided directly. Only one of transaction hash and transaction data can be passed.
@@ -317,11 +356,14 @@ class AdminApi:
         :return: Transaction details
         :rtype: dict
         """
+        problems = []
+
         if tx_hash != None and tx_raw != None:
             ValueError('Specify only one of hash or raw tx')
 
         if tx_raw != None:
-            tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
+            tx_hash = add_0x(keccak256_hex_to_hex(tx_raw))
+            #tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
 
         s = celery.signature(
             'cic_eth.queue.tx.get_tx_cache',
@@ -332,31 +374,35 @@ class AdminApi:
         tx = s.apply_async().get()
   
         source_token = None
-        if tx['source_token'] != zero_address:
+        if tx['source_token'] != ZERO_ADDRESS:
             try:
-                source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
+                source_token = registry.by_address(tx['source_token'])
+                #source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
             except UnknownContractError:
-                source_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
-                source_token = CICRegistry.add_token(chain_spec, source_token_contract)
+                #source_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #source_token = CICRegistry.add_token(chain_spec, source_token_contract)
                 logg.warning('unknown source token contract {}'.format(tx['source_token']))
 
         destination_token = None
-        if tx['source_token'] != zero_address:
+        if tx['source_token'] != ZERO_ADDRESS:
             try:
-                destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
+                #destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
+                destination_token = registry.by_address(tx['destination_token'])
             except UnknownContractError:
-                destination_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
-                destination_token = CICRegistry.add_token(chain_spec, destination_token_contract)
+                #destination_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
+                #destination_token = CICRegistry.add_token(chain_spec, destination_token_contract)
                 logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
 
-        c = RpcClient(chain_spec)
-        if len(c.w3.eth.getCode(tx['sender'])) > 0:
+        o = code(tx['sender'])
+        r = self.rpc.do(o)
+        if len(strip_0x(r, allow_empty=True)) > 0:
             try:
-                sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
-                tx['sender_description'] = 'Contract {}'.format(sender_contract.identifier())
+                #sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
+                sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
+                tx['sender_description'] = 'Contract at {}'.format(tx['sender']) #sender_contract)
             except UnknownContractError:
                 tx['sender_description'] = 'Unknown contract'
             except KeyError as e:
@@ -366,7 +412,7 @@ class AdminApi:
                     'cic_eth.eth.account.have',
                     [
                         tx['sender'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -379,7 +425,7 @@ class AdminApi:
                     'cic_eth.eth.account.role',
                     [
                         tx['sender'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -388,11 +434,13 @@ class AdminApi:
                 if role != None:
                     tx['sender_description'] = role
 
-
-        if len(c.w3.eth.getCode(tx['recipient'])) > 0:
+        o = code(tx['recipient'])
+        r = self.rpc.do(o)
+        if len(strip_0x(r, allow_empty=True)) > 0:
             try:
-                recipient_contract = CICRegistry.get_address(chain_spec, tx['recipient'])
-                tx['recipient_description'] = 'Contract {}'.format(recipient_contract.identifier())
+                #recipient_contract = CICRegistry.by_address(tx['recipient'])
+                recipient_contract = registry.by_address(tx['recipient'])
+                tx['recipient_description'] = 'Contract at {}'.format(tx['recipient']) #recipient_contract)
             except UnknownContractError as e:
                 tx['recipient_description'] = 'Unknown contract'
             except KeyError as e:
@@ -402,7 +450,7 @@ class AdminApi:
                     'cic_eth.eth.account.have',
                     [
                         tx['recipient'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -415,7 +463,7 @@ class AdminApi:
                     'cic_eth.eth.account.role',
                     [
                         tx['recipient'],
-                        str(chain_spec),
+                        chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -434,29 +482,43 @@ class AdminApi:
 
         tx['network_status'] = 'Not submitted'
 
+        r = None
         try:
-            c.w3.eth.getTransaction(tx_hash)
+            o = transaction(tx_hash)
+            r = self.rpc.do(o)
+        except Exception as e:
+            logg.warning('(too permissive exception handler, please fix!) {}'.format(e))
             tx['network_status'] = 'Mempool'
-        except web3.exceptions.TransactionNotFound:
-            pass
 
-        try:
-            r = c.w3.eth.getTransactionReceipt(tx_hash)
-            if r.status == 1:
-                tx['network_status'] = 'Confirmed'
-                tx['block'] = r.blockNumber
-                tx['tx_index'] = r.transactionIndex
-            else:
-                tx['network_status'] = 'Reverted'
-        except web3.exceptions.TransactionNotFound:
-            pass
+        if r != None:
+            try:
+                o = receipt(tx_hash)
+                r = self.rpc.do(o)
+                logg.debug('h {} o {}'.format(tx_hash, o))
+                if int(strip_0x(r['status'])) == 1:
+                    tx['network_status'] = 'Confirmed'
+                else:
+                    tx['network_status'] = 'Reverted'
+                tx['network_block_number'] = r.blockNumber
+                tx['network_tx_index'] = r.transactionIndex
+                if tx['block_number'] == None:
+                    problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber))
+            except Exception as e:
+                logg.warning('too permissive exception handler, please fix!')
+                pass
 
-        tx['sender_gas_balance'] = c.w3.eth.getBalance(tx['sender'])
-        tx['recipient_gas_balance'] = c.w3.eth.getBalance(tx['recipient'])
+        o = balance(tx['sender'])
+        r = self.rpc.do(o)
+        tx['sender_gas_balance'] = r
 
-        tx_unpacked = unpack_signed_raw_tx(bytes.fromhex(tx['signed_tx'][2:]), chain_spec.chain_id())
+        o = balance(tx['recipient'])
+        r = self.rpc.do(o)
+        tx['recipient_gas_balance'] = r
+
+        tx_unpacked = unpack(bytes.fromhex(tx['signed_tx'][2:]), chain_spec.chain_id())
         tx['gas_price'] = tx_unpacked['gasPrice']
         tx['gas_limit'] = tx_unpacked['gas']
+        tx['data'] = tx_unpacked['data']
 
         s = celery.signature(
             'cic_eth.queue.tx.get_state_log',
@@ -468,4 +530,14 @@ class AdminApi:
         t = s.apply_async()
         tx['status_log'] = t.get()
 
-        return tx
+        if len(problems) > 0:
+            sys.stderr.write('\n')
+            for p in problems:
+                sys.stderr.write('!!!{}\n'.format(p))
+
+        if renderer == None:
+            return tx
+
+        r = renderer(tx)
+        w.write(r + '\n')
+        return None

apps/cic-eth/cic_eth/api/api_task.py

@@ -8,12 +8,10 @@ import logging
 
 # external imports 
 import celery
-#from cic_registry.chain import ChainSpec
-from cic_registry import CICRegistry
+from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 
 # local imports
-from cic_eth.eth.factory import TxFactory
 from cic_eth.db.enum import LockEnum
 
 app = celery.current_app
@@ -82,18 +80,24 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     [from_token_symbol, to_token_symbol],
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     from_address,
                     ],
                 queue=self.queue,
                 )
+        s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [],
+                queue=self.queue,
+                )
         s_tokens = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
                 [
                     self.chain_str,
                     ],
@@ -106,11 +110,12 @@ class Api:
                     target_return,
                     minimum_return,
                     to_address,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
-        s_check.link(s_tokens)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
         if self.callback_param != None:
             s_convert.link(self.callback_success)
             s_tokens.link(s_convert).on_error(self.callback_error)
@@ -137,20 +142,26 @@ class Api:
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     [from_token_symbol, to_token_symbol],
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     from_address,
                     ],
                 queue=self.queue,
                 )
+        s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [],
+                queue=self.queue,
+                )
         s_tokens = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
                 [
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
@@ -161,11 +172,12 @@ class Api:
                     target_return,
                     minimum_return,
                     from_address,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
-        s_check.link(s_tokens)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
         if self.callback_param != None:
             s_convert.link(self.callback_success)
             s_tokens.link(s_convert).on_error(self.callback_error)
@@ -194,30 +206,38 @@ class Api:
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     [token_symbol],
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     from_address,
                     ],
                 queue=self.queue,
                 )
-        s_tokens = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
+        s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
                 [
-                    self.chain_str,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
         s_transfer = celery.signature(
-                'cic_eth.eth.token.transfer',
+                'cic_eth.eth.erc20.transfer',
                 [
                     from_address,
                     to_address,
                     value,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
-        s_check.link(s_tokens)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
         if self.callback_param != None:
             s_transfer.link(self.callback_success)
             s_tokens.link(s_transfer).on_error(self.callback_error)
@@ -228,82 +248,6 @@ class Api:
         return t
 
 
-    def transfer_request(self, from_address, to_address, spender_address, value, token_symbol):
-        """Executes a chain of celery tasks that issues a transfer request of ERC20 tokens from one address to another.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of recipient
-        :type to_address: str, 0x-hex
-        :param spender_address: Ethereum address that is executing transfer (typically an escrow contract)
-        :type spender_address: str, 0x-hex
-        :param value: Estimated return from conversion
-        :type  value: int
-        :param token_symbol: ERC20 token symbol of token to send
-        :type token_symbol: str
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [token_symbol],
-                    self.chain_str,
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens_transfer_approval = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
-                [
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens_approve = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
-                [
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        s_approve = celery.signature(
-                'cic_eth.eth.token.approve',
-                [
-                    from_address,
-                    spender_address,
-                    value,
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        s_transfer_approval = celery.signature(
-                'cic_eth.eth.request.transfer_approval_request',
-                [
-                    from_address,
-                    to_address,
-                    value,
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                )
-        # TODO: make approve and transfer_approval chainable so callback can be part of the full chain
-        if self.callback_param != None:
-            s_transfer_approval.link(self.callback_success)
-            s_tokens_approve.link(s_approve)
-            s_tokens_transfer_approval.link(s_transfer_approval).on_error(self.callback_error)
-        else:
-            s_tokens_approve.link(s_approve)
-            s_tokens_transfer_approval.link(s_transfer_approval)
-
-        g = celery.group(s_tokens_approve, s_tokens_transfer_approval) #s_tokens.apply_async(queue=self.queue)
-        s_check.link(g)
-        t = s_check.apply_async()
-        #t = s_tokens.apply_async(queue=self.queue)
-        return t
-
-
     def balance(self, address, token_symbol, include_pending=True):
         """Calls the provided callback with the current token balance of the given address.
 
@@ -320,18 +264,18 @@ class Api:
             logg.warning('balance pointlessly called with no callback url')
 
         s_tokens = celery.signature(
-                'cic_eth.eth.token.resolve_tokens_by_symbol',
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
                 [
                     [token_symbol],
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
         s_balance = celery.signature(
-                'cic_eth.eth.token.balance',
+                'cic_eth.eth.erc20.balance',
                 [
                     address,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
@@ -347,7 +291,7 @@ class Api:
                     'cic_eth.queue.balance.balance_incoming',
                     [
                         address,
-                        self.chain_str,
+                        self.chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -355,7 +299,7 @@ class Api:
                     'cic_eth.queue.balance.balance_outgoing',
                     [
                         address,
-                        self.chain_str,
+                        self.chain_spec.asdict(),
                         ],
                     queue=self.queue,
                     )
@@ -363,16 +307,22 @@ class Api:
             s_balance_incoming.link(s_balance_outgoing)
             last_in_chain = s_balance_outgoing
 
-        one = celery.chain(s_tokens, s_balance)
-        two = celery.chain(s_tokens, s_balance_incoming)
-        three = celery.chain(s_tokens, s_balance_outgoing)
+            one = celery.chain(s_tokens, s_balance)
+            two = celery.chain(s_tokens, s_balance_incoming)
+            three = celery.chain(s_tokens, s_balance_outgoing)
 
-        t = None
-        if self.callback_param != None:
-            s_result.link(self.callback_success).on_error(self.callback_error)
-            t = celery.chord([one, two, three])(s_result)
+            t = None
+            if self.callback_param != None:
+                s_result.link(self.callback_success).on_error(self.callback_error)
+                t = celery.chord([one, two, three])(s_result)
+            else:
+                t = celery.chord([one, two, three])(s_result)
         else:
-            t = celery.chord([one, two, three])(s_result)
+            # TODO: Chord is inefficient with only one chain, but assemble_balances must be able to handle different structures in order to avoid chord
+            one = celery.chain(s_tokens, s_balance)
+            if self.callback_param != None:
+                s_result.link(self.callback_success).on_error(self.callback_error)
+            t = celery.chord([one])(s_result)
     
         return t
 
@@ -391,7 +341,7 @@ class Api:
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     password,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.CREATE,
                     ],
                 queue=self.queue,
@@ -399,7 +349,7 @@ class Api:
         s_account = celery.signature(
                 'cic_eth.eth.account.create',
                 [
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
@@ -408,14 +358,22 @@ class Api:
             s_account.link(self.callback_success)
 
         if register:
-            s_register = celery.signature(
-                'cic_eth.eth.account.register', 
+            s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
                 [
-                    self.chain_str,
+                    'ACCOUNT_REGISTRY_WRITER',
                     ],
                 queue=self.queue,
                 )
-            s_account.link(s_register)
+            s_register = celery.signature(
+                'cic_eth.eth.account.register', 
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+            s_nonce.link(s_register)
+            s_account.link(s_nonce)
 
         t = s_check.apply_async(queue=self.queue)
         return t
@@ -433,19 +391,27 @@ class Api:
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     address,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     ],
                 queue=self.queue,
                 )
+        s_nonce = celery.signature(
+                'cic_eth.eth.tx.reserve_nonce',
+                [
+                    'GAS_GIFTER',
+                    ],
+                queue=self.queue,
+                )
         s_refill = celery.signature(
                 'cic_eth.eth.tx.refill_gas',
                 [
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
-        s_check.link(s_refill) 
+        s_nonce.link(s_refill) 
+        s_check.link(s_nonce)
         if self.callback_param != None:
             s_refill.link(self.callback_success)
 
@@ -483,14 +449,15 @@ class Api:
         s_brief = celery.signature(
             'cic_eth.ext.tx.tx_collate',
             [
-                self.chain_str,
+                self.chain_spec.asdict(),
                 offset,
                 limit
                 ],
             queue=self.queue,
             )
+        s_local.link(s_brief)
         if self.callback_param != None:
-            s_assemble.link(self.callback_success).on_error(self.callback_error)
+            s_brief.link(self.callback_success).on_error(self.callback_error)
 
         t = None
         if external_task != None:
@@ -508,18 +475,17 @@ class Api:
                 'cic_eth.ext.tx.list_tx_by_bloom',
                 [
                     address,
-                    self.chain_str,
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                     )
             c = celery.chain(s_external_get, s_external_process)
             t = celery.chord([s_local, c])(s_brief)
         else:
-            t = s_local.apply_sync()
+            t = s_local.apply_async(queue=self.queue)
 
         return t
 
-
     def ping(self, r):
         """A noop callback ping for testing purposes.
 

apps/cic-eth/cic_eth/callbacks/tcp.py

@@ -20,5 +20,10 @@ def tcp(self, result, destination, status_code):
     (host, port) = destination.split(':')
     logg.debug('tcp callback to {} {}'.format(host, port))
     s.connect((host, int(port)))
-    s.send(json.dumps(result).encode('utf-8'))
+    data = {
+            'root_id': self.request.root_id,
+            'status': status_code,
+            'result': result,
+            }
+    s.send(json.dumps(data).encode('utf-8'))
     s.close()

apps/cic-eth/cic_eth/db/enum.py

@@ -103,6 +103,9 @@ def status_str(v, bits_only=False):
         except ValueError:
             pass
 
+    if v == 0:
+        return 'NONE'
+
     for i in range(16):
         b = (1 << i)
         if (b & 0xffff) & v:

apps/cic-eth/cic_eth/db/migrations/default/versions/3b693afd526a_nonce_reservation.py

@@ -0,0 +1,31 @@
+"""Nonce reservation
+
+Revision ID: 3b693afd526a
+Revises: f738d9962fdf
+Create Date: 2021-03-05 07:09:50.898728
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3b693afd526a'
+down_revision = 'f738d9962fdf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'nonce_task_reservation',
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('address_hex', sa.String(42), nullable=False),
+        sa.Column('nonce', sa.Integer, nullable=False),
+        sa.Column('key', sa.String, nullable=False),
+        sa.Column('date_created', sa.DateTime, nullable=False),
+        )
+
+
+def downgrade():
+    op.drop_table('nonce_task_reservation')

apps/cic-eth/cic_eth/db/migrations/default/versions/ec40ac0974c1_add_chain_syncer.py

@@ -0,0 +1,28 @@
+"""Add chain syncer
+
+Revision ID: ec40ac0974c1
+Revises: 6ac7a1dadc46
+Create Date: 2021-02-23 06:10:19.246304
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from chainsyncer.db.migrations.sqlalchemy import (
+        chainsyncer_upgrade,
+        chainsyncer_downgrade,
+        )
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec40ac0974c1'
+down_revision = '6ac7a1dadc46'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    chainsyncer_upgrade(0, 0, 1)
+
+
+def downgrade():
+    chainsyncer_downgrade(0, 0, 1)

apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py

@@ -0,0 +1,32 @@
+"""debug output
+
+Revision ID: f738d9962fdf
+Revises: ec40ac0974c1
+Create Date: 2021-03-04 08:32:43.281214
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f738d9962fdf'
+down_revision = 'ec40ac0974c1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'debug',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('tag', sa.String, nullable=False),
+            sa.Column('description', sa.String, nullable=False),
+            sa.Column('date_created', sa.DateTime, nullable=False),
+            )
+    pass
+
+
+def downgrade():
+    op.drop_table('debug')
+    pass

apps/cic-eth/cic_eth/db/migrations/postgresql/versions/3b693afd526a_nonce_reservation.py

@@ -0,0 +1,31 @@
+"""Nonce reservation
+
+Revision ID: 3b693afd526a
+Revises: f738d9962fdf
+Create Date: 2021-03-05 07:09:50.898728
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3b693afd526a'
+down_revision = 'f738d9962fdf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'nonce_task_reservation',
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('address_hex', sa.String(42), nullable=False),
+        sa.Column('nonce', sa.Integer, nullable=False),
+        sa.Column('key', sa.String, nullable=False),
+        sa.Column('date_created', sa.DateTime, nullable=False),
+        )
+
+
+def downgrade():
+    op.drop_table('nonce_task_reservation')

apps/cic-eth/cic_eth/db/migrations/postgresql/versions/89e1e9baa53c_add_account_lock.py

@@ -24,6 +24,7 @@ def upgrade():
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False),
+            sa.Column("otx_id", sa.Integer, nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
 

apps/cic-eth/cic_eth/db/migrations/postgresql/versions/ec40ac0974c1_add_chain_syncer.py

@@ -0,0 +1,28 @@
+"""Add chain syncer
+
+Revision ID: ec40ac0974c1
+Revises: 6ac7a1dadc46
+Create Date: 2021-02-23 06:10:19.246304
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from chainsyncer.db.migrations.sqlalchemy import (
+        chainsyncer_upgrade,
+        chainsyncer_downgrade,
+        )
+
+
+# revision identifiers, used by Alembic.
+revision = 'ec40ac0974c1'
+down_revision = '6ac7a1dadc46'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    chainsyncer_upgrade(0, 0, 1)
+
+
+def downgrade():
+    chainsyncer_downgrade(0, 0, 1)

apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py

@@ -0,0 +1,32 @@
+"""debug output
+
+Revision ID: f738d9962fdf
+Revises: ec40ac0974c1
+Create Date: 2021-03-04 08:32:43.281214
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f738d9962fdf'
+down_revision = 'ec40ac0974c1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'debug',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column('tag', sa.String, nullable=False),
+            sa.Column('description', sa.String, nullable=False),
+            sa.Column('date_created', sa.DateTime, nullable=False),
+            )
+    pass
+
+
+def downgrade():
+    op.drop_table('debug')
+    pass

apps/cic-eth/cic_eth/db/models/base.py

@@ -54,7 +54,7 @@ class SessionBase(Model):
 
 
     @staticmethod
-    def connect(dsn, pool_size=8, debug=False):
+    def connect(dsn, pool_size=16, debug=False):
         """Create new database connection engine and connect to database backend.
 
         :param dsn: DSN string defining connection.
@@ -114,9 +114,8 @@ class SessionBase(Model):
 
     @staticmethod
     def release_session(session=None):
-        session.flush()
         session_key = str(id(session))
         if SessionBase.localsessions.get(session_key) != None:
-            logg.debug('destroying session {}'.format(session_key))
+            logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()

apps/cic-eth/cic_eth/db/models/debug.py

@@ -0,0 +1,23 @@
+# standard imports
+import datetime
+import logging
+
+# external imports
+from sqlalchemy import Column, String, DateTime
+
+# local imports
+from .base import SessionBase
+
+
+class Debug(SessionBase):
+
+    __tablename__ = 'debug'
+
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    tag = Column(String)
+    description = Column(String)
+
+
+    def __init__(self, tag, description):
+        self.tag = tag
+        self.description = description

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,7 +4,7 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
-from cic_registry import zero_address
+from chainlib.eth.constant import ZERO_ADDRESS
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -35,7 +35,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=zero_address, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -55,11 +55,9 @@ class Lock(SessionBase):
         :returns: New flag state of entry
         :rtype: number
         """
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        session = SessionBase.bind_session(session)
 
-        q = localsession.query(Lock)
+        q = session.query(Lock)
         #q = q.join(TxCache, isouter=True)
         q = q.filter(Lock.address==address)
         q = q.filter(Lock.blockchain==chain_str)
@@ -71,7 +69,8 @@ class Lock(SessionBase):
             lock.address = address
             lock.blockchain = chain_str
             if tx_hash != None:
-                q = localsession.query(Otx)
+                session.flush()
+                q = session.query(Otx)
                 q = q.filter(Otx.tx_hash==tx_hash)
                 otx = q.first()
                 if otx != None:
@@ -80,17 +79,16 @@ class Lock(SessionBase):
         lock.flags |= flags
         r = lock.flags
 
-        localsession.add(lock)
-        localsession.commit()
+        session.add(lock)
+        session.commit()
+
+        SessionBase.release_session(session)
 
-        if session == None:
-            localsession.close()
-        
         return r
 
 
     @staticmethod
-    def reset(chain_str, flags, address=zero_address, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -110,11 +108,9 @@ class Lock(SessionBase):
         :returns: New flag state of entry
         :rtype: number
         """
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        session = SessionBase.bind_session(session)
 
-        q = localsession.query(Lock)
+        q = session.query(Lock)
         #q = q.join(TxCache, isouter=True)
         q = q.filter(Lock.address==address)
         q = q.filter(Lock.blockchain==chain_str)
@@ -124,20 +120,19 @@ class Lock(SessionBase):
         if lock != None:
             lock.flags &= ~flags
             if lock.flags == 0:
-                localsession.delete(lock)
+                session.delete(lock)
             else:
-                localsession.add(lock)
+                session.add(lock)
                 r = lock.flags
-            localsession.commit()
+            session.commit()
 
-        if session == None:
-            localsession.close()
+        SessionBase.release_session(session)
 
         return r
 
 
     @staticmethod
-    def check(chain_str, flags, address=zero_address, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.
@@ -156,22 +151,20 @@ class Lock(SessionBase):
         :rtype: number
         """
 
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        session = SessionBase.bind_session(session)
 
-        q = localsession.query(Lock)
+        q = session.query(Lock)
         #q = q.join(TxCache, isouter=True)
         q = q.filter(Lock.address==address)
         q = q.filter(Lock.blockchain==chain_str)
         q = q.filter(Lock.flags.op('&')(flags)==flags)
         lock = q.first()
-        if session == None:
-            localsession.close()
        
         r = 0
         if lock != None:
             r = lock.flags & flags
+
+        SessionBase.release_session(session)
         return r
 
 

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -1,11 +1,16 @@
 # standard imports
 import logging
+import datetime
 
 # third-party imports
-from sqlalchemy import Column, String, Integer
+from sqlalchemy import Column, String, Integer, DateTime
 
 # local imports
 from .base import SessionBase
+from cic_eth.error import (
+        InitializationError,
+        IntegrityError,
+        )
 
 logg = logging.getLogger()
 
@@ -21,12 +26,9 @@ class Nonce(SessionBase):
 
     @staticmethod
     def get(address, session=None):
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        session = SessionBase.bind_session(session)
 
-
-        q = localsession.query(Nonce)
+        q = session.query(Nonce)
         q = q.filter(Nonce.address_hex==address)
         nonce = q.first()
 
@@ -34,8 +36,7 @@ class Nonce(SessionBase):
         if nonce != None:
             nonce_value = nonce.nonce;
 
-        if session == None:
-            localsession.close()
+        SessionBase.release_session(session)
 
         return nonce_value
 
@@ -55,7 +56,43 @@ class Nonce(SessionBase):
 
 
     @staticmethod
-    def next(address, initial_if_not_exists=0):
+    def __inc(conn, address):
+        #conn.execute("UPDATE nonce set nonce = nonce + 1 WHERE address_hex = '{}'".format(address))
+        q = conn.query(Nonce)
+        q = q.filter(Nonce.address_hex==address)
+        q = q.with_for_update()
+        o = q.first()
+        nonce =  o.nonce
+        o.nonce += 1
+        conn.add(o)
+        conn.flush()
+        return nonce
+
+
+    @staticmethod
+    def __init(conn, address, nonce):
+        conn.execute("INSERT INTO nonce (nonce, address_hex) VALUES ({}, '{}')".format(nonce, address))
+
+
+    @staticmethod
+    def init(address, nonce=0, session=None):
+        session = SessionBase.bind_session(session)
+
+        q = session.query(Nonce)
+        q = q.filter(Nonce.address_hex==address)
+        o = q.first()
+        if o != None:
+            session.flush()
+            raise InitializationError('nonce on {} already exists ({})'.format(address, o.nonce))
+        session.flush()
+        Nonce.__init(session, address, nonce)
+
+        SessionBase.release_session(session)
+
+
+    # TODO: Incrementing nonce MUST be done by separate tasks.
+    @staticmethod
+    def next(address, initial_if_not_exists=0, session=None):
         """Generate next nonce for the given address.
 
         If there is no previous nonce record for the address, the nonce may be initialized to a specified value, or 0 if no value has been given.
@@ -67,20 +104,106 @@ class Nonce(SessionBase):
         :returns: Nonce
         :rtype: number
         """
-        conn = Nonce.engine.connect()
-        if Nonce.transactional:
-            conn.execute('BEGIN')
-            conn.execute('LOCK TABLE nonce IN SHARE ROW EXCLUSIVE MODE')
-        nonce = Nonce.__get(conn, address)
+        session = SessionBase.bind_session(session)
+        
+        #session.begin_nested()
+        #conn = Nonce.engine.connect()
+        #if Nonce.transactional:
+        #    conn.execute('BEGIN')
+        #    conn.execute('LOCK TABLE nonce IN SHARE ROW EXCLUSIVE MODE')
+        #    logg.debug('locking nonce table for address {}'.format(address))
+        #nonce = Nonce.__get(conn, address)
+        nonce = Nonce.__get(session, address)
         logg.debug('get nonce {} for address {}'.format(nonce, address))
         if nonce == None:
             nonce = initial_if_not_exists
-            conn.execute("INSERT INTO nonce (nonce, address_hex) VALUES ({}, '{}')".format(nonce, address))
             logg.debug('setting default nonce to {} for address {}'.format(nonce, address))
-        Nonce.__set(conn, address, nonce+1)
-        if Nonce.transactional:
-            conn.execute('COMMIT')
-        conn.close()
+            #Nonce.__init(conn, address, nonce)
+            Nonce.__init(session, address, nonce)
+        #Nonce.__set(conn, address, nonce+1)
+        nonce = Nonce.__inc(session, address)
+        #if Nonce.transactional:
+            #conn.execute('COMMIT')
+        #    logg.debug('unlocking nonce table for address {}'.format(address))
+        #conn.close()
+        #session.commit()
+
+        SessionBase.release_session(session)
         return nonce
 
 
+class NonceReservation(SessionBase):
+
+    __tablename__ = 'nonce_task_reservation'
+
+    address_hex = Column(String(42))
+    nonce = Column(Integer)
+    key = Column(String)
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+
+
+    @staticmethod
+    def peek(address, key, session=None):
+        session = SessionBase.bind_session(session)
+
+        q = session.query(NonceReservation)
+        q = q.filter(NonceReservation.key==key)
+        q = q.filter(NonceReservation.address_hex==address)
+        o = q.first()
+
+        r = None
+        if o != None:
+            r = (o.key, o.nonce)
+
+        session.flush()
+
+        SessionBase.release_session(session)
+
+        return r
+
+
+    @staticmethod
+    def release(address, key, session=None):
+
+        session = SessionBase.bind_session(session)
+
+        o = NonceReservation.peek(address, key, session=session)
+
+        if o == None:
+            SessionBase.release_session(session)
+            raise IntegrityError('"release" called on key {} address {} which does not exists'.format(key, address))
+
+        q = session.query(NonceReservation)
+        q = q.filter(NonceReservation.key==key)
+        q = q.filter(NonceReservation.address_hex==address)
+        o = q.first()
+        r = (o.key, o.nonce)
+
+        session.delete(o)
+        session.flush()
+
+        SessionBase.release_session(session)
+
+        return r
+
+
+    @staticmethod
+    def next(address, key, session=None):
+        session = SessionBase.bind_session(session)
+
+        o = NonceReservation.peek(address, key, session)
+        if o != None:
+            raise IntegrityError('"next" called on nonce for key {} address {} during active key {}'.format(key, address, o[0]))
+
+        nonce = Nonce.next(address, session=session)
+
+        o = NonceReservation()
+        o.nonce = nonce
+        o.key = key
+        o.address_hex = address
+        session.add(o)
+        r = (key, nonce)
+       
+        SessionBase.release_session(session)
+
+        return r

apps/cic-eth/cic_eth/db/models/otx.py

@@ -2,7 +2,7 @@
 import datetime
 import logging
 
-# third-party imports
+# external imports
 from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey
 from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
 
@@ -15,7 +15,6 @@ from cic_eth.db.enum import (
         is_error_status,
         )
 from cic_eth.db.error import TxStateChangeError
-#from cic_eth.eth.util import address_hex_from_signed_tx
 
 logg = logging.getLogger()
 
@@ -79,6 +78,13 @@ class Otx(SessionBase):
         return r
 
 
+    def __status_not_set(self, status):
+        r = not(self.status & status)
+        if r:
+            logg.warning('status bit {} not set on {}'.format(status.name, self.tx_hash))
+        return r
+
+
     def set_block(self, block, session=None):
         """Set block number transaction was mined in.
 
@@ -88,19 +94,16 @@ class Otx(SessionBase):
         :type block: number
         :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
         """
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        session = SessionBase.bind_session(session)
 
         if self.block != None:
+            SessionBase.release_session(session)
             raise TxStateChangeError('Attempted set block {} when block was already {}'.format(block, self.block))
         self.block = block
-        localsession.add(self)
-        localsession.flush()
+        session.add(self)
+        session.flush()
 
-        if session==None:
-            localsession.commit()
-            localsession.close()
+        SessionBase.release_session(session)
 
 
     def waitforgas(self, session=None):
@@ -116,8 +119,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('GAS_ISSUES cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.GAS_ISSUES, session)
@@ -140,8 +145,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('FUBAR cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if is_error_status(self.status):
+            SessionBase.release_session(session)
             raise TxStateChangeError('FUBAR cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.UNKNOWN_ERROR | StatusBits.FINAL, session)
@@ -163,10 +170,13 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('REJECTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('REJECTED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
         if is_error_status(self.status):
+            SessionBase.release_session(session)
             raise TxStateChangeError('REJECTED cannot be set on an entry with an error state already set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.NODE_ERROR | StatusBits.FINAL, session)
@@ -186,10 +196,13 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already IN_NETWORK ({})'.format(status_str(self.status)))
         if self.status & StatusBits.OBSOLETE:
+            SessionBase.release_session(session)
             raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry already OBSOLETE ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.OBSOLETE, session)
@@ -209,6 +222,7 @@ class Otx(SessionBase):
 
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('OVERRIDDEN/OBSOLETED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.MANUAL, session)
@@ -231,8 +245,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('RETRY cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if not is_error_status(self.status) and not StatusBits.IN_NETWORK & self.status > 0:
+            SessionBase.release_session(session)
             raise TxStateChangeError('RETRY cannot be set on an entry that has no error ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.QUEUED, session)
@@ -257,8 +273,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('READYSEND cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if is_error_status(self.status):
+            SessionBase.release_session(session)
             raise TxStateChangeError('READYSEND cannot be set on an errored state ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.QUEUED, session)
@@ -283,6 +301,7 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('SENT cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.IN_NETWORK, session)
@@ -307,8 +326,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('SENDFAIL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('SENDFAIL cannot be set on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
 
         self.__set_status(StatusBits.LOCAL_ERROR | StatusBits.DEFERRED, session)
@@ -320,6 +341,34 @@ class Otx(SessionBase):
         SessionBase.release_session(session)
 
 
+    def dequeue(self, session=None):
+        """Marks that a process to execute send attempt is underway
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.__status_not_set(StatusBits.QUEUED):
+            return
+
+        session = SessionBase.bind_session(session)
+
+        if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
+            raise TxStateChangeError('QUEUED cannot be unset on an entry with FINAL state set ({})'.format(status_str(self.status)))
+        if self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
+            raise TxStateChangeError('QUEUED cannot be unset on an entry with IN_NETWORK state set ({})'.format(status_str(self.status)))
+
+        self.__reset_status(StatusBits.QUEUED, session)
+
+        if self.tracing:
+            self.__state_log(session=session)
+
+        SessionBase.release_session(session)
+
+
+
     def minefail(self, block, session=None):
         """Marks that transaction was mined but code execution did not succeed.
 
@@ -335,8 +384,10 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('REVERTED cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if not self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('REVERTED cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
 
         if block != None:
@@ -364,27 +415,17 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('CANCEL cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
 
         if confirmed:
-            if not self.status & StatusBits.OBSOLETE:
+            if self.status > 0 and not self.status & StatusBits.OBSOLETE:
+                SessionBase.release_session(session)
                 raise TxStateChangeError('CANCEL can only be set on an entry marked OBSOLETE ({})'.format(status_str(self.status)))
             self.__set_status(StatusEnum.CANCELLED, session)
         else:
             self.__set_status(StatusEnum.OBSOLETED, session)
 
-
-#        if confirmed:
-#            if self.status != StatusEnum.OBSOLETED:
-#                logg.warning('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
-#                #raise TxStateChangeError('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
-#            self.__set_status(StatusEnum.CANCELLED, session)
-#        elif self.status != StatusEnum.OBSOLETED:
-#            if self.status > StatusEnum.SENT:
-#                logg.warning('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
-#                #raise TxStateChangeError('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
-        #    self.__set_status(StatusEnum.OBSOLETED, session)
-
         if self.tracing:
             self.__state_log(session=session)
 
@@ -404,10 +445,13 @@ class Otx(SessionBase):
         session = SessionBase.bind_session(session)
 
         if self.status & StatusBits.FINAL:
+            SessionBase.release_session(session)
             raise TxStateChangeError('SUCCESS cannot be set on an entry with FINAL state set ({})'.format(status_str(self.status)))
         if not self.status & StatusBits.IN_NETWORK:
+            SessionBase.release_session(session)
             raise TxStateChangeError('SUCCESS cannot be set on an entry without IN_NETWORK state set ({})'.format(status_str(self.status)))
         if is_error_status(self.status):
+            SessionBase.release_session(session)
             raise TxStateChangeError('SUCCESS cannot be set on an entry with error state set ({})'.format(status_str(self.status)))
 
         if block != None:
@@ -488,22 +532,23 @@ class Otx(SessionBase):
         session.add(l)
 
 
+    # TODO: it is not safe to return otx here unless session has been passed in
     @staticmethod
     def add(nonce, address, tx_hash, signed_tx, session=None):
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
+        external_session = session != None
+
+        session = SessionBase.bind_session(session)
 
         otx = Otx(nonce, address, tx_hash, signed_tx)
-        localsession.add(otx)
-        localsession.flush()
+        session.add(otx)
+        session.flush()
         if otx.tracing:
-            otx.__state_log(session=localsession)
-        localsession.flush()
+            otx.__state_log(session=session)
+        session.flush()
 
-        if session==None:
-            localsession.commit()
-            localsession.close()
+        SessionBase.release_session(session)
+        
+        if not external_session:
             return None
 
         return otx

apps/cic-eth/cic_eth/db/models/role.py

@@ -1,9 +1,9 @@
 # standard imports
 import logging
 
-# third-party imports
+# external imports
 from sqlalchemy import Column, String, Text
-from cic_registry import zero_address
+from chainlib.eth.constant import ZERO_ADDRESS
 
 # local imports
 from .base import SessionBase
@@ -40,12 +40,14 @@ class AccountRole(SessionBase):
 
         session = SessionBase.bind_session(session)
 
-        role = AccountRole.get_role(tag, session)
+        role = AccountRole.__get_role(tag, session)
     
-        r = zero_address
+        r = ZERO_ADDRESS
         if role != None:
             r = role.address_hex
 
+        session.flush()
+
         SessionBase.release_session(session)
 
         return r
@@ -63,6 +65,8 @@ class AccountRole(SessionBase):
         session = SessionBase.bind_session(session)
         
         role = AccountRole.__get_role(tag, session)
+
+        session.flush()
         
         SessionBase.release_session(session)
 
@@ -74,7 +78,6 @@ class AccountRole(SessionBase):
         q = session.query(AccountRole)
         q = q.filter(AccountRole.tag==tag)
         r = q.first()
-        session.flush()
         return r
 
 
@@ -93,10 +96,12 @@ class AccountRole(SessionBase):
         """
         session = SessionBase.bind_session(session)
         
-        role = AccountRole.get_role(tag, session)
+        role = AccountRole.__get_role(tag, session)
         if role == None:
             role = AccountRole(tag)
         role.address_hex = address_hex
+
+        session.flush()
         
         SessionBase.release_session(session)
 
@@ -128,4 +133,4 @@ class AccountRole(SessionBase):
 
     def __init__(self, tag):
         self.tag = tag
-        self.address_hex = zero_address
+        self.address_hex = ZERO_ADDRESS

apps/cic-eth/cic_eth/db/models/tx.py

@@ -85,26 +85,27 @@ class TxCache(SessionBase):
         :param tx_hash_new: tx hash to associate the copied entry with
         :type tx_hash_new: str, 0x-hex
         """
-        localsession = SessionBase.bind_session(session)
+        session = SessionBase.bind_session(session)
         
-        q = localsession.query(TxCache)
+        q = session.query(TxCache)
         q = q.join(Otx)
         q = q.filter(Otx.tx_hash==tx_hash_original)
         txc = q.first()
 
         if txc == None:
-            SessionBase.release_session(localsession)
+            SessionBase.release_session(session)
             raise NotLocalTxError('original {}'.format(tx_hash_original))
         if txc.block_number != None:
-            SessionBase.release_session(localsession)
+            SessionBase.release_session(session)
             raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
 
-        q = localsession.query(Otx)
+        session.flush()
+        q = session.query(Otx)
         q = q.filter(Otx.tx_hash==tx_hash_new)
         otx = q.first()
 
         if otx == None:
-            SessionBase.release_session(localsession)
+            SessionBase.release_session(session)
             raise NotLocalTxError('new {}'.format(tx_hash_new))
 
         txc_new = TxCache(
@@ -115,18 +116,21 @@ class TxCache(SessionBase):
                 txc.destination_token_address,
                 int(txc.from_value),
                 int(txc.to_value),
+                session=session,
                 )
-        localsession.add(txc_new)
-        localsession.commit()
+        session.add(txc_new)
+        session.commit()
 
-        SessionBase.release_session(localsession)
+        SessionBase.release_session(session)
 
 
     def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None, session=None):
-        localsession = SessionBase.bind_session(session)
-        tx = localsession.query(Otx).filter(Otx.tx_hash==tx_hash).first()
+        session = SessionBase.bind_session(session)
+        q = session.query(Otx)
+        q = q.filter(Otx.tx_hash==tx_hash)
+        tx = q.first()
         if tx == None:
-            SessionBase.release_session(localsession)
+            SessionBase.release_session(session)
             raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
         self.otx_id = tx.id
 
@@ -139,9 +143,9 @@ class TxCache(SessionBase):
         self.block_number = block_number
         self.tx_index = tx_index
         # not automatically set in sqlite, it seems:
-        self.date_created = datetime.datetime.now()
+        self.date_created = datetime.datetime.utcnow()
         self.date_updated = self.date_created
         self.date_checked = self.date_created
 
-        SessionBase.release_session(localsession)
+        SessionBase.release_session(session)
 

apps/cic-eth/cic_eth/error.py

@@ -54,8 +54,29 @@ class RoleMissingError(Exception):
     pass
 
 
+class IntegrityError(Exception):
+    """Exception raised to signal irregularities with deduplication and ordering of tasks
+
+    """
+    pass
+
+
 class LockedError(Exception):
     """Exception raised when attempt is made to execute action that is deactivated by lock
 
     """
     pass
+
+
+class SignerError(Exception):
+    """Exception raised when signer is unavailable or generates an error
+
+    """
+    pass
+
+
+class EthError(Exception):
+    """Exception raised when unspecified error from evm node is encountered
+
+    """
+    pass

apps/cic-eth/cic_eth/eth/__init__.py

@@ -1,16 +0,0 @@
-"""Ethereum batch functions and utilities
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-
-"""
-# standard imports
-import os
-
-# local imports
-from .rpc import RpcClient
-
-registry_extra_identifiers = {
-        'Faucet': '0x{:0<64s}'.format(b'Faucet'.hex()),
-        'TransferApproval': '0x{:0<64s}'.format(b'TransferApproval'.hex()),
-        }
-

apps/cic-eth/cic_eth/eth/account.py

@@ -1,137 +1,60 @@
 # standard imports
 import logging
 
-# third-party imports
-import web3
+# external imports
 import celery
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-from erc20_single_shot_faucet import Faucet
-from cic_registry import zero_address
+from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from chainlib.eth.constant import ZERO_ADDRESS
+from hexathon import (
+        strip_0x,
+        )
+from chainlib.connection import RPCConnection
+from chainlib.eth.sign import (
+        new_account,
+        sign_message,
+        )
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
+from chainlib.chain import ChainSpec
+from eth_accounts_index import AccountRegistry
+from sarafu_faucet import MinterFaucet as Faucet
 
 # local import
-from cic_eth.eth import RpcClient
-from cic_eth.eth import registry_extra_identifiers
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.eth.factory import TxFactory
+from cic_eth_registry import CICRegistry
+from cic_eth.eth.gas import (
+        create_check_gas_task,
+        )
+#from cic_eth.eth.factory import TxFactory
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.tx import TxCache
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.error import RoleMissingError
+from cic_eth.error import (
+        RoleMissingError,
+        SignerError,
+        )
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        CriticalSQLAlchemyAndSignerTask,
+        BaseTask,
+        )
+from cic_eth.eth.nonce import (
+        CustodialTaskNonceOracle,
+        )
+from cic_eth.queue.tx import (
+        register_tx,
+        )
 
-#logg = logging.getLogger(__name__)
 logg = logging.getLogger()
 celery_app = celery.current_app 
-
-
-class AccountTxFactory(TxFactory):
-    """Factory for creating account index contract transactions
-    """
-    def add(
-            self,
-            address,
-            chain_spec,
-            ):
-        """Register an Ethereum account address with the on-chain account registry
-
-        :param address: Ethereum account address to add
-        :type address: str, 0x-hex
-        :param chain_spec: Chain to build transaction for
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Unsigned "AccountRegistry.add" transaction in standard Ethereum format
-        :rtype: dict
-        """
-
-        c = CICRegistry.get_contract(chain_spec, 'AccountRegistry')
-        f = c.function('add')
-        tx_add_buildable = f(
-                address,
-                )
-        gas = c.gas('add')
-        tx_add = tx_add_buildable.buildTransaction({
-            'from': self.address,
-            'gas': gas,
-            'gasPrice': self.gas_price,
-            'chainId': chain_spec.chain_id(),
-            'nonce': self.next_nonce(),
-            'value': 0,
-            })
-        return tx_add
-
-
-    def gift(
-            self,
-            address,
-            chain_spec,
-        ):
-        """Trigger the on-chain faucet to disburse tokens to the provided Ethereum account
-
-        :param address: Ethereum account address to gift to
-        :type address: str, 0x-hex
-        :param chain_spec: Chain to build transaction for
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Unsigned "Faucet.giveTo" transaction in standard Ethereum format
-        :rtype: dict
-        """
-
-        c = CICRegistry.get_contract(chain_spec, 'Faucet')
-        f = c.function('giveTo')
-        tx_add_buildable = f(address)
-        gas = c.gas('add')
-        tx_add = tx_add_buildable.buildTransaction({
-            'from': self.address,
-            'gas': gas,
-            'gasPrice': self.gas_price,
-            'chainId': chain_spec.chain_id(),
-            'nonce': self.next_nonce(),
-            'value': 0,
-            })
-        return tx_add
-
-
-def unpack_register(data):
-    """Verifies that a transaction is an "AccountRegister.add" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != '0a3b0a4f':
-        raise ValueError('Invalid account index register data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        }
-
-
-def unpack_gift(data):
-    """Verifies that a transaction is a "Faucet.giveTo" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != '63e4bff4':
-        raise ValueError('Invalid account index register data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        }
      
 
-@celery_app.task()
-def create(password, chain_str):
+# TODO: Separate out nonce initialization task
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def create(self, password, chain_spec_dict):
     """Creates and stores a new ethereum account in the keystore.
 
     The password is passed on to the wallet backend, no encryption is performed in the task worker.
@@ -143,27 +66,27 @@ def create(password, chain_str):
     :returns: Ethereum address of newly created account
     :rtype: str, 0x-hex
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    a = c.w3.eth.personal.new_account(password)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    a = None
+    conn = RPCConnection.connect(chain_spec, 'signer')
+    o = new_account()
+    a = conn.do(o)
+    conn.disconnect()
+
+    if a == None:
+        raise SignerError('create account')
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
-    n = c.w3.eth.getTransactionCount(a, 'pending')
-    session = SessionBase.create_session()
-    o = session.query(Nonce).filter(Nonce.address_hex==a).first()
-    if o == None:
-        o = Nonce()
-        o.address_hex = a
-        o.nonce = n
-        session.add(o)
-        session.commit()
+    session = self.create_session()
+    Nonce.init(a, session=session)
+    session.commit()
     session.close()
     return a
 
 
-@celery_app.task(bind=True, throws=(RoleMissingError,))
-def register(self, account_address, chain_str, writer_address=None):
+@celery_app.task(bind=True, throws=(RoleMissingError,), base=CriticalSQLAlchemyAndSignerTask)
+def register(self, account_address, chain_spec_dict, writer_address=None):
     """Creates a transaction to add the given address to the accounts index.
 
     :param account_address: Ethereum address to add
@@ -176,34 +99,52 @@ def register(self, account_address, chain_str, writer_address=None):
     :returns: The account_address input param
     :rtype: str, 0x-hex
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
-    session = SessionBase.create_session()
+    session = self.create_session()
     if writer_address == None:
-        writer_address = AccountRole.get_address('ACCOUNTS_INDEX_WRITER', session)
-    session.close()
-
-    if writer_address == zero_address:
-        raise RoleMissingError(account_address)
+        writer_address = AccountRole.get_address('ACCOUNT_REGISTRY_WRITER', session=session)
 
+    if writer_address == ZERO_ADDRESS:
+        session.close()
+        raise RoleMissingError('writer address for regsistering {}'.format(account_address))
 
     logg.debug('adding account address {} to index; writer {}'.format(account_address, writer_address))
-    queue = self.request.delivery_info['routing_key']
+    queue = self.request.delivery_info.get('routing_key')
 
-    c = RpcClient(chain_spec, holder_address=writer_address)
-    txf = AccountTxFactory(writer_address, c)
+    # Retrieve account index address
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+    call_address = AccountRole.get_address('DEFAULT', session=session)
+    if writer_address == ZERO_ADDRESS:
+        session.close()
+        raise RoleMissingError('call address for resgistering {}'.format(account_address))
+    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
+   
+    # Generate and sign transaction
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+    nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
+    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    account_registry = AccountRegistry(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
+    (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
+    rpc_signer.disconnect()
 
-    tx_add = txf.add(account_address, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_add, chain_str, queue, 'cic_eth.eth.account.cache_account_data')
+    # add transaction to queue
+    cache_task = 'cic_eth.eth.account.cache_account_data'
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
 
-    gas_budget = tx_add['gas'] * tx_add['gasPrice']
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('register user tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+    rpc.disconnect()
 
-    logg.debug('register user tx {}'.format(tx_hash_hex))
-    s = create_check_gas_and_send_task(
+    s = create_check_gas_task(
             [tx_signed_raw_hex],
-            chain_str,
+            chain_spec,
             writer_address,
-            gas_budget,
+            gas=gas_budget,
             tx_hashes_hex=[tx_hash_hex],
             queue=queue,
             )
@@ -211,8 +152,8 @@ def register(self, account_address, chain_str, writer_address=None):
     return account_address
 
 
-@celery_app.task(bind=True)
-def gift(self, account_address, chain_str):
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def gift(self, account_address, chain_spec_dict):
     """Creates a transaction to invoke the faucet contract for the given address.
 
     :param account_address: Ethereum address to give to
@@ -222,34 +163,51 @@ def gift(self, account_address, chain_str):
     :returns: Raw signed transaction
     :rtype: list with transaction as only element
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     logg.debug('gift account address {} to index'.format(account_address))
-    queue = self.request.delivery_info['routing_key']
+    queue = self.request.delivery_info.get('routing_key')
 
-    c = RpcClient(chain_spec, holder_address=account_address)
-    txf = AccountTxFactory(account_address, c)
+    # Retrieve account index address
+    session = self.create_session()
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+    faucet_address = registry.by_name('Faucet', sender_address=self.call_address)
 
-    tx_add = txf.gift(account_address, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_add, chain_str, queue, 'cic_eth.eth.account.cache_gift_data')
+    # Generate and sign transaction
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+    nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
+    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
+    faucet = Faucet(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
+    (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
+    rpc_signer.disconnect()
 
-    gas_budget = tx_add['gas'] * tx_add['gasPrice']
+    # add transaction to queue
+    cache_task = 'cic_eth.eth.account.cache_gift_data'
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task, session=session)
+    session.commit()
+    session.close()
 
-    logg.debug('register user tx {}'.format(tx_hash_hex))
-    s = create_check_gas_and_send_task(
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('register user tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+    rpc.disconnect()
+
+    s = create_check_gas_task(
             [tx_signed_raw_hex],
-            chain_str,
+            chain_spec,
             account_address,
             gas_budget,
             [tx_hash_hex],
             queue=queue,
             )
     s.apply_async()
+
     return [tx_signed_raw_hex]
 
 
 @celery_app.task(bind=True)
-def have(self, account, chain_str):
+def have(self, account, chain_spec_dict):
     """Check whether the given account exists in keystore
 
     :param account: Account to check
@@ -259,17 +217,38 @@ def have(self, account, chain_str):
     :returns: Account, or None if not exists
     :rtype: Varies
     """
-    c = RpcClient(account)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    o = sign_message(account, '0x2a')
     try:
-        c.w3.eth.sign(account, text='2a')
-        return account
+        conn = RPCConnection.connect(chain_spec, 'signer')
     except Exception as e:
         logg.debug('cannot sign with {}: {}'.format(account, e))
         return None
 
+    try:
+        conn.do(o)
+        conn.disconnect()
+        return account
+    except Exception as e:
+        logg.debug('cannot sign with {}: {}'.format(account, e))
+        conn.disconnect()
+        return None
+   
 
-@celery_app.task(bind=True)
-def role(self, account, chain_str):
+@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+def set_role(self, tag, address, chain_spec_dict):
+    if not to_checksum_address(address):
+        raise ValueError('invalid checksum address {}'.format(address))
+    session = SessionBase.create_session()
+    role = AccountRole.set(tag, address, session=session) 
+    session.add(role)
+    session.commit()
+    session.close()
+    return tag
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def role(self, address, chain_spec_dict):
     """Return account role for address
 
     :param account: Account to check
@@ -279,14 +258,18 @@ def role(self, account, chain_str):
     :returns: Account, or None if not exists
     :rtype: Varies
     """
-    return AccountRole.role_for(account)
+    session = self.create_session()
+    role_tag =  AccountRole.role_for(address, session=session)
+    session.close()
+    return role_tag
 
 
-@celery_app.task()
+@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def cache_gift_data(
+    self,
     tx_hash_hex,
     tx_signed_raw_hex,
-    chain_str,
+    chain_spec_dict,
         ):
     """Generates and commits transaction cache metadata for a Faucet.giveTo transaction
 
@@ -299,21 +282,20 @@ def cache_gift_data(
     :returns: Transaction hash and id of cache element in storage backend, respectively
     :rtype: tuple
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    tx_data = unpack_gift(tx['data'])
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
+    tx_data = Faucet.parse_give_to_request(tx['data'])
 
-    session = SessionBase.create_session()
+    session = self.create_session()
 
     tx_cache = TxCache(
         tx_hash_hex,
         tx['from'],
         tx['to'],
-        zero_address,
-        zero_address,
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
         0,
         0,
         session=session,
@@ -326,11 +308,12 @@ def cache_gift_data(
     return (tx_hash_hex, cache_id)
 
 
-@celery_app.task()
+@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def cache_account_data(
+    self,
     tx_hash_hex,
     tx_signed_raw_hex,
-    chain_str,
+    chain_spec_dict,
         ):
     """Generates and commits transaction cache metadata for an AccountsIndex.add  transaction
 
@@ -343,21 +326,18 @@ def cache_account_data(
     :returns: Transaction hash and id of cache element in storage backend, respectively
     :rtype: tuple
     """
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    tx_data = unpack_register(tx['data'])
+    tx = unpack(tx_signed_raw_bytes, chain_id=chain_spec.chain_id())
+    tx_data = AccountRegistry.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(
         tx_hash_hex,
         tx['from'],
         tx['to'],
-        zero_address,
-        zero_address,
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
         0,
         0,
         session=session,

apps/cic-eth/cic_eth/eth/erc20.py

@@ -0,0 +1,305 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.chain import ChainSpec
+from chainlib.connection import RPCConnection
+from chainlib.eth.erc20 import ERC20
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.db.models.tx import TxCache
+from cic_eth.db.models.base import SessionBase
+from cic_eth.db.models.role import AccountRole
+from cic_eth.error import TokenCountError, PermanentTxError, OutOfGasError, NotLocalTxError
+from cic_eth.queue.tx import register_tx
+from cic_eth.eth.gas import (
+        create_check_gas_task,
+        MaxGasOracle,
+        )
+#from cic_eth.eth.factory import TxFactory
+from cic_eth.ext.address import translate_address
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        CriticalWeb3Task,
+        CriticalSQLAlchemyAndSignerTask,
+    )
+from cic_eth.eth.nonce import CustodialTaskNonceOracle
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(base=CriticalWeb3Task)
+def balance(tokens, holder_address, chain_spec_dict):
+    """Return token balances for a list of tokens for given address
+
+    :param tokens: Token addresses
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param chain_spec_dict: Chain spec string representation
+    :type chain_spec_dict: str
+    :return: List of balances
+    :rtype: list of int
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    caller_address = ERC20Token.caller_address 
+
+    for t in tokens:
+        address = t['address']
+        token = ERC20Token(rpc, address)
+        c = ERC20()
+        o = c.balance_of(address, holder_address, sender_address=caller_address)
+        r = rpc.do(o)
+        t['balance_network'] = c.parse_balance(r)
+    rpc.disconnect()
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
+    """Transfer ERC20 tokens between addresses
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :raises TokenCountError: More than one token is passed in tokens list
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_spec.chain_id())
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_transfer_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def approve(self, tokens, holder_address, spender_address, value, chain_spec_dict):
+    """Approve ERC20 transfer on behalf of holder address
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :raises TokenCountError: More than one token is passed in tokens list
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_spec.chain_id())
+    (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_approve_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+@celery_app.task(bind=True, base=CriticalWeb3Task)
+def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
+    """Returns contract addresses of an array of ERC20 token symbols
+
+    :param token_symbols: Token symbols to resolve
+    :type token_symbols: list of str
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+
+    :return: Respective token contract addresses
+    :rtype: list of str, 0x-hex
+    """
+    tokens = []
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+    session = self.create_session()
+    sender_address = AccountRole.get_address('DEFAULT', session)
+    session.close()
+    for token_symbol in token_symbols:
+        token_address = registry.by_name(token_symbol, sender_address=sender_address)
+        logg.debug('token {}'.format(token_address))
+        tokens.append({
+            'address': token_address,
+            'converters': [],
+            })
+    rpc.disconnect()
+    return tokens
+
+
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_transfer_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_transfer
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
+
+    tx_data = ERC20.parse_transfer_request(tx['data'])
+    recipient_address = tx_data[0]
+    token_value = tx_data[1]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+
+
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_approve_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_approve
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
+
+    tx_data = ERC20.parse_approve_request(tx['data'])
+    recipient_address = tx_data[0]
+    token_value = tx_data[1]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+

apps/cic-eth/cic_eth/eth/factory.py

@@ -1,41 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from cic_registry import CICRegistry
-from cic_eth.eth.nonce import NonceOracle
-from cic_eth.eth import RpcClient
-
-logg = logging.getLogger(__name__)
-
-
-class TxFactory:
-    """Base class for transaction factory classes.
-
-    :param from_address: Signer address to create transaction on behalf of
-    :type from_address: str, 0x-hex
-    :param rpc_client: RPC connection object to use to acquire account nonce if no record in nonce cache
-    :type rpc_client: cic_eth.eth.rpc.RpcClient
-    """
-
-    gas_price = 100
-    """Gas price, updated between batches"""
-
-
-    def __init__(self, from_address, rpc_client):
-        self.address = from_address
-
-        self.default_nonce = rpc_client.w3.eth.getTransactionCount(from_address, 'pending')
-        self.nonce_oracle = NonceOracle(from_address, self.default_nonce)
-
-        TxFactory.gas_price = rpc_client.gas_price()
-        logg.debug('txfactory instance address {} gas price'.format(self.address, self.gas_price))
-        
-
-    def next_nonce(self):
-        """Returns the current cached nonce value, and increments it for next transaction.
-
-        :returns: Nonce
-        :rtype: number
-        """
-        return self.nonce_oracle.next()

apps/cic-eth/cic_eth/eth/gas.py

@@ -1,74 +1,138 @@
 # standard imports
 import logging
 
+# external imports
+import celery
+from chainlib.eth.gas import price
+from hexathon import strip_0x
+
 # local imports
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 
 logg = logging.getLogger()
 
+#
+#class GasOracle():
+#    """Provides gas pricing for transactions.
+#
+#    :param w3: Web3 object
+#    :type w3: web3.Web3
+#    """
+#
+#    __safe_threshold_amount_value = 2000000000 * 60000 * 3
+#    __refill_amount_value = __safe_threshold_amount_value * 5
+#    default_gas_limit = 21000
+#
+#    def __init__(self, conn):
+#        o = price()
+#        r = conn.do(o)
+#        b = bytes.from_hex(strip_0x(r))
+#        self.gas_price_current = int.from_bytes(b, 'big')
+#
+#        #self.w3 = w3
+#        #self.gas_price_current = w3.eth.gas_price()
+#
+#
+#    def safe_threshold_amount(self):
+#        """The gas balance threshold under which a new gas refill transaction should be initiated.
+#
+#        :returns: Gas token amount
+#        :rtype: number
+#        """
+#        g = GasOracle.__safe_threshold_amount_value
+#        logg.warning('gas safe threshold is currently hardcoded to {}'.format(g))
+#        return g
+#
+#
+#    def refill_amount(self):
+#        """The amount of gas tokens to send in a gas refill transaction.
+#
+#        :returns: Gas token amount
+#        :rtype: number
+#        """
+#        g = GasOracle.__refill_amount_value
+#        logg.warning('gas refill amount is currently hardcoded to {}'.format(g))
+#        return g
+#
+# 
+#    def gas_provider(self):
+#        """Gas provider address.
+#
+#        :returns: Etheerum account address
+#        :rtype: str, 0x-hex
+#        """
+#        session = SessionBase.create_session()
+#        a = AccountRole.get_address('GAS_GIFTER', session)
+#        logg.debug('gasgifter {}'.format(a))
+#        session.close()
+#        return a
+#
+#
+#    def gas_price(self, category='safe'):
+#        """Get projected gas price to use for a transaction at the current moment.
+#
+#        When the category parameter is implemented, it can be used to control the priority of a transaction in the network.
+#
+#        :param category: Bid level category to return price for. Currently has no effect.
+#        :type category: str 
+#        :returns: Gas price
+#        :rtype: number
+#        """
+#        #logg.warning('gas price hardcoded to category "safe"')
+#        #g = 100
+#        #return g
+#        return self.gas_price_current
 
-class GasOracle():
-    """Provides gas pricing for transactions.
 
-    :param w3: Web3 object
-    :type w3: web3.Web3
+class MaxGasOracle:
+
+    def gas(code=None):
+        return 8000000
+
+
+def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
+    """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
+
+    If tx_hashes_hex is not spefified, a preceding task chained to check_gas must supply the transaction hashes as its return value.
+
+    :param tx_signed_raws_hex: Raw signed transaction data
+    :type tx_signed_raws_hex: list of str, 0x-hex
+    :param chain_spec: Chain spec of address to add check gas for
+    :type chain_spec: chainlib.chain.ChainSpec
+    :param holder_address: Address sending the transactions
+    :type holder_address: str, 0x-hex
+    :param gas: Gas budget hint for transactions
+    :type gas: int
+    :param tx_hashes_hex: Transaction hashes
+    :type tx_hashes_hex: list of str, 0x-hex
+    :param queue: Task queue
+    :type queue: str
+    :returns: Signature of task chain
+    :rtype: celery.Signature
     """
-
-    __safe_threshold_amount_value = 2000000000 * 60000 * 3
-    __refill_amount_value = __safe_threshold_amount_value * 5
-    default_gas_limit = 21000
-
-    def __init__(self, w3):
-        self.w3 = w3
-        self.gas_price_current = w3.eth.gas_price()
-
-
-    def safe_threshold_amount(self):
-        """The gas balance threshold under which a new gas refill transaction should be initiated.
-
-        :returns: Gas token amount
-        :rtype: number
-        """
-        g = GasOracle.__safe_threshold_amount_value
-        logg.warning('gas safe threshold is currently hardcoded to {}'.format(g))
-        return g
-
-
-    def refill_amount(self):
-        """The amount of gas tokens to send in a gas refill transaction.
-
-        :returns: Gas token amount
-        :rtype: number
-        """
-        g = GasOracle.__refill_amount_value
-        logg.warning('gas refill amount is currently hardcoded to {}'.format(g))
-        return g
-
- 
-    def gas_provider(self):
-        """Gas provider address.
-
-        :returns: Etheerum account address
-        :rtype: str, 0x-hex
-        """
-        session = SessionBase.create_session()
-        a = AccountRole.get_address('GAS_GIFTER', session)
-        session.close()
-        return a
-
-
-    def gas_price(self, category='safe'):
-        """Get projected gas price to use for a transaction at the current moment.
-
-        When the category parameter is implemented, it can be used to control the priority of a transaction in the network.
-
-        :param category: Bid level category to return price for. Currently has no effect.
-        :type category: str 
-        :returns: Gas price
-        :rtype: number
-        """
-        #logg.warning('gas price hardcoded to category "safe"')
-        #g = 100
-        #return g
-        return self.gas_price_current
+    s_check_gas = None
+    if tx_hashes_hex != None:
+        s_check_gas = celery.signature(
+                'cic_eth.eth.tx.check_gas',
+                [
+                    tx_hashes_hex,
+                    chain_spec.asdict(),
+                    tx_signed_raws_hex,
+                    holder_address,
+                    gas,
+                    ],
+                queue=queue,
+                )
+    else:
+        s_check_gas = celery.signature(
+                'cic_eth.eth.tx.check_gas',
+                [
+                    chain_spec.asdict(),
+                    tx_signed_raws_hex,
+                    holder_address,
+                    gas,
+                    ],
+                queue=queue,
+                )
+    return s_check_gas

apps/cic-eth/cic_eth/eth/meta.py

@@ -0,0 +1,71 @@
+# extended imports
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.status import Status as TxStatus
+
+
+class ExtendedTx:
+
+    _default_decimals = 6
+
+    def __init__(self, rpc, tx_hash, chain_spec):
+        self.rpc = rpc
+        self.chain_spec = chain_spec
+        self.hash = tx_hash
+        self.sender = None
+        self.sender_label = None
+        self.recipient = None
+        self.recipient_label = None
+        self.source_token_value = 0
+        self.destination_token_value = 0
+        self.source_token = ZERO_ADDRESS
+        self.destination_token = ZERO_ADDRESS
+        self.source_token_symbol = ''
+        self.destination_token_symbol = ''
+        self.source_token_decimals = ExtendedTx._default_decimals
+        self.destination_token_decimals = ExtendedTx._default_decimals
+        self.status = TxStatus.PENDING.name
+        self.status_code = TxStatus.PENDING.value
+
+
+    def set_actors(self, sender, recipient, trusted_declarator_addresses=None):
+        self.sender = sender
+        self.recipient = recipient
+        if trusted_declarator_addresses != None:
+            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain_spec)
+            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain_spec)
+
+
+    def set_tokens(self, source, source_value, destination=None, destination_value=None):
+        if destination == None:
+            destination = source
+        if destination_value == None:
+            destination_value = source_value
+        st = ERC20Token(self.rpc, source)
+        dt = ERC20Token(self.rpc, destination)
+        self.source_token = source
+        self.source_token_symbol = st.symbol
+        self.source_token_name = st.name
+        self.source_token_decimals = st.decimals
+        self.source_token_value = source_value
+        self.destination_token = destination
+        self.destination_token_symbol = dt.symbol
+        self.destination_token_name = dt.name
+        self.destination_token_decimals = dt.decimals
+        self.destination_token_value = destination_value
+
+
+    def set_status(self, n):
+        if n:
+            self.status = TxStatus.ERROR.name
+        else:
+            self.status = TxStatus.SUCCESS.name
+        self.status_code = n
+
+
+    def to_dict(self):
+        o = {}
+        for attr in dir(self):
+            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'to_dict']:
+                continue
+            o[attr] = getattr(self, attr)
+        return o 

apps/cic-eth/cic_eth/eth/nonce.py

@@ -1,7 +1,10 @@
 # local imports
-from cic_eth.db.models.nonce import Nonce
+from cic_eth.db.models.nonce import (
+        Nonce,
+        NonceReservation,
+        )
 
-class NonceOracle():
+class CustodialTaskNonceOracle():
     """Ensures atomic nonce increments for all transactions across all tasks and threads.
 
     :param address: Address to generate nonces for
@@ -9,15 +12,21 @@ class NonceOracle():
     :param default_nonce: Initial nonce value to use if no nonce cache entry already exists
     :type default_nonce: number
     """
-    def __init__(self, address, default_nonce):
+    def __init__(self, address, uuid, session=None):
         self.address = address
-        self.default_nonce = default_nonce
+        self.uuid = uuid
+        self.session = session
 
 
-    def next(self):
+    def get_nonce(self):
+        return self.next_nonce()
+
+
+    def next_nonce(self):
         """Get next unique nonce.
 
         :returns: Nonce
         :rtype: number
         """
-        return Nonce.next(self.address, self.default_nonce)
+        r = NonceReservation.release(self.address, self.uuid, session=self.session)
+        return r[1]

apps/cic-eth/cic_eth/eth/request.py

@@ -1,194 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import web3
-import celery
-from erc20_approval_escrow import TransferApproval
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth import RpcClient
-from cic_eth.eth.factory import TxFactory
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.error import TokenCountError
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-contract_function_signatures = {
-        'request': 'b0addede',
-        }
-
-
-class TransferRequestTxFactory(TxFactory):
-    """Factory for creating Transfer request transactions using the TransferApproval contract backend
-    """
-    def request(
-            self,
-            token_address,
-            beneficiary_address,
-            amount,
-            chain_spec,
-            ):
-        """Create a new TransferApproval.request transaction
-
-        :param token_address: Token to create transfer request for
-        :type token_address: str, 0x-hex
-        :param beneficiary_address: Beneficiary of token transfer
-        :type beneficiary_address: str, 0x-hex
-        :param amount: Amount of tokens to transfer
-        :type amount: number
-        :param chain_spec: Chain spec
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Transaction in standard Ethereum format
-        :rtype: dict
-        """
-        transfer_approval = CICRegistry.get_contract(chain_spec, 'TransferApproval', 'TransferAuthorization')
-        fn = transfer_approval.function('createRequest')
-        tx_approval_buildable = fn(beneficiary_address, token_address, amount)
-        transfer_approval_gas = transfer_approval.gas('createRequest')
-
-        tx_approval = tx_approval_buildable.buildTransaction({
-            'from': self.address,
-            'gas': transfer_approval_gas,
-            'gasPrice': self.gas_price,
-            'chainId': chain_spec.chain_id(),
-            'nonce': self.next_nonce(),
-            })
-        return tx_approval
-
-
-def unpack_transfer_approval_request(data):
-    """Verifies that a transaction is an "TransferApproval.request" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != contract_function_signatures['request']:
-        raise ValueError('Invalid transfer request data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        'token': web3.Web3.toChecksumAddress('0x' + d[128-40:128]),
-        'amount': int(d[128:], 16)
-        }
-
-
-@celery_app.task(bind=True)
-def transfer_approval_request(self, tokens, holder_address, receiver_address, value, chain_str):
-    """Creates a new transfer approval
-
-    :param tokens: Token to generate transfer request for
-    :type tokens: list with single token spec as dict
-    :param holder_address: Address to generate transfer on behalf of
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Address to transfser tokens to
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of tokens to transfer
-    :type value: number
-    :param chain_spec: Chain spec string representation
-    :type chain_spec: str
-    :raises cic_eth.error.TokenCountError: More than one token in tokens argument
-    :returns: Raw signed transaction
-    :rtype: list with transaction as only element
-    """
-
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    t = tokens[0]
-
-    c = RpcClient(holder_address)
-
-    txf = TransferRequestTxFactory(holder_address, c)
-
-    tx_transfer = txf.request(t['address'], receiver_address, value, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.request.otx_cache_transfer_approval_request')
-
-    gas_budget = tx_transfer['gas'] * tx_transfer['gasPrice']
-
-    s = create_check_gas_and_send_task(
-             [tx_signed_raw_hex],
-             chain_str,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return [tx_signed_raw_hex]
-
-
-@celery_app.task()
-def otx_cache_transfer_approval_request(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-    """Generates and commits transaction cache metadata for an TransferApproval.request transaction
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx_signed_raw_hex: Raw signed transaction
-    :type tx_signed_raw_hex: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    logg.debug('in otx acche transfer approval request')
-    (txc, cache_id) = cache_transfer_approval_request_data(tx_hash_hex, tx)
-    return txc
-
-
-@celery_app.task()
-def cache_transfer_approval_request_data(
-    tx_hash_hex,
-    tx,
-        ):
-    """Helper function for otx_cache_transfer_approval_request
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    tx_data = unpack_transfer_approval_request(tx['data'])
-    logg.debug('tx approval request data {}'.format(tx_data))
-    logg.debug('tx approval request {}'.format(tx))
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx_data['to'],
-        tx_data['token'],
-        tx_data['token'],
-        tx_data['amount'],
-        tx_data['amount'],
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/rpc.py

@@ -1,39 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from cic_eth.eth.gas import GasOracle
-
-logg = logging.getLogger()
-
-
-class RpcClient(GasOracle):
-    """RPC wrapper for web3 enabling gas calculation helpers and signer middleware.
-
-    :param chain_spec: Chain spec
-    :type chain_spec: cic_registry.chain.ChainSpec
-    :param holder_address: DEPRECATED Address of subject of the session. 
-    :type holder_address: str, 0x-hex
-    """
-
-    signer_ipc_path = None
-    """Unix socket path to JSONRPC signer and keystore"""
-
-    web3_constructor = None
-    """Custom function to build a web3 object with middleware plugins"""
-
-
-    def __init__(self, chain_spec, holder_address=None):
-        (self.provider, w3) = RpcClient.web3_constructor()
-        super(RpcClient, self).__init__(w3)
-        self.chain_spec = chain_spec
-        if holder_address != None:
-            self.holder_address = holder_address
-            logg.info('gasprice {}'.format(self.gas_price()))
-
-
-    @staticmethod
-    def set_constructor(web3_constructor):
-        """Sets the constructor to use for building the web3 object.
-        """
-        RpcClient.web3_constructor = web3_constructor

apps/cic-eth/cic_eth/eth/task.py

@@ -1,136 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.eth import RpcClient
-from cic_eth.queue.tx import create as queue_create
-
-celery_app = celery.current_app
-logg = celery_app.log.get_default_logger()
-
-
-@celery_app.task()
-def sign_tx(tx, chain_str):
-    """Sign a single transaction against the given chain specification.
-
-    :param tx: Transaction in standard Ethereum format
-    :type tx: dict
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and raw signed transaction, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    tx_transfer_signed = c.w3.eth.sign_transaction(tx) 
-    logg.debug('tx_transfer_signed {}'.format(tx_transfer_signed))
-    tx_hash = c.w3.keccak(hexstr=tx_transfer_signed['raw'])
-    tx_hash_hex = tx_hash.hex()
-    return (tx_hash_hex, tx_transfer_signed['raw'],)
-
-
-def sign_and_register_tx(tx, chain_str, queue, cache_task=None):
-    """Signs the provided transaction, and adds it to the transaction queue cache (with status PENDING).
-
-    :param tx: Standard ethereum transaction data
-    :type tx: dict
-    :param chain_str: Chain spec, string representation
-    :type chain_str: str
-    :param queue: Task queue
-    :type queue: str
-    :param cache_task: Cache task to call with signed transaction. If None, no task will be called.
-    :type cache_task: str
-    :returns: Tuple; Transaction hash, signed raw transaction data
-    :rtype: tuple
-    """
-    (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, chain_str)
-
-    logg.debug('adding queue tx {}'.format(tx_hash_hex))
-
-#    s = celery.signature(
-#        'cic_eth.queue.tx.create',
-#        [
-#            tx['nonce'],
-#            tx['from'],
-#            tx_hash_hex,
-#            tx_signed_raw_hex,
-#            chain_str,
-#            ],
-#        queue=queue,
-#        )
-
-    # TODO: consider returning this as a signature that consequtive tasks can be linked to
-    queue_create(
-        tx['nonce'],
-        tx['from'],
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-    )        
-
-    if cache_task != None:
-        logg.debug('adding cache task {} tx {}'.format(cache_task, tx_hash_hex))
-        s_cache = celery.signature(
-                cache_task,
-                [
-                    tx_hash_hex,
-                    tx_signed_raw_hex,
-                    chain_str,
-                    ],
-                queue=queue,
-                )
-        s_cache.apply_async()
-
-    return (tx_hash_hex, tx_signed_raw_hex,)
-
-
-# TODO: rename as we will not be sending task in the chain, this is the responsibility of the dispatcher
-def create_check_gas_and_send_task(tx_signed_raws_hex, chain_str, holder_address, gas, tx_hashes_hex=None, queue=None):
-    """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
-
-    If tx_hashes_hex is not spefified, a preceding task chained to check_gas must supply the transaction hashes as its return value.
-
-    :param tx_signed_raws_hex: Raw signed transaction data
-    :type tx_signed_raws_hex: list of str, 0x-hex
-    :param chain_str: Chain spec, string representation
-    :type chain_str: str
-    :param holder_address: Address sending the transactions
-    :type holder_address: str, 0x-hex
-    :param gas: Gas budget hint for transactions
-    :type gas: int
-    :param tx_hashes_hex: Transaction hashes
-    :type tx_hashes_hex: list of str, 0x-hex
-    :param queue: Task queue
-    :type queue: str
-    :returns: Signature of task chain
-    :rtype: celery.Signature
-    """
-    s_check_gas = None
-    if tx_hashes_hex != None:
-        s_check_gas = celery.signature(
-                'cic_eth.eth.tx.check_gas',
-                [
-                    tx_hashes_hex,
-                    chain_str,
-                    tx_signed_raws_hex,
-                    holder_address,
-                    gas,
-                    ],
-                queue=queue,
-                )
-    else:
-        s_check_gas = celery.signature(
-                'cic_eth.eth.tx.check_gas',
-                [
-                    chain_str,
-                    tx_signed_raws_hex,
-                    holder_address,
-                    gas,
-                    ],
-                queue=queue,
-                )
-    return s_check_gas

apps/cic-eth/cic_eth/eth/token.py

@@ -1,506 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-import requests
-import web3
-from cic_registry import CICRegistry
-from cic_registry import zero_address
-from cic_registry.chain import ChainSpec
-
-# platform imports
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth import RpcClient
-from cic_eth.error import TokenCountError, PermanentTxError, OutOfGasError, NotLocalTxError
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.eth.factory import TxFactory
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.ext.address import translate_address
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-# TODO: fetch from cic-contracts instead when implemented
-contract_function_signatures = {
-        'transfer': 'a9059cbb',
-        'approve': '095ea7b3',
-        'transferfrom': '23b872dd',
-        }
-
-
-class TokenTxFactory(TxFactory):
-    """Factory for creating ERC20 token transactions.
-    """
-    def approve(
-            self,
-            token_address,
-            spender_address,
-            amount,
-            chain_spec,
-            ):
-        """Create an ERC20 "approve" transaction
-
-        :param token_address: ERC20 contract address
-        :type token_address: str, 0x-hex
-        :param spender_address: Address to approve spending for
-        :type spender_address: str, 0x-hex
-        :param amount: Amount of tokens to approve
-        :type amount: int
-        :param chain_spec: Chain spec
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Unsigned "approve" transaction in standard Ethereum format
-        :rtype: dict
-        """
-        source_token = CICRegistry.get_address(chain_spec, token_address)
-        source_token_contract = source_token.contract
-        tx_approve_buildable = source_token_contract.functions.approve(
-            spender_address,
-            amount,
-        )
-        source_token_gas = source_token.gas('transfer')
-
-        tx_approve = tx_approve_buildable.buildTransaction({
-            'from': self.address,
-            'gas': source_token_gas,
-            'gasPrice': self.gas_price,
-            'chainId': chain_spec.chain_id(),
-            'nonce': self.next_nonce(),
-            })
-        return tx_approve
-
-
-    def transfer(
-        self,
-        token_address,
-        receiver_address,
-        value,
-        chain_spec,
-        ):
-        """Create an ERC20 "transfer" transaction
-
-        :param token_address: ERC20 contract address
-        :type token_address: str, 0x-hex
-        :param receiver_address: Address to send tokens to
-        :type receiver_address: str, 0x-hex
-        :param amount: Amount of tokens to send
-        :type amount: int
-        :param chain_spec: Chain spec
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Unsigned "transfer" transaction in standard Ethereum format
-        :rtype: dict
-        """
-        source_token = CICRegistry.get_address(chain_spec, token_address)
-        source_token_contract = source_token.contract
-        transfer_buildable = source_token_contract.functions.transfer(
-                receiver_address,
-                value,
-                )
-        source_token_gas = source_token.gas('transfer')
-
-        tx_transfer = transfer_buildable.buildTransaction(
-                {
-                    'from': self.address,
-                    'gas': source_token_gas,
-                    'gasPrice': self.gas_price,
-                    'chainId': chain_spec.chain_id(),
-                    'nonce': self.next_nonce(),
-                })
-        return tx_transfer
-
-
-def unpack_transfer(data):
-    """Verifies that a transaction is an "ERC20.transfer" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != contract_function_signatures['transfer']:
-        raise ValueError('Invalid transfer data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        'amount': int(d[64:], 16)
-        }
-
-
-def unpack_transferfrom(data):
-    """Verifies that a transaction is an "ERC20.transferFrom" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != contract_function_signatures['transferfrom']:
-        raise ValueError('Invalid transferFrom data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'from': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        'to': web3.Web3.toChecksumAddress('0x' + d[128-40:128]),
-        'amount': int(d[128:], 16)
-        }
-
-
-def unpack_approve(data):
-    """Verifies that a transaction is an "ERC20.approve" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != contract_function_signatures['approve']:
-        raise ValueError('Invalid approval data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        'amount': int(d[64:], 16)
-        }
-
-
-@celery_app.task()
-def balance(tokens, holder_address, chain_str):
-    """Return token balances for a list of tokens for given address
-
-    :param tokens: Token addresses
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :return: List of balances
-    :rtype: list of int
-    """
-    #abi = ContractRegistry.abi('ERC20Token')
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    for t in tokens:
-        #token = CICRegistry.get_address(t['address'])
-        #abi = token.abi()
-        #o = c.w3.eth.contract(abi=abi, address=t['address'])
-        o = CICRegistry.get_address(chain_spec, t['address']).contract
-        b = o.functions.balanceOf(holder_address).call()
-        t['balance_network'] = b
-
-    return tokens
-
-
-@celery_app.task(bind=True)
-def transfer(self, tokens, holder_address, receiver_address, value, chain_str):
-    """Transfer ERC20 tokens between addresses
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Token receiver address
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :raises TokenCountError: More than one token is passed in tokens list
-    :return: Transaction hash for tranfer operation
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    # retrieve the token interface
-    t = tokens[0]
-
-    c = RpcClient(chain_spec, holder_address=holder_address)
-
-    txf = TokenTxFactory(holder_address, c)
-
-    tx_transfer = txf.transfer(t['address'], receiver_address, value, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, cache_task='cic_eth.eth.token.otx_cache_transfer')
-    
-    gas_budget = tx_transfer['gas'] * tx_transfer['gasPrice']
-
-    s = create_check_gas_and_send_task(
-             [tx_signed_raw_hex],
-             chain_str,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return tx_hash_hex
-
-
-@celery_app.task(bind=True)
-def approve(self, tokens, holder_address, spender_address, value, chain_str):
-    """Approve ERC20 transfer on behalf of holder address
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Token receiver address
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :raises TokenCountError: More than one token is passed in tokens list
-    :return: Transaction hash for tranfer operation
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    # retrieve the token interface
-    t = tokens[0]
-
-    c = RpcClient(chain_spec, holder_address=holder_address)
-
-    txf = TokenTxFactory(holder_address, c)
-
-    tx_transfer = txf.approve(t['address'], spender_address, value, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, cache_task='cic_eth.eth.token.otx_cache_approve')
-    
-    gas_budget = tx_transfer['gas'] * tx_transfer['gasPrice']
-
-    s = create_check_gas_and_send_task(
-             [tx_signed_raw_hex],
-             chain_str,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return tx_hash_hex
-
-
-@celery_app.task()
-def resolve_tokens_by_symbol(token_symbols, chain_str):
-    """Returns contract addresses of an array of ERC20 token symbols
-
-    :param token_symbols: Token symbols to resolve
-    :type token_symbols: list of str
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-
-    :return: Respective token contract addresses
-    :rtype: list of str, 0x-hex
-    """
-    tokens = []
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    for token_symbol in token_symbols:
-        token = CICRegistry.get_token(chain_spec, token_symbol)
-        tokens.append({
-            'address': token.address(),
-            'converters': [],
-            })
-    return tokens
-
-
-@celery_app.task()
-def otx_cache_transfer(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-    """Generates and commits transaction cache metadata for an ERC20.transfer or ERC20.transferFrom transaction
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx_signed_raw_hex: Raw signed transaction
-    :type tx_signed_raw_hex: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    (txc, cache_id) = cache_transfer_data(tx_hash_hex, tx)
-    return txc
-
-
-@celery_app.task()
-def cache_transfer_data(
-    tx_hash_hex,
-    tx,
-        ):
-    """Helper function for otx_cache_transfer
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    tx_data = unpack_transfer(tx['data'])
-    logg.debug('tx data {}'.format(tx_data))
-    logg.debug('tx {}'.format(tx))
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx_data['to'],
-        tx['to'],
-        tx['to'],
-        tx_data['amount'],
-        tx_data['amount'],
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)
-
-
-@celery_app.task()
-def otx_cache_approve(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-    """Generates and commits transaction cache metadata for an ERC20.approve transaction
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx_signed_raw_hex: Raw signed transaction
-    :type tx_signed_raw_hex: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    (txc, cache_id) = cache_approve_data(tx_hash_hex, tx)
-    return txc
-
-
-@celery_app.task()
-def cache_approve_data(
-    tx_hash_hex,
-    tx,
-        ):
-    """Helper function for otx_cache_approve
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    tx_data = unpack_approve(tx['data'])
-    logg.debug('tx data {}'.format(tx_data))
-    logg.debug('tx {}'.format(tx))
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx_data['to'],
-        tx['to'],
-        tx['to'],
-        tx_data['amount'],
-        tx_data['amount'],
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)
-
-
-class ExtendedTx:
-
-    _default_decimals = 6
-
-    def __init__(self, tx_hash, chain_spec):
-        self._chain_spec = chain_spec
-        self.chain = str(chain_spec)
-        self.hash = tx_hash
-        self.sender = None
-        self.sender_label = None
-        self.recipient = None
-        self.recipient_label = None
-        self.source_token_value = 0
-        self.destination_token_value = 0
-        self.source_token = zero_address
-        self.destination_token = zero_address
-        self.source_token_symbol = ''
-        self.destination_token_symbol = ''
-        self.source_token_decimals = ExtendedTx._default_decimals
-        self.destination_token_decimals = ExtendedTx._default_decimals
-
-
-    def set_actors(self, sender, recipient, trusted_declarator_addresses=None):
-        self.sender = sender
-        self.recipient = recipient
-        if trusted_declarator_addresses != None:
-            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain)
-            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain)
-
-
-    def set_tokens(self, source, source_value, destination=None, destination_value=None):
-        if destination == None:
-            destination = source
-        if destination_value == None:
-            destination_value = source_value
-        st = CICRegistry.get_address(self._chain_spec, source)
-        dt = CICRegistry.get_address(self._chain_spec, destination)
-        self.source_token = source
-        self.source_token_symbol = st.symbol()
-        self.source_token_decimals = st.decimals()
-        self.source_token_value = source_value
-        self.destination_token = destination
-        self.destination_token_symbol = dt.symbol()
-        self.destination_token_decimals = dt.decimals()
-        self.destination_token_value = destination_value
-
-
-    def to_dict(self):
-        o = {}
-        for attr in dir(self):
-            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'to_dict']:
-                continue
-            o[attr] = getattr(self, attr)
-        return o 

apps/cic-eth/cic_eth/eth/tx.py

@@ -4,15 +4,42 @@ import logging
 # third-party imports
 import celery
 import requests
-import web3
-from cic_registry import zero_address
-from cic_registry.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.chain import ChainSpec
+from chainlib.eth.address import is_checksum_address
+from chainlib.eth.gas import balance
+from chainlib.eth.error import (
+        EthException,
+        NotFoundEthException,
+        )
+from chainlib.eth.tx import (
+        transaction,
+        receipt,
+        raw,
+        TxFormat,
+        unpack,
+        )
+from chainlib.connection import RPCConnection
+from chainlib.hash import keccak256_hex_to_hex
+from chainlib.eth.gas import Gas
+from chainlib.eth.contract import (
+        abi_decode_single,
+        ABIContractType,
+        )
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
-from .rpc import RpcClient
-from cic_eth.db import Otx, SessionBase
+from cic_eth.db import (
+        Otx,
+        SessionBase,
+        )
 from cic_eth.db.models.tx import TxCache
+from cic_eth.db.models.nonce import NonceReservation
 from cic_eth.db.models.lock import Lock
+from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import (
         LockEnum,
         StatusBits,
@@ -20,18 +47,30 @@ from cic_eth.db.enum import (
 from cic_eth.error import PermanentTxError
 from cic_eth.error import TemporaryTxError
 from cic_eth.error import NotLocalTxError
-from cic_eth.queue.tx import create as queue_create
-from cic_eth.queue.tx import get_tx
-from cic_eth.queue.tx import get_nonce_tx
+#from cic_eth.queue.tx import create as queue_create
+from cic_eth.queue.tx import (
+        get_tx,
+        register_tx,
+        get_nonce_tx,
+        )
 from cic_eth.error import OutOfGasError
 from cic_eth.error import LockedError
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import sign_and_register_tx, create_check_gas_and_send_task
-from cic_eth.eth.task import sign_tx
-from cic_eth.eth.nonce import NonceOracle
-from cic_eth.error import AlreadyFillingGasError
-from cic_eth.eth.util import tx_hex_string
+from cic_eth.eth.gas import (
+        create_check_gas_task,
+        )
+from cic_eth.eth.nonce import CustodialTaskNonceOracle
+from cic_eth.error import (
+        AlreadyFillingGasError,
+        EthError,
+        )
 from cic_eth.admin.ctrl import lock_send
+from cic_eth.task import (
+        CriticalSQLAlchemyTask,
+        CriticalWeb3Task,
+        CriticalWeb3AndSignerTask,
+        CriticalSQLAlchemyAndSignerTask,
+        CriticalSQLAlchemyAndWeb3Task,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -40,8 +79,8 @@ MAX_NONCE_ATTEMPTS = 3
 
 
 # TODO this function is too long
-@celery_app.task(bind=True, throws=(OutOfGasError))
-def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=None):
+@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -50,8 +89,8 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
 
     :param tx_hashes: Transaction hashes due to be submitted
     :type tx_hashes: list of str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
+    :param chain_spec_dict: Chain spec dict representation
+    :type chain_spec_dict: dict
     :param txs: Signed raw transaction data, corresponding to tx_hashes
     :type txs: list of str, 0x-hex
     :param address: Sender address
@@ -65,31 +104,53 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
         for i in range(len(tx_hashes)):
             o = get_tx(tx_hashes[i])
             txs.append(o['signed_tx'])
-            logg.debug('ooooo {}'.format(o))
             if address == None:
                 address = o['address']
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    #if not web3.Web3.isChecksumAddress(address):
+    if not is_checksum_address(address):
+        raise ValueError('invalid address {}'.format(address))
 
-    queue = self.request.delivery_info['routing_key']
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
-    #c = RpcClient(chain_spec, holder_address=address)
-    c = RpcClient(chain_spec)
+    queue = self.request.delivery_info.get('routing_key')
+
+    conn = RPCConnection.connect(chain_spec)
 
     # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
-    balance = c.w3.eth.getBalance(address) 
-    logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
+    gas_balance = 0
+    try:
+        o = balance(address)
+        r = conn.do(o)
+        conn.disconnect()
+        gas_balance = abi_decode_single(ABIContractType.UINT256, r)
+    except EthException as e:
+        conn.disconnect()
+        raise EthError('gas_balance call for {}: {}'.format(address, e))
 
-    if gas_required > balance:
+    logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
+    session = SessionBase.create_session()
+    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    session.close()
+
+    if gas_required > gas_balance:
+        s_nonce = celery.signature(
+            'cic_eth.eth.tx.reserve_nonce',
+            [
+                address,
+                gas_provider,
+                ],
+            queue=queue,
+            )
         s_refill_gas = celery.signature(
             'cic_eth.eth.tx.refill_gas',
             [
-                address,
-                chain_str,
+                chain_spec_dict,
                 ],
             queue=queue,
                 )
-        s_refill_gas.apply_async()
+        s_nonce.link(s_refill_gas)
+        s_nonce.apply_async()
         wait_tasks = []
         for tx_hash in tx_hashes:
             s = celery.signature(
@@ -101,20 +162,28 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
                 )
             wait_tasks.append(s)
         celery.group(wait_tasks)()
-        raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, balance))
+        raise OutOfGasError('need to fill gas, required {}, had {}'.format(gas_required, gas_balance))
 
-    safe_gas = c.safe_threshold_amount()
-    if balance < safe_gas:
+    safe_gas = self.safe_gas_threshold_amount
+    if gas_balance < safe_gas:
+        s_nonce = celery.signature(
+            'cic_eth.eth.tx.reserve_nonce',
+            [
+                address,
+                gas_provider,
+                ],
+            queue=queue,
+            )
         s_refill_gas = celery.signature(
             'cic_eth.eth.tx.refill_gas',
             [
-                address,
-                chain_str,
+                chain_spec_dict,
                 ],
             queue=queue,
                 )
-        s_refill_gas.apply_async()
-        logg.debug('requested refill from {} to {}'.format(c.gas_provider(), address))
+        s_nonce.link(s_refill_gas)
+        s_nonce.apply_async()
+        logg.debug('requested refill from {} to {}'.format(gas_provider, address))
     ready_tasks = []
     for tx_hash in tx_hashes:
         s = celery.signature(
@@ -131,7 +200,7 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
 
 
 # TODO: chain chainable transactions that use hashes as inputs may be chained to this function to output signed txs instead.
-@celery_app.task(bind=True)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def hashes_to_txs(self, tx_hashes):
     """Return a list of raw signed transactions from the local transaction queue corresponding to a list of transaction hashes.
 
@@ -146,8 +215,6 @@ def hashes_to_txs(self, tx_hashes):
 
     queue = self.request.delivery_info['routing_key']
 
-    #otxs = ','.format("'{}'".format(tx_hash) for tx_hash in tx_hashes)
-
     session = SessionBase.create_session()
     q = session.query(Otx.signed_tx)
     q = q.filter(Otx.tx_hash.in_(tx_hashes))
@@ -164,157 +231,9 @@ def hashes_to_txs(self, tx_hashes):
     return txs
 
 
-# TODO: Move this and send to subfolder submodule
-class ParityNodeHandler:
-    def __init__(self, chain_spec, queue):
-        self.chain_spec = chain_spec
-        self.chain_str = str(chain_spec)
-        self.queue = queue
-
-    def handle(self, exception, tx_hash_hex, tx_hex):
-        meth = self.handle_default
-        if isinstance(exception, (ValueError)):
-#            s_debug = celery.signature(
-#                'cic_eth.admin.debug.out_tmp',
-#                [tx_hash_hex, '{}: {}'.format(tx_hash_hex, exception)],
-#                queue=queue,
-#                )
-#            s_debug.apply_async()
-            earg = exception.args[0]
-            if earg['code'] == -32010:
-                logg.debug('skipping lock for code {}'.format(earg['code']))
-                meth = self.handle_invalid_parameters
-            elif earg['code'] == -32602:
-                meth = self.handle_invalid_encoding
-            else:
-                meth = self.handle_invalid
-        elif isinstance(exception, (requests.exceptions.ConnectionError)):
-            meth = self.handle_connection
-        (t, e_fn, message) = meth(tx_hash_hex, tx_hex)
-        return (t, e_fn, '{} {}'.format(message, exception))
-           
-
-    def handle_connection(self, tx_hash_hex, tx_hex):
-        s_set_sent = celery.signature(
-            'cic_eth.queue.tx.set_sent_status',
-            [
-                tx_hash_hex,
-                True,
-                ],
-            queue=self.queue,
-            )
-        t = s_set_sent.apply_async()
-        return (t, TemporaryTxError, 'Sendfail {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
-
-
-    def handle_invalid_encoding(self, tx_hash_hex, tx_hex):
-        tx_bytes = bytes.fromhex(tx_hex[2:])
-        tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id())
-        s_lock = celery.signature(
-            'cic_eth.admin.ctrl.lock_send',
-            [
-                tx_hash_hex,
-                self.chain_str,
-                tx['from'],
-                tx_hash_hex,
-                ],
-            queue=self.queue,
-            )
-        s_set_reject = celery.signature(
-            'cic_eth.queue.tx.set_rejected',
-            [],
-            queue=self.queue,
-            )
-        nonce_txs = get_nonce_tx(tx['nonce'], tx['from'], self.chain_spec.chain_id())
-        attempts = len(nonce_txs)
-        if attempts < MAX_NONCE_ATTEMPTS:
-            logg.debug('nonce {} address {} retries {} < {}'.format(tx['nonce'], tx['from'], attempts, MAX_NONCE_ATTEMPTS))
-            s_resend = celery.signature(
-                    'cic_eth.eth.tx.resend_with_higher_gas',
-                    [
-                        self.chain_str,
-                        None,
-                        1.01,
-                        ],
-                    queue=self.queue,
-                    )
-            s_unlock = celery.signature(
-                    'cic_eth.admin.ctrl.unlock_send',
-                    [
-                        self.chain_str,
-                        tx['from'],
-                        ],
-                    queue=self.queue,
-                    )
-            s_resend.link(s_unlock)
-            s_set_reject.link(s_resend)
-
-        s_lock.link(s_set_reject)
-        t = s_lock.apply_async()
-        return (t, PermanentTxError, 'Reject invalid encoding {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
-
-
-    def handle_invalid_parameters(self, tx_hash_hex, tx_hex):
-        s_sync = celery.signature(
-            'cic_eth.eth.tx.sync_tx',
-            [
-                tx_hash_hex,
-                self.chain_str,
-                ],
-            queue=self.queue,
-            )
-        t = s_sync.apply_async()
-        return (t, PermanentTxError, 'Reject invalid parameters {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
-
-
-    def handle_invalid(self, tx_hash_hex, tx_hex):
-        tx_bytes = bytes.fromhex(tx_hex[2:])
-        tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id())
-        s_lock = celery.signature(
-            'cic_eth.admin.ctrl.lock_send',
-            [
-                tx_hash_hex,
-                self.chain_str,
-                tx['from'],
-                tx_hash_hex,
-                ],
-            queue=self.queue,
-            )
-        s_set_reject = celery.signature(
-            'cic_eth.queue.tx.set_rejected',
-            [],
-            queue=self.queue,
-            )
-        s_lock.link(s_set_reject)
-        t = s_lock.apply_async()
-        return (t, PermanentTxError, 'Reject invalid {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
-
-
-    def handle_default(self, tx_hash_hex, tx_hex):
-        tx_bytes = bytes.fromhex(tx_hex[2:])
-        tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id())
-        s_lock = celery.signature(
-                'cic_eth.admin.ctrl.lock_send',
-                [
-                    tx_hash_hex,
-                    self.chain_str,
-                    tx['from'],
-                    tx_hash_hex,
-                    ],
-                queue=self.queue,
-                )
-        s_set_fubar = celery.signature(
-            'cic_eth.queue.tx.set_fubar',
-            [],
-            queue=self.queue,
-            )
-        s_lock.link(s_set_fubar)
-        t = s_lock.apply_async()
-        return (t, PermanentTxError, 'Fubar {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id())))
-
-
-@celery_app.task(bind=True)
-def send(self, txs, chain_str):
+# TODO: A lock should be introduced to ensure that the send status change and the transaction send is atomic.
+@celery_app.task(bind=True, base=CriticalWeb3Task)
+def send(self, txs, chain_spec_dict):
     """Send transactions to the network.
 
     If more than one transaction is passed to the task, it will spawn a new send task with the remaining transaction(s) after the first in the list has been processed.
@@ -339,25 +258,17 @@ def send(self, txs, chain_str):
     if len(txs) == 0:
         raise ValueError('no transaction to send')
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_hex = txs[0]
-    logg.debug('send transaction {}'.format(tx_hex))
 
-    tx_hash = web3.Web3.keccak(hexstr=tx_hex)
-    tx_hash_hex = tx_hash.hex()
+    tx_hash_hex = add_0x(keccak256_hex_to_hex(tx_hex))
 
-    queue = self.request.delivery_info.get('routing_key', None)
+    logg.debug('send transaction {} -> {}'.format(tx_hash_hex, tx_hex))
+
+    queue = self.request.delivery_info.get('routing_key')
 
-    c = RpcClient(chain_spec)
     r = None
-    try:
-        r = c.w3.eth.send_raw_transaction(tx_hex)
-    except Exception as e:
-        raiser = ParityNodeHandler(chain_spec, queue)
-        (t, e, m) = raiser.handle(e, tx_hash_hex, tx_hex)
-        raise e(m)
-
     s_set_sent = celery.signature(
         'cic_eth.queue.tx.set_sent_status',
         [
@@ -366,6 +277,11 @@ def send(self, txs, chain_str):
             ],
             queue=queue,
         )
+
+    o = raw(tx_hex)
+    conn = RPCConnection.connect(chain_spec, 'default')
+    conn.do(o)
+
     s_set_sent.apply_async()
 
     tx_tail = txs[1:]
@@ -377,11 +293,13 @@ def send(self, txs, chain_str):
                 )
         s.apply_async()
 
-    return r.hex()
+    return tx_hash_hex
 
 
-@celery_app.task(bind=True, throws=(AlreadyFillingGasError))
-def refill_gas(self, recipient_address, chain_str):
+# TODO: if this method fails the nonce will be out of sequence. session needs to be extended to include the queue create, so that nonce is rolled back if the second sql query fails. Better yet, split each state change into separate tasks.
+# TODO: method is too long, factor out code for clarity
+@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3AndSignerTask)
+def refill_gas(self, recipient_address, chain_spec_dict):
     """Executes a native token transaction to fund the recipient's gas expenditures.
 
     :param recipient_address: Recipient in need of gas
@@ -392,8 +310,14 @@ def refill_gas(self, recipient_address, chain_str):
     :returns: Transaction hash.
     :rtype: str, 0x-hex
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    # essentials
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
 
+    # Determine value of gas tokens to send
+    # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
+    # however, we will perform a 0-value transaction instead
+    zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
     q = session.query(Otx.tx_hash)
@@ -402,56 +326,37 @@ def refill_gas(self, recipient_address, chain_str):
     q = q.filter(TxCache.from_value!=0)
     q = q.filter(TxCache.recipient==recipient_address)
     c = q.count()
-    session.close()
     if c > 0:
-        raise AlreadyFillingGasError(recipient_address)
+        logg.warning('already filling gas {}'.format(str(AlreadyFillingGasError(recipient_address))))
+        zero_amount = True
+    session.flush()
 
-    queue = self.request.delivery_info['routing_key']
+    # finally determine the value to send
+    refill_amount = 0
+    if not zero_amount:
+        refill_amount = self.safe_gas_refill_amount
 
-    c = RpcClient(chain_spec)
-    clogg = celery_app.log.get_default_logger()
-    logg.debug('refill gas from provider address {}'.format(c.gas_provider()))
-    default_nonce = c.w3.eth.getTransactionCount(c.gas_provider(), 'pending')
-    nonce_generator = NonceOracle(c.gas_provider(), default_nonce)
-    nonce = nonce_generator.next()
-    gas_price = c.gas_price()
-    gas_limit = c.default_gas_limit
-    refill_amount = c.refill_amount()
-    logg.debug('tx send gas price {} nonce {}'.format(gas_price, nonce))
+    # determine sender
+    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    session.flush()
 
-    # create and sign transaction
-    tx_send_gas = {
-                'from': c.gas_provider(),
-                'to': recipient_address,
-                'gas': gas_limit,
-                'gasPrice': gas_price,
-                'chainId': chain_spec.chain_id(),
-                'nonce': nonce,
-                'value': refill_amount,
-                'data': '',
-            }
-    tx_send_gas_signed = c.w3.eth.sign_transaction(tx_send_gas)
-    tx_hash = web3.Web3.keccak(hexstr=tx_send_gas_signed['raw'])
-    tx_hash_hex = tx_hash.hex()
+    # set up evm RPC connection
+    rpc = RPCConnection.connect(chain_spec, 'default')
 
-    # TODO: route this through sign_and_register_tx instead
+    # set up transaction builder
+    nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc)
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+    c = Gas(signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
+
+    # build and add transaction
+    logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
     logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
-    queue_create(
-        nonce,
-        c.gas_provider(),
-        tx_hash_hex,
-        tx_send_gas_signed['raw'],
-        chain_str,
-        )
+    cache_task = 'cic_eth.eth.tx.cache_gas_data'
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
 
-    s_tx_cache = celery.signature(
-            'cic_eth.eth.tx.cache_gas_refill_data',
-            [
-                tx_hash_hex,
-                tx_send_gas,
-                ],
-            queue=queue,
-            )
+    # add transaction to send queue
     s_status = celery.signature(
         'cic_eth.queue.tx.set_ready',
         [
@@ -459,11 +364,12 @@ def refill_gas(self, recipient_address, chain_str):
             ],
         queue=queue,
             )
-    celery.group(s_tx_cache, s_status)()
-    return tx_send_gas_signed['raw']
+    t = s_status.apply_async()
+
+    return tx_signed_raw_hex
 
 
-@celery_app.task(bind=True)
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_factor=1.1):
     """Create a new transaction from an existing one with same nonce and higher gas price.
 
@@ -481,19 +387,18 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
     """
     session = SessionBase.create_session()
 
-    
     q = session.query(Otx)
     q = q.filter(Otx.tx_hash==txold_hash_hex)
     otx = q.first()
-    session.close()
     if otx == None:
+        session.close()
         raise NotLocalTxError(txold_hash_hex)
 
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
 
     tx_signed_raw_bytes = bytes.fromhex(otx.signed_tx[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
+    tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
     logg.debug('resend otx {} {}'.format(tx, otx.signed_tx))
 
     queue = self.request.delivery_info['routing_key']
@@ -521,8 +426,10 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
         tx_hash_hex,
         tx_signed_raw_hex,
         chain_str,
+        session=session,
             )
-    TxCache.clone(txold_hash_hex, tx_hash_hex)
+    TxCache.clone(txold_hash_hex, tx_hash_hex, session=session)
+    session.close()
 
     s = create_check_gas_and_send_task(
             [tx_signed_raw_hex],
@@ -537,19 +444,63 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
     return tx_hash_hex
 
 
-@celery_app.task(bind=True, throws=(web3.exceptions.TransactionNotFound,))
-def sync_tx(self, tx_hash_hex, chain_str):
+@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
+def reserve_nonce(self, chained_input, signer_address=None):
 
-    queue = self.request.delivery_info['routing_key']
+    self.log_banner()
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
+    session = SessionBase.create_session()
+
+    address = None
+    if signer_address == None:
+        address = chained_input
+        logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
+    else:
+        #if web3.Web3.isChecksumAddress(signer_address):
+        if is_checksum_address(signer_address):
+            address = signer_address
+            logg.debug('explicit address for reserve nonce {}'.format(signer_address))
+        else:
+            address = AccountRole.get_address(signer_address, session=session)
+            logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
+
+    if not is_checksum_address(address):
+        raise ValueError('invalid result when resolving address for nonce {}'.format(address))
+
+    root_id = self.request.root_id
+    r = NonceReservation.next(address, root_id)
+    logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
+
+    session.commit()
+
+    session.close()
+
+    return chained_input
+
+
+@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
+def sync_tx(self, tx_hash_hex, chain_spec_dict):
+    """Force update of network status of a simgle transaction
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    """
+
+    queue = self.request.delivery_info.get('routing_key')
+
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    conn = RPCConnection.connect(chain_spec, 'default')
+    o = transaction(tx_hash_hex)
+    tx = conn.do(o)
 
-    tx = c.w3.eth.getTransaction(tx_hash_hex)
     rcpt = None
     try:
-        rcpt = c.w3.eth.getTransactionReceipt(tx_hash_hex)
-    except web3.exceptions.TransactionNotFound as e:
+        o = receipt(tx_hash_hex)
+        rcpt = conn.do(o)
+    except NotFoundEthException as e:
         pass
 
     if rcpt != None:
@@ -579,79 +530,54 @@ def sync_tx(self, tx_hash_hex, chain_str):
     s.apply_async()
 
 
-
-@celery_app.task(bind=True)
-def resume_tx(self, txpending_hash_hex, chain_str):
-    """Queue a suspended tranaction for (re)sending
-
-    :param txpending_hash_hex: Transaction hash
-    :type txpending_hash_hex: str, 0x-hex
-    :param chain_str: Chain spec, string representation
-    :type chain_str: str
-    :raises NotLocalTxError: Transaction does not exist in the local queue
-    :returns: Transaction hash
-    :rtype: str, 0x-hex
-    """
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    session = SessionBase.create_session()
-    q = session.query(Otx.signed_tx)
-    q = q.filter(Otx.tx_hash==txpending_hash_hex)
-    r = q.first()
-    session.close()
-    if r == None:
-        raise NotLocalTxError(txpending_hash_hex)
-
-    tx_signed_raw_hex = r[0]
-    tx_signed_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_bytes, chain_spec.chain_id())
-
-    queue = self.request.delivery_info['routing_key']
-
-    s = create_check_gas_and_send_task(
-            [tx_signed_raw_hex],
-            chain_str,
-            tx['from'],
-            tx['gasPrice'] * tx['gas'],
-            [txpending_hash_hex],
-            queue=queue,
-            )
-    s.apply_async()
-    return txpending_hash_hex
+#
+#@celery_app.task(bind=True)
+#def resume_tx(self, txpending_hash_hex, chain_str):
+#    """Queue a suspended tranaction for (re)sending
+#
+#    :param txpending_hash_hex: Transaction hash
+#    :type txpending_hash_hex: str, 0x-hex
+#    :param chain_str: Chain spec, string representation
+#    :type chain_str: str
+#    :raises NotLocalTxError: Transaction does not exist in the local queue
+#    :returns: Transaction hash
+#    :rtype: str, 0x-hex
+#    """
+#
+#    chain_spec = ChainSpec.from_chain_str(chain_str)
+#
+#    session = SessionBase.create_session()
+#    q = session.query(Otx.signed_tx)
+#    q = q.filter(Otx.tx_hash==txpending_hash_hex)
+#    r = q.first()
+#    session.close()
+#    if r == None:
+#        raise NotLocalTxError(txpending_hash_hex)
+#
+#    tx_signed_raw_hex = r[0]
+#    tx_signed_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+#    tx = unpack_signed_raw_tx(tx_signed_bytes, chain_spec.chain_id())
+#
+#    queue = self.request.delivery_info['routing_key']
+#
+#    s = create_check_gas_and_send_task(
+#            [tx_signed_raw_hex],
+#            chain_str,
+#            tx['from'],
+#            tx['gasPrice'] * tx['gas'],
+#            [txpending_hash_hex],
+#            queue=queue,
+#            )
+#    s.apply_async()
+#    return txpending_hash_hex
 
 
-@celery_app.task()
-def otx_cache_parse_tx(
+# TODO: Move to cic_eth.eth.gas
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_gas_data(
         tx_hash_hex,
         tx_signed_raw_hex,
-        chain_str,
-        ):
-    """Generates and commits transaction cache metadata for a gas refill transaction
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx_signed_raw_hex: Raw signed transaction
-    :type tx_signed_raw_hex: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    (txc, cache_id) = cache_gas_refill_data(tx_hash_hex, tx)
-    return txc
-
-
-@celery_app.task()
-def cache_gas_refill_data(
-        tx_hash_hex,
-        tx,
+        chain_spec_dict,
         ):
     """Helper function for otx_cache_parse_tx
 
@@ -662,12 +588,16 @@ def cache_gas_refill_data(
     :returns: Transaction hash and id of cache element in storage backend, respectively
     :rtype: tuple
     """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec.chain_id())
+
     tx_cache = TxCache(
         tx_hash_hex,
         tx['from'],
         tx['to'],
-        zero_address,
-        zero_address,
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
         tx['value'],
         tx['value'],
             )

apps/cic-eth/cic_eth/eth/util.py

@@ -1,108 +0,0 @@
-# standard imports
-import logging
-import sha3
-import web3
-
-# third-party imports
-from rlp import decode as rlp_decode
-from rlp import encode as rlp_encode
-from eth_keys import KeyAPI
-
-logg = logging.getLogger()
-
-field_debugs = [
-        'nonce',
-        'gasPrice',
-        'gas',
-        'to',
-        'value',
-        'data',
-        'v',
-        'r',
-        's',
-        ]
-
-
-def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
-    d = rlp_decode(tx_raw_bytes)
-
-    logg.debug('decoding using chain id {}'.format(chain_id))
-    j = 0
-    for i in d:
-        logg.debug('decoded {}: {}'.format(field_debugs[j], i.hex()))
-        j += 1
-    vb = chain_id
-    if chain_id != 0:
-        v = int.from_bytes(d[6], 'big')
-        vb = v - (chain_id * 2) - 35
-    while len(d[7]) < 32:
-        d[7] = b'\x00' + d[7]
-    while len(d[8]) < 32:
-        d[8] = b'\x00' + d[8]
-    s = b''.join([d[7], d[8], bytes([vb])])
-    so = KeyAPI.Signature(signature_bytes=s)
-
-    h = sha3.keccak_256()
-    h.update(rlp_encode(d))
-    signed_hash = h.digest()
-
-    d[6] = chain_id
-    d[7] = b''
-    d[8] = b''
-
-    h = sha3.keccak_256()
-    h.update(rlp_encode(d))
-    unsigned_hash = h.digest()
-    
-    p = so.recover_public_key_from_msg_hash(unsigned_hash)
-    a = p.to_checksum_address()
-    logg.debug('decoded recovery byte {}'.format(vb))
-    logg.debug('decoded address {}'.format(a))
-    logg.debug('decoded signed hash {}'.format(signed_hash.hex()))
-    logg.debug('decoded unsigned hash {}'.format(unsigned_hash.hex()))
-
-    to = d[3].hex() or None
-    if to != None:
-        to = web3.Web3.toChecksumAddress('0x' + to)
-
-    return {
-        'from': a,
-        'nonce': int.from_bytes(d[0], 'big'),
-        'gasPrice': int.from_bytes(d[1], 'big'),
-        'gas': int.from_bytes(d[2], 'big'),
-        'to': to, 
-        'value': int.from_bytes(d[4], 'big'),
-        'data': '0x' + d[5].hex(),
-        'v': chain_id,
-        'r': '0x' + s[:32].hex(),
-        's': '0x' + s[32:64].hex(),
-        'chainId': chain_id,
-        'hash': '0x' + signed_hash.hex(),
-        'hash_unsigned': '0x' + unsigned_hash.hex(),
-            }
-
-
-def unpack_signed_raw_tx_hex(tx_raw_hex, chain_id):
-    return unpack_signed_raw_tx(bytes.fromhex(tx_raw_hex[2:]), chain_id)
-
-
-# TODO: consider moving tx string representation generation from api_admin to here
-def tx_string(tx_raw_bytes, chain_id):
-    tx_unpacked = unpack_signed_raw_tx(tx_raw_bytes, chain_id)
-    return 'tx nonce {} from {} to {} hash {}'.format(
-            tx_unpacked['nonce'],
-            tx_unpacked['from'],
-            tx_unpacked['to'],
-            tx_unpacked['hash'],
-        )
-
-def tx_hex_string(tx_hex, chain_id):
-    if len(tx_hex) < 2:
-        raise ValueError('invalid data length')
-    elif tx_hex[:2] == '0x':
-        tx_hex = tx_hex[2:]
-
-    tx_raw_bytes = bytes.fromhex(tx_hex)
-    return tx_string(tx_raw_bytes, chain_id)
-
-

apps/cic-eth/cic_eth/ext/address.py

@@ -3,20 +3,34 @@ import logging
 
 # third-party imports
 import celery
-from cic_registry.chain import ChainSpec
-from cic_registry import CICRegistry
+from chainlib.chain import ChainSpec
+from chainlib.connection import RPCConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from cic_eth_registry import CICRegistry
+from eth_address_declarator import AddressDeclarator
+
+# local imports
+from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 
 logg = logging.getLogger()
 
 
-def translate_address(address, trusted_addresses, chain_spec):
+def translate_address(address, trusted_addresses, chain_spec, sender_address=ZERO_ADDRESS):
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+    
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    c = AddressDeclarator()
+
     for trusted_address in trusted_addresses:
-        o = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', 'Declarator')
-        fn = o.function('declaration')
-        declaration_hex = fn(trusted_address, address).call()
-        declaration_bytes = declaration_hex[0].rstrip(b'\x00')
+        o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
+        r = rpc.do(o)
+        declaration_hex = AddressDeclarator.parse_declaration(r)
+        declaration_hex = declaration_hex[0].rstrip('0')
+        declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None
         try:
             declaration = declaration_bytes.decode('utf-8', errors='strict')
@@ -25,19 +39,19 @@ def translate_address(address, trusted_addresses, chain_spec):
         return declaration
 
 
-@celery_app.task()
-def translate_tx_addresses(tx, trusted_addresses, chain_str):
+@celery_app.task(bind=True, base=BaseTask)
+def translate_tx_addresses(self, tx, trusted_addresses, chain_spec_dict):
 
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     declaration = None
     if tx['sender_label'] == None:
-        declaration = translate_address(tx['sender'], trusted_addresses, chain_spec)
+        declaration = translate_address(tx['sender'], trusted_addresses, chain_spec, self.call_address)
     tx['sender_label'] = declaration
 
     declaration = None
     if tx['recipient_label'] == None:
-        declaration = translate_address(tx['recipient'], trusted_addresses, chain_spec)
+        declaration = translate_address(tx['recipient'], trusted_addresses, chain_spec, self.call_address)
     tx['recipient_label'] = declaration
 
     return tx

apps/cic-eth/cic_eth/ext/tx.py

@@ -3,21 +3,28 @@ import logging
 import math
 
 # third-pary imports
-import web3
 import celery
 import moolb
-from cic_registry.chain import ChainSpec
-from cic_registry.registry import CICRegistry
+from chainlib.chain import ChainSpec
+from chainlib.connection import RPCConnection
+from chainlib.eth.tx import (
+        unpack,
+        transaction_by_block,
+        receipt,
+        )
+from chainlib.eth.block import block_by_number
+from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
 
 # local imports
-from cic_eth.eth.rpc import RpcClient
 from cic_eth.db.models.otx import Otx
-from cic_eth.eth.util import unpack_signed_raw_tx
 from cic_eth.db.enum import StatusEnum
-from cic_eth.eth.token import unpack_transfer
 from cic_eth.queue.tx import get_tx_cache
 from cic_eth.queue.time import tx_times
+from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -26,8 +33,8 @@ MAX_BLOCK_TX = 250
 
 
 # TODO: Make this method easier to read
-@celery_app.task()
-def list_tx_by_bloom(bloomspec, address, chain_str):
+@celery_app.task(bind=True, base=BaseTask)
+def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     """Retrieve external transaction data matching the provided filter
 
     The bloom filter representation with the following structure (the size of the filter will be inferred from the size of the provided filter data):
@@ -49,8 +56,11 @@ def list_tx_by_bloom(bloomspec, address, chain_str):
     :returns: dict of transaction data as dict, keyed by transaction hash
     :rtype: dict of dict
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    chain_str = str(chain_spec)
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+
     block_filter_data = bytes.fromhex(bloomspec['block_filter'])
     tx_filter_data = bytes.fromhex(bloomspec['blocktx_filter'])
     databitlen = len(block_filter_data)*8
@@ -62,47 +72,53 @@ def list_tx_by_bloom(bloomspec, address, chain_str):
         block_height_bytes = block_height.to_bytes(4, 'big')
         if block_filter.check(block_height_bytes):
             logg.debug('filter matched block {}'.format(block_height))
-            block = c.w3.eth.getBlock(block_height, True)
+            o = block_by_number(block_height)
+            block = rpc.do(o)
+            logg.debug('block {}'.format(block))
 
-            for tx_index in range(0, len(block.transactions)):
+            for tx_index in range(0, len(block['transactions'])):
                 composite = tx_index + block_height
                 tx_index_bytes = composite.to_bytes(4, 'big')
                 if tx_filter.check(tx_index_bytes):
                     logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
 
                     try:
-                        tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
-                    except web3.exceptions.TransactionNotFound:
-                        logg.debug('false positive on block {} tx {}'.format(block_height, tx_index))
+                        #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
+                        o = transaction_by_block(block['hash'], tx_index)
+                        tx = rpc.do(o)
+                    except Exception as e:
+                        logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e))
                         continue
                     tx_address = None
                     tx_token_value = 0
                     try:
-                        transfer_data = unpack_transfer(tx['data'])
-                        tx_address = transfer_data['to']
-                        tx_token_value = transfer_data['amount']
+                        transfer_data = ERC20.parse_transfer_request(tx['data'])
+                        tx_address = transfer_data[0]
+                        tx_token_value = transfer_data[1]
                     except ValueError:
                         logg.debug('not a transfer transaction, skipping {}'.format(tx))
                         continue
                     if address == tx_address:
                         status = StatusEnum.SENT
                         try:
-                            rcpt = c.w3.eth.getTransactionReceipt(tx.hash)
+                            o = receipt(tx['hash'])
+                            rcpt = rpc.do(o)
                             if rcpt['status'] == 0:
                                 pending = StatusEnum.REVERTED
                             else:
                                 pending = StatusEnum.SUCCESS
-                        except web3.exceptions.TransactionNotFound:
+                        except Exception as e:
+                            logg.error('skipping receipt lookup for {}: {}'.format(tx['hash'], e))
                             pass
 
-                        tx_hash_hex = tx['hash'].hex()
-
-                        token = CICRegistry.get_address(chain_spec, tx['to'])
-                        token_symbol = token.symbol()
-                        token_decimals = token.decimals()
-                        times = tx_times(tx_hash_hex, chain_str)
+                        # TODO: pass through registry to validate declarator entry of token
+                        #token = registry.by_address(tx['to'], sender_address=self.call_address)
+                        token = ERC20Token(rpc, tx['to'])
+                        token_symbol = token.symbol
+                        token_decimals = token.decimals
+                        times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
-                            'hash': tx_hash_hex,
+                            'hash': tx['hash'],
                             'sender': tx['from'],
                             'recipient': tx_address,
                             'source_value': tx_token_value,
@@ -121,7 +137,7 @@ def list_tx_by_bloom(bloomspec, address, chain_str):
                             tx_r['date_created'] = times['queue']
                         else:
                             tx_r['date_created'] = times['network']
-                        txs[tx_hash_hex] = tx_r
+                        txs[tx['hash']] = tx_r
                         break
     return txs
 
@@ -130,7 +146,7 @@ def list_tx_by_bloom(bloomspec, address, chain_str):
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
 @celery_app.task()
-def tx_collate(tx_batches, chain_str, offset, limit, newest_first=True):
+def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -147,7 +163,10 @@ def tx_collate(tx_batches, chain_str, offset, limit, newest_first=True):
     :rtype: list
     """
     txs_by_block = {}
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    if isinstance(tx_batches, dict):
+        tx_batches = [tx_batches]
 
     for b in tx_batches:
         for v in b.values():
@@ -155,7 +174,7 @@ def tx_collate(tx_batches, chain_str, offset, limit, newest_first=True):
             k = None
             try:
                 hx = strip_0x(v)
-                tx = unpack_signed_raw_tx(bytes.fromhex(hx), chain_spec.chain_id())
+                tx = unpack(bytes.fromhex(hx), chain_spec.chain_id())
                 txc = get_tx_cache(tx['hash'])
                 txc['timestamp'] = int(txc['date_created'].timestamp())
                 txc['hash'] = txc['tx_hash']

apps/cic-eth/cic_eth/queue/balance.py

@@ -3,10 +3,10 @@ import logging
 
 # third-party imports
 import celery
+from chainlib.chain import ChainSpec
 from hexathon import strip_0x
 
 # local imports
-from cic_registry.chain import ChainSpec
 from cic_eth.db import SessionBase
 from cic_eth.db.models.otx import Otx
 from cic_eth.db.models.tx import TxCache
@@ -14,13 +14,14 @@ from cic_eth.db.enum import (
         StatusBits,
         dead,
         )
+from cic_eth.task import CriticalSQLAlchemyTask
 
 celery_app = celery.current_app
 
 logg = logging.getLogger()
 
 
-def __balance_outgoing_compatible(token_address, holder_address, chain_str):
+def __balance_outgoing_compatible(token_address, holder_address):
     session = SessionBase.create_session()
     q = session.query(TxCache.from_value)
     q = q.join(Otx)
@@ -35,8 +36,8 @@ def __balance_outgoing_compatible(token_address, holder_address, chain_str):
     return delta
 
 
-@celery_app.task()
-def balance_outgoing(tokens, holder_address, chain_str):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def balance_outgoing(tokens, holder_address, chain_spec_dict):
     """Retrieve accumulated value of unprocessed transactions sent from the given address.
 
     :param tokens: list of token spec dicts with addresses to retrieve balances for
@@ -48,15 +49,15 @@ def balance_outgoing(tokens, holder_address, chain_str):
     :returns: Tokens dicts with outgoing balance added
     :rtype: dict
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
     for t in tokens: 
-        b = __balance_outgoing_compatible(t['address'], holder_address, chain_str)
+        b = __balance_outgoing_compatible(t['address'], holder_address)
         t['balance_outgoing'] = b
 
     return tokens
 
 
-def __balance_incoming_compatible(token_address, receiver_address, chain_str):
+def __balance_incoming_compatible(token_address, receiver_address):
     session = SessionBase.create_session()
     q = session.query(TxCache.to_value)
     q = q.join(Otx)
@@ -73,8 +74,8 @@ def __balance_incoming_compatible(token_address, receiver_address, chain_str):
     return delta
 
 
-@celery_app.task()
-def balance_incoming(tokens, receipient_address, chain_str):
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def balance_incoming(tokens, receipient_address, chain_spec_dict):
     """Retrieve accumulated value of unprocessed transactions to be received by the given address.
 
     :param tokens: list of token spec dicts with addresses to retrieve balances for
@@ -86,9 +87,9 @@ def balance_incoming(tokens, receipient_address, chain_str):
     :returns: Tokens dicts with outgoing balance added
     :rtype: dict
     """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
     for t in tokens: 
-        b = __balance_incoming_compatible(t['address'], receipient_address, chain_str)
+        b = __balance_incoming_compatible(t['address'], receipient_address)
         t['balance_incoming'] = b
 
     return tokens
@@ -106,6 +107,7 @@ def assemble_balances(balances_collection):
     :rtype: list of dicts
     """
     tokens = {}
+    logg.debug('received collection {}'.format(balances_collection))
     for c in balances_collection:
         for b in c:
             address = b['address']

apps/cic-eth/cic_eth/queue/time.py

@@ -2,35 +2,37 @@
 import logging
 
 # third-party imports
-import web3
 import celery
-from cic_registry.chain import ChainSpec
+from chainlib.chain import ChainSpec
+from chainlib.connection import RPCConnection
+from chainlib.eth.block import block_by_hash
+from chainlib.eth.tx import receipt
 
 # local imports
-from cic_eth.eth.rpc import RpcClient
 from cic_eth.db.models.otx import Otx
 from cic_eth.error import NotLocalTxError
+from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
 
 celery_app = celery.current_app
 
 logg = logging.getLogger()
 
 
-# TODO: This method does not belong in the _queue_ module, it operates across queue and network
-@celery_app.task()
-def tx_times(tx_hash, chain_str):
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
+def tx_times(tx_hash, chain_spec):
+    rpc = RPCConnection.connect(chain_spec, 'default')
     time_pair = {
             'network': None,
             'queue': None,
             }
     try:
-        rcpt = c.w3.eth.getTransactionReceipt(tx_hash)
-        block = c.w3.eth.getBlock(rcpt['blockHash'])
+        o = receipt(tx_hash)
+        r = rpc.do(o)
+        o = block_by_hash(r['block_hash'])
+        block = rpc.do(o)
         logg.debug('rcpt {}'.format(block))
         time_pair['network'] = block['timestamp']
-    except web3.exceptions.TransactionNotFound:
+    except Exception as e:
+        logg.debug('error with getting timestamp details for {}: {}'.format(tx_hash, e))
         pass
 
     otx = Otx.load(tx_hash)

apps/cic-eth/cic_eth/queue/tx.py

@@ -3,17 +3,16 @@ import logging
 import time
 import datetime
 
-# third-party imports
+# external imports
 import celery
 from hexathon import strip_0x
 from sqlalchemy import or_
 from sqlalchemy import not_
 from sqlalchemy import tuple_
 from sqlalchemy import func
+from chainlib.eth.tx import unpack
 
 # local imports
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
 from cic_eth.db.models.otx import Otx
 from cic_eth.db.models.otx import OtxStateLog
 from cic_eth.db.models.tx import TxCache
@@ -26,16 +25,17 @@ from cic_eth.db.enum import (
         is_alive,
         dead,
         )
-from cic_eth.eth.util import unpack_signed_raw_tx # TODO: should not be in same sub-path as package that imports queue.tx
+from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.error import NotLocalTxError
 from cic_eth.error import LockedError
+from cic_eth.db.enum import status_str
 
 celery_app = celery.current_app
 #logg = celery_app.log.get_default_logger()
 logg = logging.getLogger()
 
 
-def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predecessors=True, session=None):
+def create(nonce, holder_address, tx_hash, signed_tx, chain_spec, obsolete_predecessors=True, session=None):
     """Create a new transaction queue record.
 
     :param nonce: Transaction nonce
@@ -46,13 +46,13 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
     :type tx_hash: str, 0x-hex
     :param signed_tx: Signed raw transaction
     :type signed_tx: str, 0x-hex
-    :param chain_str: Chain spec string representation to create transaction for
-    :type chain_str: str
+    :param chain_spec: Chain spec to create transaction for
+    :type chain_spec: ChainSpec
     :returns: transaction hash
     :rtype: str, 0x-hash
     """
     session = SessionBase.bind_session(session)
-    lock = Lock.check_aggregate(chain_str, LockEnum.QUEUE, holder_address, session=session) 
+    lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session) 
     if lock > 0:
         SessionBase.release_session(session)
         raise LockedError(lock)
@@ -67,17 +67,26 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
     session.flush()
 
     if obsolete_predecessors:
-        # TODO: obsolete previous txs from same holder with same nonce
         q = session.query(Otx)
         q = q.join(TxCache)
         q = q.filter(Otx.nonce==nonce)
         q = q.filter(TxCache.sender==holder_address)
         q = q.filter(Otx.tx_hash!=tx_hash)
-        q = q.filter(Otx.status<=StatusEnum.SENT)
+        q = q.filter(Otx.status.op('&')(StatusBits.FINAL)==0)
 
         for otx in q.all():
             logg.info('otx {} obsoleted by {}'.format(otx.tx_hash, tx_hash))
-            otx.cancel(confirmed=False, session=session)
+            try:
+                otx.cancel(confirmed=False, session=session)
+            except TxStateChangeError as e:
+                logg.exception('obsolete fail: {}'.format(e))
+                session.close()
+                raise(e)
+            except Exception as e:
+                logg.exception('obsolete UNEXPECTED fail: {}'.format(e))
+                session.close()
+                raise(e)
+
 
     session.commit()
     SessionBase.release_session(session)
@@ -85,8 +94,52 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
     return tx_hash
 
 
+def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=None, session=None):
+    """Signs the provided transaction, and adds it to the transaction queue cache (with status PENDING).
+
+    :param tx: Standard ethereum transaction data
+    :type tx: dict
+    :param chain_spec: Chain spec of transaction to add to queue
+    :type chain_spec: chainlib.chain.ChainSpec
+    :param queue: Task queue
+    :type queue: str
+    :param cache_task: Cache task to call with signed transaction. If None, no task will be called.
+    :type cache_task: str
+    :raises: sqlalchemy.exc.DatabaseError
+    :returns: Tuple; Transaction hash, signed raw transaction data
+    :rtype: tuple
+    """
+    logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
+    tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw, chain_id=chain_spec.chain_id())
+
+    create(
+        tx['nonce'],
+        tx['from'],
+        tx_hash_hex,
+        tx_signed_raw_hex,
+        chain_spec,
+        session=session,
+    )        
+
+    if cache_task != None:
+        logg.debug('adding cache task {} tx {}'.format(cache_task, tx_hash_hex))
+        s_cache = celery.signature(
+                cache_task,
+                [
+                    tx_hash_hex,
+                    tx_signed_raw_hex,
+                    chain_spec.asdict(),
+                    ],
+                queue=queue,
+                )
+        s_cache.apply_async()
+
+    return (tx_hash_hex, tx_signed_raw_hex,)
+
+
 # TODO: Replace set_* with single task for set status
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_sent_status(tx_hash, fail=False):
     """Used to set the status after a send attempt
 
@@ -107,10 +160,20 @@ def set_sent_status(tx_hash, fail=False):
         session.close()
         return False
 
-    if fail:
-        o.sendfail(session=session)
-    else:
-        o.sent(session=session)
+    try:
+        if fail:
+            o.sendfail(session=session)
+        else:
+            o.sent(session=session)
+    except TxStateChangeError as e:
+        logg.exception('set sent fail: {}'.format(e))
+        session.close()
+        raise(e)
+    except Exception as e:
+        logg.exception('set sent UNEXPECED fail: {}'.format(e))
+        session.close()
+        raise(e)
+
 
     session.commit()
     session.close()
@@ -118,7 +181,7 @@ def set_sent_status(tx_hash, fail=False):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_final_status(tx_hash, block=None, fail=False):
     """Used to set the status of an incoming transaction result. 
 
@@ -154,10 +217,20 @@ def set_final_status(tx_hash, block=None, fail=False):
     q = q.filter(Otx.tx_hash==tx_hash)
     o = q.first()
 
-    if fail:
-        o.minefail(block, session=session)
-    else:
-        o.success(block, session=session)
+    try:
+        if fail:
+            o.minefail(block, session=session)
+        else:
+            o.success(block, session=session)
+        session.commit()
+    except TxStateChangeError as e:
+        logg.exception('set final fail: {}'.format(e))
+        session.close()
+        raise(e)
+    except Exception as e:
+        logg.exception('set final UNEXPECED fail: {}'.format(e))
+        session.close()
+        raise(e)
 
     q = session.query(Otx)
     q = q.join(TxCache)
@@ -166,15 +239,23 @@ def set_final_status(tx_hash, block=None, fail=False):
     q = q.filter(Otx.tx_hash!=tx_hash)
 
     for otwo in q.all():
-        otwo.cancel(True, session=session)
-
+        try:
+            otwo.cancel(True, session=session)
+        except TxStateChangeError as e:
+            logg.exception('cancel non-final fail: {}'.format(e))
+            session.close()
+            raise(e)
+        except Exception as e:
+            logg.exception('cancel non-final UNEXPECTED fail: {}'.format(e))
+            session.close()
+            raise(e)
     session.commit()
     session.close()
 
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_cancel(tx_hash, manual=False):
     """Used to set the status when a transaction is cancelled.
 
@@ -195,18 +276,22 @@ def set_cancel(tx_hash, manual=False):
 
     session.flush()
 
-    if manual:
-        o.override(session=session)
-    else:
-        o.cancel(session=session)
-
-    session.commit()
+    try:
+        if manual:
+            o.override(session=session)
+        else:
+            o.cancel(session=session)
+        session.commit()
+    except TxStateChangeError as e:
+        logg.exception('set cancel fail: {}'.format(e))
+    except Exception as e:
+        logg.exception('set cancel UNEXPECTED fail: {}'.format(e))
     session.close()
 
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_rejected(tx_hash):
     """Used to set the status when the node rejects sending a transaction to network
 
@@ -232,7 +317,7 @@ def set_rejected(tx_hash):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_fubar(tx_hash):
     """Used to set the status when an unexpected error occurs.
 
@@ -258,7 +343,7 @@ def set_fubar(tx_hash):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_manual(tx_hash):
     """Used to set the status when queue is manually changed
 
@@ -284,7 +369,7 @@ def set_manual(tx_hash):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_ready(tx_hash):
     """Used to mark a transaction as ready to be sent to network
 
@@ -310,7 +395,27 @@ def set_ready(tx_hash):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def set_dequeue(tx_hash):
+    session = SessionBase.create_session()
+    q = session.query(Otx)
+    q = q.filter(Otx.tx_hash==tx_hash)
+    o = q.first()
+    if o == None:
+        session.close()
+        raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
+
+    session.flush()
+
+    o.dequeue(session=session)
+    session.commit()
+    session.close()
+
+    return tx_hash
+
+
+
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def set_waitforgas(tx_hash):
     """Used to set the status when a transaction must be deferred due to gas refill
 
@@ -336,7 +441,7 @@ def set_waitforgas(tx_hash):
     return tx_hash
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_state_log(tx_hash):
 
     logs = []
@@ -355,7 +460,7 @@ def get_state_log(tx_hash):
     return logs
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx_cache(tx_hash):
     """Returns an aggregate dictionary of outgoing transaction data and metadata
 
@@ -386,7 +491,7 @@ def get_tx_cache(tx_hash):
         'tx_hash': otx.tx_hash,
         'signed_tx': otx.signed_tx,
         'nonce': otx.nonce,
-        'status': StatusEnum(otx.status).name,
+        'status': status_str(otx.status),
         'status_code': otx.status,
         'source_token': txc.source_token_address,
         'destination_token': txc.destination_token_address,
@@ -404,7 +509,7 @@ def get_tx_cache(tx_hash):
     return tx
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_lock(address=None):
     """Retrieve all active locks
 
@@ -442,7 +547,7 @@ def get_lock(address=None):
     return locks
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx(tx_hash):
     """Retrieve a transaction queue record by transaction hash
 
@@ -453,7 +558,9 @@ def get_tx(tx_hash):
     :rtype: dict
     """
     session = SessionBase.create_session()
-    tx = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
+    q = session.query(Otx)
+    q = q.filter(Otx.tx_hash==tx_hash)
+    tx = q.first()
     if tx == None:
         session.close()
         raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
@@ -469,7 +576,7 @@ def get_tx(tx_hash):
     return o
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_nonce_tx(nonce, sender, chain_id):
     """Retrieve all transactions for address with specified nonce
 
@@ -489,7 +596,7 @@ def get_nonce_tx(nonce, sender, chain_id):
     txs = {}
     for r in q.all():
         tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
-        tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
+        tx = unpack(tx_signed_bytes, chain_id)
         if sender == None or tx['from'] == sender:
             txs[r.tx_hash] = r.signed_tx
 
@@ -534,7 +641,7 @@ def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
 
     for r in q.all():
         tx_signed_bytes = bytes.fromhex(r.signed_tx[2:])
-        tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
+        tx = unpack(tx_signed_bytes, chain_id)
         if sender == None or tx['from'] == sender:
             #gas += tx['gas'] * tx['gasPrice']
             txs[r.tx_hash] = r.signed_tx
@@ -544,7 +651,7 @@ def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
     return txs
 
 
-def get_status_tx(status, before=None, exact=False, limit=0, session=None):
+def get_status_tx(status, not_status=None, before=None, exact=False, limit=0, session=None):
     """Retrieve transaction with a specific queue status.
 
     :param status: Status to match transactions with
@@ -560,11 +667,15 @@ def get_status_tx(status, before=None, exact=False, limit=0, session=None):
     session = SessionBase.bind_session(session)
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.date_updated<before)
+   #     before = datetime.datetime.utcnow()
+    if before != None:
+        q = q.filter(TxCache.date_updated<before)
     if exact:
-        q = q.filter(Otx.status==status.value)
+        q = q.filter(Otx.status==status)
     else:
-        q = q.filter(Otx.status.op('&')(status.value)==status.value)
+        q = q.filter(Otx.status.op('&')(status)>0)
+        if not_status != None:
+            q = q.filter(Otx.status.op('&')(not_status)==0)
     i = 0
     for o in q.all():
         if limit > 0 and i == limit:
@@ -636,7 +747,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
             continue
 
         tx_signed_bytes = bytes.fromhex(o.signed_tx[2:])
-        tx = unpack_signed_raw_tx(tx_signed_bytes, chain_id)
+        tx = unpack(tx_signed_bytes, chain_id)
         txs[o.tx_hash] = o.signed_tx
         
         q = session.query(TxCache)
@@ -652,7 +763,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
     return txs
 
 
-@celery_app.task()
+@celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None):
     """Returns all local queue transactions for a given Ethereum address
 

apps/cic-eth/cic_eth/runnable/create.py

@@ -1,16 +1,19 @@
 #!/usr/bin/python
-#import socket
 import sys
 import os
 import logging
 import uuid
 import json
-
-import celery
-from cic_eth.api import Api
-import confini
 import argparse
+
+# external imports
+import celery
+import confini
 import redis
+from xdg.BaseDirectory import xdg_config_home
+
+# local imports
+from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger('create_account_script')
@@ -49,6 +52,7 @@ args_override = {
         'REDIS_DB': getattr(args, 'redis_db'),
         }
 config.dict_override(args_override, 'cli')
+
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -3,34 +3,28 @@ import argparse
 import sys
 import os
 import logging
-import re
 
 # third-party imports
 import confini
 import celery
-import web3
-from cic_registry.chain import ChainSpec
-from cic_registry import zero_address
+from chainlib.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.address import is_checksum_address
 
 # local imports
 from cic_eth.api import AdminApi
-from cic_eth.eth.rpc import RpcClient
 from cic_eth.db.enum import LockEnum
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-logging.getLogger('web3').setLevel(logging.WARNING)
-logging.getLogger('urllib3').setLevel(logging.WARNING)
-
-
-default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
 argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
-argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format')
+argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -40,7 +34,7 @@ argparser.add_argument('-vv', help='be more verbose', action='store_true')
 
 def process_lock_args(argparser):
     argparser.add_argument('flags', type=str, help='Flags to manipulate')
-    argparser.add_argument('address', default=zero_address, nargs='?', type=str, help='Ethereum address to unlock,')
+    argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,')
 
 sub = argparser.add_subparsers()
 sub.dest = "command"
@@ -69,30 +63,12 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = web3.Web3.WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = web3.Web3.HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-
-
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 queue = args.q
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-chain_str = str(chain_spec)
-c = RpcClient(chain_spec)
-admin_api = AdminApi(c)
+admin_api = AdminApi(None)
 
 
 def lock_names_to_flag(s):
@@ -108,14 +84,14 @@ def lock_names_to_flag(s):
 def main():
     if args.command == 'unlock':
         flags = lock_names_to_flag(args.flags)
-        if not web3.Web3.isChecksumAddress(args.address):
+        if not is_checksum_address(args.address):
             raise ValueError('Invalid checksum address {}'.format(args.address))
 
         s = celery.signature(
             'cic_eth.admin.ctrl.unlock',
             [
                 None,
-                chain_str,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],
@@ -127,14 +103,14 @@ def main():
 
     if args.command == 'lock':
         flags = lock_names_to_flag(args.flags)
-        if not web3.Web3.isChecksumAddress(args.address):
+        if not is_checksum_address(args.address):
             raise ValueError('Invalid checksum address {}'.format(args.address))
 
         s = celery.signature(
             'cic_eth.admin.ctrl.lock',
             [
                 None,
-                chain_str,
+                chain_spec.asdict(),
                 args.address,
                 flags,
                 ],

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -11,38 +11,43 @@ import datetime
 # third-party imports
 import confini
 import celery
-import web3
-from web3 import HTTPProvider, WebsocketProvider
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
+from cic_eth_registry import CICRegistry
+from chainlib.chain import ChainSpec
+from chainlib.eth.tx import unpack
+from chainlib.connection import RPCConnection
+from chainsyncer.error import SyncDone
+from hexathon import strip_0x
 
 # local imports
 import cic_eth
-from cic_eth.eth import RpcClient
 from cic_eth.db import SessionBase
 from cic_eth.db.enum import StatusEnum
+from cic_eth.db.enum import StatusBits
 from cic_eth.db.enum import LockEnum
 from cic_eth.db import dsn_from_config
-from cic_eth.queue.tx import get_upcoming_tx
+from cic_eth.queue.tx import (
+        get_upcoming_tx,
+        set_dequeue,
+        )
 from cic_eth.admin.ctrl import lock_send
-from cic_eth.sync.error import LoopDone
 from cic_eth.eth.tx import send as task_tx_send
-from cic_eth.error import PermanentTxError
-from cic_eth.error import TemporaryTxError
-from cic_eth.eth.util import unpack_signed_raw_tx_hex
+from cic_eth.error import (
+        PermanentTxError,
+        TemporaryTxError,
+        NotLocalTxError,
+        )
+#from cic_eth.eth.util import unpack_signed_raw_tx_hex
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
-logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
-logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
 
 
 config_dir = os.path.join('/usr/local/etc/cic-eth')
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider')
 argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
 argparser.add_argument('-v', help='be verbose', action='store_true')
@@ -59,6 +64,11 @@ os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
 config.process()
 # override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        }
+config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@@ -70,25 +80,12 @@ queue = args.q
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
+RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
 
 run = True
 
-
 class DispatchSyncer:
 
     yield_delay = 0.005
@@ -108,13 +105,20 @@ class DispatchSyncer:
         chain_str = str(self.chain_spec)
         for k in txs.keys():
             tx_raw = txs[k]
-            tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id())
+            tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
+            tx = unpack(tx_raw_bytes, self.chain_spec.chain_id())
+            
+            try:
+                set_dequeue(tx['hash'])
+            except NotLocalTxError as e:
+                logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
+                continue
 
             s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
                     [tx_raw],
-                    chain_str,
+                    self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     tx['from'],
                     ],
@@ -123,18 +127,19 @@ class DispatchSyncer:
             s_send = celery.signature(
                     'cic_eth.eth.tx.send',
                     [
-                        chain_str,      
+                        self.chain_spec.asdict(),
                         ], 
                     queue=queue,
                     )
             s_check.link(s_send)
             t = s_check.apply_async()
+            logg.info('processed {}'.format(k))
 
 
     def loop(self, w3, interval):
         while run:
             txs = {}
-            typ = StatusEnum.READYSEND
+            typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(typ, chain_id=self.chain_id)
             for k in utxs.keys():
                 txs[k] = utxs[k]
@@ -147,17 +152,11 @@ class DispatchSyncer:
 
 
 def main(): 
-
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-    c = RpcClient(chain_spec)
-
-    CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-
     syncer = DispatchSyncer(chain_spec)
+    conn = RPCConnection.connect(chain_spec, 'default')
     try:
-        syncer.loop(c.w3, float(config.get('DISPATCHER_LOOP_INTERVAL')))
-    except LoopDone as e:
+        syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    except SyncDone as e:
         sys.stderr.write("dispatcher done at block {}\n".format(e))
 
     sys.exit(0)

apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py

@@ -2,3 +2,4 @@ from .callback import CallbackFilter
 from .tx import TxFilter
 from .gas import GasFilter
 from .register import RegistrationFilter
+from .transferauth import TransferAuthFilter

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -2,40 +2,81 @@
 import logging
 
 # third-party imports
-import web3
 import celery
-from cic_registry.error import UnknownContractError
+from cic_eth_registry.error import UnknownContractError
+from chainlib.status import Status as TxStatus
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.error import RequestMismatchException
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.erc20 import ERC20
+from hexathon import strip_0x
 
 # local imports
 from .base import SyncFilter
-from cic_eth.eth.token import unpack_transfer
-from cic_eth.eth.token import unpack_transferfrom
-from cic_eth.eth.token import ExtendedTx
-from .base import SyncFilter
+from cic_eth.eth.meta import ExtendedTx
 
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
-transfer_method_signature = '0xa9059cbb' # keccak256(transfer(address,uint256))
-transferfrom_method_signature = '0x23b872dd' # keccak256(transferFrom(address,address,uint256))
-giveto_method_signature = '0x63e4bff4' # keccak256(giveTo(address))
+
+def parse_transfer(tx):
+    r = ERC20.parse_transfer_request(tx.payload)
+    transfer_data = {}
+    transfer_data['to'] = r[0]
+    transfer_data['value'] = r[1]
+    transfer_data['from'] = tx['from']
+    transfer_data['token_address'] = tx['to']
+    return ('transfer', transfer_data)
+
+
+def parse_transferfrom(tx):
+    r = ERC20.parse_transfer_request(tx.payload)
+    transfer_data = unpack_transferfrom(tx.payload)
+    transfer_data['from'] = r[0]
+    transfer_data['to'] = r[1]
+    transfer_data['value'] = r[2]
+    transfer_data['token_address'] = tx['to']
+    return ('transferfrom', transfer_data)
+
+
+def parse_giftto(tx):
+    # TODO: broken
+    logg.error('broken')
+    return
+    transfer_data = unpack_gift(tx.payload)
+    transfer_data['from'] = tx.inputs[0]
+    transfer_data['value'] = 0
+    transfer_data['token_address'] = ZERO_ADDRESS
+    # TODO: would be better to query the gift amount from the block state
+    for l in tx.logs:
+        topics = l['topics']
+        logg.debug('topixx {}'.format(topics))
+        if strip_0x(topics[0]) == '45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
+            #transfer_data['value'] = web3.Web3.toInt(hexstr=strip_0x(l['data']))
+            transfer_data['value'] = int.from_bytes(bytes.fromhex(strip_0x(l_data)))
+            #token_address_bytes = topics[2][32-20:]
+            token_address = strip_0x(topics[2])[64-40:]
+            transfer_data['token_address'] = to_checksum_address(token_address)
+    return ('tokengift', transfer_data)
 
 
 class CallbackFilter(SyncFilter):
 
     trusted_addresses = []
 
-    def __init__(self, method, queue):
+    def __init__(self, chain_spec, method, queue):
         self.queue = queue
         self.method = method
+        self.chain_spec = chain_spec
 
 
     def call_back(self, transfer_type, result):
+        logg.debug('result {}'.format(result))
         s = celery.signature(
             self.method,
             [
                 result,
                 transfer_type,
-                int(rcpt.status == 0),
+                int(result['status_code'] == 0),
             ],
             queue=self.queue,
             )
@@ -50,58 +91,70 @@ class CallbackFilter(SyncFilter):
 #            )
 #        s_translate.link(s)
 #        s_translate.apply_async()
-        s.apply_async()
+        t = s.apply_async()
+        return s
 
 
-    def parse_data(self, tx, rcpt):
-        transfer_type = 'transfer'
+    def parse_data(self, tx):
+        transfer_type = None
         transfer_data = None
-        method_signature = tx.input[:10]
+        # TODO: what's with the mix of attributes and dict keys
+        logg.debug('have payload {}'.format(tx.payload))
+        method_signature = tx.payload[:8]
 
-        if method_signature == transfer_method_signature:
-            transfer_data = unpack_transfer(tx.input)
-            transfer_data['from'] = tx['from']
-            transfer_data['token_address'] = tx['to']
+        logg.debug('tx status {}'.format(tx.status))
 
-        elif method_signature == transferfrom_method_signature:
-            transfer_type = 'transferfrom'
-            transfer_data = unpack_transferfrom(tx.input)
-            transfer_data['token_address'] = tx['to']
+        for parser in [
+                parse_transfer,
+                parse_transferfrom,
+                parse_giftto,
+                ]:
+            try:
+                (transfer_type, transfer_data) = parser(tx)
+                break
+            except RequestMismatchException:
+                continue
 
-        # TODO: do not rely on logs here
-        elif method_signature == giveto_method_signature:
-            transfer_type = 'tokengift'
-            transfer_data = unpack_gift(tx.input)
-            for l in rcpt.logs:
-                if l.topics[0].hex() == '0x45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
-                    transfer_data['value'] = web3.Web3.toInt(hexstr=l.data)
-                    token_address_bytes = l.topics[2][32-20:]
-                    transfer_data['token_address'] = web3.Web3.toChecksumAddress(token_address_bytes.hex())
-                    transfer_data['from'] = rcpt.to
+
+        logg.debug('resolved method {}'.format(transfer_type))
+
+        if transfer_data != None:
+            transfer_data['status'] = tx.status
 
         return (transfer_type, transfer_data)
 
 
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        logg.debug('applying callback filter "{}:{}"'.format(self.queue, self.method))
-        chain_str = str(chain_spec)
-
-        transfer_data = self.parse_data(tx, rcpt)
-
+    def filter(self, conn, block, tx, db_session=None):
         transfer_data = None
-        if len(tx.input) < 10:
-            logg.debug('callbacks filter data length not sufficient for method signature in tx {}, skipping'.format(tx['hash']))
+        transfer_type = None
+        try:
+            (transfer_type, transfer_data) = self.parse_data(tx)
+        except TypeError:
+            logg.debug('invalid method data length for tx {}'.format(tx.hash))
             return
 
-        logg.debug('checking callbacks filter input {}'.format(tx.input[:10]))
+        if len(tx.payload) < 8:
+            logg.debug('callbacks filter data length not sufficient for method signature in tx {}, skipping'.format(tx.hash))
+            return
+
+        logg.debug('checking callbacks filter input {}'.format(tx.payload[:8]))
 
         if transfer_data != None:
             token_symbol = None
             result = None
             try:
-                tokentx = ExtendedTx(self.chain_spec)
+                tokentx = ExtendedTx(conn, tx.hash, self.chain_spec)
                 tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses)
                 tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
-                self.call_back(tokentx.to_dict())
+                if transfer_data['status'] == 0:
+                    tokentx.set_status(1)
+                else:
+                    tokentx.set_status(0)
+                t = self.call_back(transfer_type, tokentx.to_dict())
+                logg.info('callback success task id {} tx {}'.format(t, tx.hash))
             except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc.queue, tc.method, transfer_data['to'], tx.hash.hex()))
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc.queue, tc.method, transfer_data['to'], tx.hash))
+
+
+    def __str__(self):
+        return 'cic-eth callbacks'

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -1,8 +1,8 @@
 # standard imports
 import logging
 
-# third-party imports
-from cic_registry.chain import ChainSpec
+# external imports
+from hexathon import add_0x
 
 # local imports
 from cic_eth.db.enum import StatusBits
@@ -10,23 +10,22 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.otx import Otx
 from cic_eth.queue.tx import get_paused_txs
-from cic_eth.eth.task import create_check_gas_and_send_task
+from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 
 class GasFilter(SyncFilter):
 
-    def __init__(self, gas_provider, queue=None):
+    def __init__(self, chain_spec, queue=None):
         self.queue = queue
-        self.gas_provider = gas_provider
+        self.chain_spec = chain_spec
 
 
-    def filter(self, w3, tx, rcpt, chain_str, session=None):
-        logg.debug('applying gas filter')
-        tx_hash_hex = tx.hash.hex()
-        if tx['value'] > 0:
+    def filter(self, conn, block, tx, session):
+        tx_hash_hex = add_0x(tx.hash)
+        if tx.value > 0:
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
             session = SessionBase.bind_session(session)
             q = session.query(TxCache.recipient)
@@ -35,23 +34,26 @@ class GasFilter(SyncFilter):
             r = q.first()
 
             if r == None:
-                logg.warning('unsolicited gas refill tx {}'.format(tx_hash_hex))
+                logg.debug('unsolicited gas refill tx {}'.format(tx_hash_hex))
                 SessionBase.release_session(session)
                 return
 
-            chain_spec = ChainSpec.from_chain_str(chain_str)
-            txs = get_paused_txs(StatusBits.GAS_ISSUES, r[0], chain_spec.chain_id(), session=session)
+            txs = get_paused_txs(StatusBits.GAS_ISSUES, r[0], self.chain_spec.chain_id(), session=session)
 
             SessionBase.release_session(session)
 
+            logg.info('resuming gas-in-waiting txs for {}'.format(r[0]))
             if len(txs) > 0:
-                logg.info('resuming gas-in-waiting txs for {}: {}'.format(r[0], txs.keys()))
-                s = create_check_gas_and_send_task(
+                s = create_check_gas_task(
                         list(txs.values()),
-                        str(chain_str),
+                        self.chain_spec,
                         r[0],
                         0,
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
                 s.apply_async()
+
+
+    def __str__(self):
+        return 'eic-eth gasfilter'

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -3,37 +3,55 @@ import logging
 
 # third-party imports
 import celery
-from chainlib.eth.address import to_checksum
+from chainlib.eth.address import to_checksum_address
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
-account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd' # keccak256(AccountAdded(address,uint256))
+account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, queue):
+    def __init__(self, chain_spec, queue):
+        self.chain_spec = chain_spec
         self.queue = queue
 
 
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        logg.debug('applying registration filter')
+    def filter(self, conn, block, tx, db_session=None): 
         registered_address = None
-        for l in rcpt['logs']:
-            event_topic_hex = l['topics'][0].hex()
+        for l in tx.logs:
+            event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:
-                address_bytes = l.topics[1][32-20:]
-                address = to_checksum(address_bytes.hex())
-                logg.debug('request token gift to {}'.format(address))
-                s = celery.signature(
-                    'cic_eth.eth.account.gift',
+                # TODO: use abi conversion method instead
+
+                address_hex = strip_0x(l['topics'][1])[64-40:]
+                address = to_checksum_address(add_0x(address_hex))
+                logg.info('request token gift to {}'.format(address))
+                s_nonce = celery.signature(
+                    'cic_eth.eth.tx.reserve_nonce',
                     [
                         address,
-                        str(chain_spec),
                         ],
                     queue=self.queue,
                     )
-                s.apply_async()
+                s_gift = celery.signature(
+                    'cic_eth.eth.account.gift',
+                    [
+                        self.chain_spec.asdict(),
+                        ],
+                    queue=self.queue,
+                    )
+                s_nonce.link(s_gift)
+                s_nonce.apply_async()
+
+
+    def __str__(self):
+        return 'cic-eth account registration'
+

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -0,0 +1,87 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.contract import (
+        ABIContractType,
+        abi_decode_single,
+        )
+from cic_eth_registry import CICRegistry
+from erc20_transfer_authorization import TransferAuthorization
+
+# local imports
+from .base import SyncFilter
+
+
+logg = logging.getLogger(__name__)
+
+
+class TransferAuthFilter(SyncFilter):
+
+    def __init__(self, registry, chain_spec, conn, queue=None, call_address=ZERO_ADDRESS):
+        self.queue = queue
+        self.chain_spec = chain_spec
+        registry = CICRegistry(chain_spec, conn)
+        self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
+
+
+    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
+
+        if tx.payload == None:
+            logg.debug('no payload')
+            return False
+
+        payloadlength = len(tx.payload)
+        if payloadlength != 8+256:
+            logg.debug('{} below minimum length for a transfer auth call'.format(payloadlength))
+            logg.debug('payload {}'.format(tx.payload))
+            return False
+
+        recipient = tx.inputs[0]
+        if recipient != self.transfer_request_contract.address():
+            logg.debug('not our transfer auth contract address {}'.format(recipient))
+            return False
+
+        r = TransferAuthorization.parse_create_request_request(tx.payload) 
+           
+        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
+        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
+        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
+        value = abi_decode_single(ABIContractType.UINT256, r[3])
+
+        token_data = {
+            'address': token,
+            }
+
+        s_nonce = celery.signature(
+            'cic_eth.eth.tx.reserve_nonce',
+            [
+                [token_data],
+                sender,
+                ],
+            queue=self.queue,
+            )
+        s_approve = celery.signature(
+            'cic_eth.eth.token.approve',
+            [
+                sender,
+                recipient,
+                value,
+                self.chain_spec.asdict(),
+                ],
+            queue=self.queue,
+            )
+        s_nonce.link(s_approve)
+        t = s_nonce.apply_async()
+        return True
+
+
+    def __str__(self):
+        return 'cic-eth transfer auth filter'

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -3,39 +3,48 @@ import logging
 
 # third-party imports
 import celery
+from hexathon import (
+        add_0x,
+        )
 
 # local imports
 from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.base import SessionBase
+from chainsyncer.db.models.base import SessionBase
+from chainlib.status import Status
 from .base import SyncFilter
 
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 
 class TxFilter(SyncFilter):
 
-    def __init__(self, queue):
+    def __init__(self, chain_spec, queue):
         self.queue = queue
+        self.chain_spec = chain_spec
 
 
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        session = SessionBase.bind_session(session)
-        logg.debug('applying tx filter')
-        tx_hash_hex = tx.hash.hex()
-        otx = Otx.load(tx_hash_hex, session=session)
-        SessionBase.release_session(session)
+    def filter(self, conn, block, tx, db_session=None):
+        db_session = SessionBase.bind_session(db_session)
+        tx_hash_hex = tx.hash
+        otx = Otx.load(add_0x(tx_hash_hex), session=db_session)
         if otx == None:
             logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
             return None
-        logg.info('otx found {}'.format(otx.tx_hash))
+        logg.info('tx filter match on {}'.format(otx.tx_hash))
+        db_session.flush()
+        SessionBase.release_session(db_session)
         s = celery.signature(
                 'cic_eth.queue.tx.set_final_status',
                 [
-                    tx_hash_hex,
-                    rcpt.blockNumber,
-                    rcpt.status == 0,
+                    add_0x(tx_hash_hex),
+                    tx.block.number,
+                    tx.status == Status.ERROR,
                     ],
                 queue=self.queue,
                 )
         t = s.apply_async()
         return t
+
+
+    def __str__(self):
+        return 'cic-eth erc20 transfer filter'

apps/cic-eth/cic_eth/runnable/daemons/manager.py

@@ -1,207 +0,0 @@
-# standard imports
-import os
-import sys
-import logging
-import time
-import argparse
-import sys
-import re
-
-# third-party imports
-import confini
-import celery
-import rlp
-import web3
-from web3 import HTTPProvider, WebsocketProvider
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-from cic_registry import zero_address
-from cic_registry.chain import ChainRegistry
-from cic_registry.error import UnknownContractError
-from cic_bancor.bancor import BancorRegistryClient
-
-# local imports
-import cic_eth
-from cic_eth.eth import RpcClient
-from cic_eth.db import SessionBase
-from cic_eth.db import Otx
-from cic_eth.db import TxConvertTransfer
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.enum import StatusEnum
-from cic_eth.db import dsn_from_config
-from cic_eth.queue.tx import get_paused_txs
-from cic_eth.sync import Syncer
-from cic_eth.sync.error import LoopDone
-from cic_eth.db.error import UnknownConvertError
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.sync.backend import SyncerBackend
-from cic_eth.eth.token import unpack_transfer
-from cic_eth.eth.token import unpack_transferfrom
-from cic_eth.eth.account import unpack_gift
-from cic_eth.runnable.daemons.filters import (
-        CallbackFilter,
-        GasFilter,
-        TxFilter,
-        RegistrationFilter,
-        )
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
-logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
-
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-argparser.add_argument('mode', type=str, help='sync mode: (head|history)', default='head')
-args = argparser.parse_args(sys.argv[1:])
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-# override args
-args_override = {
-        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-
-queue = args.q
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-
-c = RpcClient(chain_spec)
-CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-chain_registry = ChainRegistry(chain_spec)
-CICRegistry.add_chain_registry(chain_registry, True)
-
-declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
-
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=1, debug=config.true('DATABASE_DEBUG'))
-
-
-def main():
-    global chain_spec, c, queue
-
-    if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
-        CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
-
-    syncers = []
-    block_offset = c.w3.eth.blockNumber
-    chain = str(chain_spec)
-
-    if SyncerBackend.first(chain):
-        from cic_eth.sync.history import HistorySyncer
-        backend = SyncerBackend.initial(chain, block_offset)
-        syncer = HistorySyncer(backend) 
-        syncers.append(syncer)
-
-    if args.mode == 'head':
-        from cic_eth.sync.head import HeadSyncer
-        block_sync = SyncerBackend.live(chain, block_offset+1)
-        syncers.append(HeadSyncer(block_sync))
-    elif args.mode == 'history':
-        from cic_eth.sync.history import HistorySyncer
-        backends = SyncerBackend.resume(chain, block_offset+1)
-        for backend in backends:
-            syncers.append(HistorySyncer(backend))
-        if len(syncers) == 0:
-            logg.info('found no unsynced history. terminating')
-            sys.exit(0)
-    else:
-        sys.stderr.write("unknown mode '{}'\n".format(args.mode))
-        sys.exit(1)
-
-#    bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
-#    bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
-#    bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
-#    bancor_registry.load() 
- 
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-    CallbackFilter.trusted_addresses = trusted_addresses
-
-    callback_filters = []
-    for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
-        task_split = cb.split(':')
-        task_queue = queue
-        if len(task_split) > 1:
-            task_queue = task_split[0]
-        callback_filter = CallbackFilter(task_split[1], task_queue)
-        callback_filters.append(callback_filter)
-
-    tx_filter = TxFilter(queue)
-
-    registration_filter = RegistrationFilter(queue)
-
-    gas_filter = GasFilter(c.gas_provider(), queue)
-
-    i = 0
-    for syncer in syncers:
-        logg.debug('running syncer index {}'.format(i))
-        syncer.filter.append(gas_filter.filter)
-        syncer.filter.append(registration_filter.filter)
-        # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
-        syncer.filter.append(tx_filter.filter)
-        #syncer.filter.append(convert_filter)
-        for cf in callback_filters:
-            syncer.filter.append(cf.filter)
-
-        try:
-            syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')))
-        except LoopDone as e:
-            sys.stderr.write("sync '{}' done at block {}\n".format(args.mode, e))
-
-        i += 1
-
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

apps/cic-eth/cic_eth/runnable/daemons/retry.py

@@ -8,9 +8,8 @@ import datetime
 import web3
 import confini
 import celery
-from web3 import HTTPProvider, WebsocketProvider
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
+from cic_eth_registry import CICRegistry
+from chainlib.chain import ChainSpec
 
 from cic_eth.db import dsn_from_config
 from cic_eth.db import SessionBase
@@ -25,19 +24,14 @@ from cic_eth.eth.util import unpack_signed_raw_tx_hex
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
-logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
-logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
-
 
 config_dir = os.path.join('/usr/local/etc/cic-eth')
 
 argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
 argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--retry-delay', dest='retry_delay', type=str, help='seconds to wait for retrying a transaction that is marked as sent')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
 argparser.add_argument('-v', help='be verbose', action='store_true')
@@ -56,6 +50,7 @@ config = confini.Config(config_dir, args.env_prefix)
 config.process()
 # override args
 args_override = {
+        'ETH_PROVIDER': getattr(args, 'p'),
         'ETH_ABI_DIR': getattr(args, 'abi_dir'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'),
@@ -71,31 +66,15 @@ queue = args.q
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
+RPCConnection.registry_location(args.p, chain_spec, tag='default')
+
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn)
 
-
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-
-
 straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
 
 # TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here
-def sendfail_filter(w3, tx_hash, rcpt, chain_str):
-    chain_spec = ChainSpec.from_chain_str(chain_str)
+def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
     tx_dict = get_tx(tx_hash)
     tx = unpack_signed_raw_tx_hex(tx_dict['signed_tx'], chain_spec.chain_id())
     logg.debug('submitting tx {} for retry'.format(tx_hash))
@@ -137,8 +116,8 @@ def sendfail_filter(w3, tx_hash, rcpt, chain_str):
 
 
 # TODO: can we merely use the dispatcher instead?
-def dispatch(chain_str):
-    txs = get_status_tx(StatusEnum.RETRY, datetime.datetime.utcnow())
+def dispatch(conn, chain_spec):
+    txs = get_status_tx(StatusEnum.RETRY, before=datetime.datetime.utcnow())
     if len(txs) == 0:
         logg.debug('no retry state txs found')
         return
@@ -199,11 +178,49 @@ def dispatch(chain_str):
 #    s_send.apply_async()
 
 
-def main(): 
+class RetrySyncer(Syncer):
 
-    c = RpcClient(chain_spec)
-    CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
+    def __init__(self, chain_spec, stalled_grace_seconds, failed_grace_seconds=None, final_func=None):
+        self.chain_spec = chain_spec
+        if failed_grace_seconds == None:
+            failed_grace_seconds = stalled_grace_seconds
+        self.stalled_grace_seconds = stalled_grace_seconds
+        self.failed_grace_seconds = failed_grace_seconds
+        self.final_func = final_func
+
+
+    def get(self):
+#            before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.failed_grace_seconds)
+#            failed_txs = get_status_tx(
+#                    StatusEnum.SENDFAIL.value,
+#                    before=before,
+#                    )
+            before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.stalled_grace_seconds)
+            stalled_txs = get_status_tx(
+                    StatusBits.IN_NETWORK.value,
+                    not_status=StatusBits.FINAL | StatusBits.MANUAL | StatusBits.OBSOLETE,
+                    before=before,
+                    )
+       #     return list(failed_txs.keys()) + list(stalled_txs.keys())
+            return stalled_txs
+
+    def process(self, conn, ref):
+        logg.debug('tx {}'.format(ref))
+        for f in self.filter:
+            f(conn, ref, None, str(self.chain_spec))
+
+
+
+    def loop(self, interval):
+        while self.running and Syncer.running_global:
+            rpc = RPCConnection.connect(self.chain_spec, 'default')
+            for tx in self.get():
+                self.process(rpc, tx)
+            if self.final_func != None:
+                self.final_func(rpc, self.chain_spec)
+            time.sleep(interval)
+
+def main(): 
 
     syncer = RetrySyncer(chain_spec, straggler_delay, final_func=dispatch)
     syncer.filter.append(sendfail_filter)

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -55,62 +55,25 @@ SessionBase.connect(dsn)
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
 queue = args.q
 
-re_transfer_approval_request = r'^/transferrequest/?'
+re_something = r'^/something/?'
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 
-def process_transfer_approval_request(session, env):
-    r = re.match(re_transfer_approval_request, env.get('PATH_INFO'))
+def process_something(session, env):
+    r = re.match(re_something, env.get('PATH_INFO'))
     if not r:
         return None
 
-    if env.get('CONTENT_TYPE') != 'application/json':
-        raise AttributeError('content type')
+    #if env.get('CONTENT_TYPE') != 'application/json':
+    #    raise AttributeError('content type')
 
-    if env.get('REQUEST_METHOD') != 'POST':
-        raise AttributeError('method')
+    #if env.get('REQUEST_METHOD') != 'POST':
+    #    raise AttributeError('method')
 
-    post_data = json.load(env.get('wsgi.input'))
-    token_address = web3.Web3.toChecksumAddress(post_data['token_address'])
-    holder_address = web3.Web3.toChecksumAddress(post_data['holder_address'])
-    beneficiary_address = web3.Web3.toChecksumAddress(post_data['beneficiary_address'])
-    value = int(post_data['value'])
-
-    logg.debug('transfer approval request token {} to {} from {} value {}'.format(
-        token_address,
-        beneficiary_address,
-        holder_address,
-        value,
-        )
-        )
-
-    s = celery.signature(
-        'cic_eth.eth.request.transfer_approval_request',
-        [
-            [
-                {
-                    'address': token_address,
-                    },
-                ],
-            holder_address,
-            beneficiary_address,
-            value,
-            config.get('CIC_CHAIN_SPEC'),
-            ],
-        queue=queue,
-     )
-    t = s.apply_async()
-    r = t.get()
-    tx_raw_bytes = bytes.fromhex(r[0][2:])
-    tx = unpack_signed_raw_tx(tx_raw_bytes, chain_spec.chain_id())
-    for r in t.collect():
-        logg.debug('result {}'.format(r))
-
-    if not t.successful():
-        raise RuntimeError(tx['hash'])
-
-    return ('text/plain', tx['hash'].encode('utf-8'),)
+    #post_data = json.load(env.get('wsgi.input'))
+    
+    #return ('text/plain', 'foo'.encode('utf-8'),)
 
 
 # uwsgi application
@@ -125,7 +88,7 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
-            process_transfer_approval_request,
+            process_something,
             ]:
         try:
             r = handler(session, env)

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -8,29 +8,21 @@ import re
 import urllib
 import websocket
 
-# third-party imports
+# external imports
 import celery
 import confini
-from crypto_dev_signer.eth.web3ext import Web3 as Web3Ext
-from web3 import HTTPProvider, WebsocketProvider
-from gas_proxy.web3 import GasMiddleware
+from chainlib.connection import RPCConnection
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.chain import ChainSpec
 
 # local imports
-from cic_registry.registry import CICRegistry
-from cic_registry.registry import ChainRegistry
-from cic_registry.registry import ChainSpec
-from cic_registry.helper.declarator import DeclaratorOracleAdapter
+from cic_eth_registry import CICRegistry
 
-from cic_bancor.bancor import BancorRegistryClient
-from cic_eth.eth import bancor
-from cic_eth.eth import token
+from cic_eth.eth import erc20
 from cic_eth.eth import tx
 from cic_eth.eth import account
-from cic_eth.eth import request
 from cic_eth.admin import debug
 from cic_eth.admin import ctrl
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.eth.rpc import GasOracle
 from cic_eth.queue import tx
 from cic_eth.queue import balance
 from cic_eth.callbacks import Callback
@@ -40,6 +32,7 @@ from cic_eth.callbacks import redis
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.otx import Otx
 from cic_eth.db import dsn_from_config
+from cic_eth.ext import tx
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -47,7 +40,7 @@ logg = logging.getLogger()
 config_dir = os.path.join('/usr/local/etc/cic-eth')
 
 argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', type=str, help='web3 provider')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
@@ -68,12 +61,12 @@ config = confini.Config(args.c, args.env_prefix)
 config.process()
 # override args
 args_override = {
-        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
+config.add(args.q, '_CELERY_QUEUE', True)
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
@@ -81,7 +74,7 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=8, debug=config.true('DATABASE_DEBUG'))
+SessionBase.connect(dsn, pool_size=50, debug=config.true('DATABASE_DEBUG'))
 
 # verify database connection with minimal sanity query
 session = SessionBase.create_session()
@@ -122,68 +115,14 @@ else:
         'result_backend': result,
         })
 
-
-# set up web3
-# TODO: web3 socket wrapping is now a lot of code. factor out
-class JSONRPCHttpSocketAdapter:
-
-    def __init__(self, url):
-        self.response = None
-        self.url = url
-
-    def send(self, data):
-        logg.debug('redirecting socket send to jsonrpc http socket adapter {} {}'.format(self.url, data))
-        req = urllib.request.Request(self.url, method='POST')
-        req.add_header('Content-type', 'application/json')
-        req.add_header('Connection', 'close')
-        res = urllib.request.urlopen(req, data=data.encode('utf-8'))
-        self.response = res.read().decode('utf-8')
-        logg.debug('setting jsonrpc http socket adapter response to {}'.format(self.response))
-
-    def recv(self, n=0):
-        return self.response
-
-
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-socket_constructor = None
-if re.match(re_websocket, blockchain_provider) != None:
-    def socket_constructor_ws():
-        return websocket.create_connection(config.get('ETH_PROVIDER'))
-    socket_constructor = socket_constructor_ws
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    def socket_constructor_http():
-        return JSONRPCHttpSocketAdapter(config.get('ETH_PROVIDER'))
-    socket_constructor = socket_constructor_http
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-
-def web3ext_constructor():
-    w3 = Web3Ext(blockchain_provider, config.get('SIGNER_SOCKET_PATH'))
-    GasMiddleware.socket_constructor = socket_constructor
-    w3.middleware_onion.add(GasMiddleware)
-
-    def sign_transaction(tx):
-        r = w3.eth.signTransaction(tx)
-        d = r.__dict__
-        for k in d.keys():
-            if k == 'tx':
-                d[k] = d[k].__dict__ 
-            else:
-                d[k] = d[k].hex()
-        return d
-
-    setattr(w3.eth, 'sign_transaction', sign_transaction)
-    setattr(w3.eth, 'send_raw_transaction', w3.eth.sendRawTransaction)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3ext_constructor) 
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
+CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
+
 
 def main():
     argv = ['worker']
@@ -196,33 +135,19 @@ def main():
     argv.append('-n')
     argv.append(args.q)
 
-    if config.true('SSL_ENABLE_CLIENT'):
-        Callback.ssl = True
-        Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
-        Callback.ssl_key_file = config.get('SSL_KEY_FILE')
-        Callback.ssl_password = config.get('SSL_PASSWORD')
+#    if config.true('SSL_ENABLE_CLIENT'):
+#        Callback.ssl = True
+#        Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
+#        Callback.ssl_key_file = config.get('SSL_KEY_FILE')
+#        Callback.ssl_password = config.get('SSL_PASSWORD')
+#
+#    if config.get('SSL_CA_FILE') != '':
+#        Callback.ssl_ca_file = config.get('SSL_CA_FILE')
 
-    if config.get('SSL_CA_FILE') != '':
-        Callback.ssl_ca_file = config.get('SSL_CA_FILE')
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+    registry_address = registry.by_name('ContractRegistry')
 
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-    c = RpcClient(chain_spec)
-    CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-
-    chain_registry = ChainRegistry(chain_spec)
-    CICRegistry.add_chain_registry(chain_registry, True)
-    try:
-        CICRegistry.get_contract(chain_spec, 'CICRegistry')
-    except Exception as e:
-        logg.exception('Eek, registry failure is baaad juju {}'.format(e))
-        sys.exit(1)
-
-    if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
-        CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
-
-    declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -230,15 +155,7 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
-    oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
-    chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
-
-
-    #chain_spec = CICRegistry.default_chain_spec
-    #bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
-    #bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
-    #bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
-    #bancor_registry.load(True)
+    
     current_app.worker_main(argv)
 
 

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -0,0 +1,150 @@
+# standard imports
+import os
+import sys
+import logging
+import time
+import argparse
+import sys
+import re
+
+# third-party imports
+import confini
+import celery
+import rlp
+import cic_base.config
+import cic_base.log
+import cic_base.argparse
+import cic_base.rpc
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.error import UnknownContractError
+from chainlib.chain import ChainSpec
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.connection import RPCConnection
+from chainlib.eth.block import (
+        block_latest,
+        )
+from hexathon import (
+        strip_0x,
+        )
+from chainsyncer.backend import SyncerBackend
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
+from chainsyncer.db.models.base import SessionBase
+
+# local imports
+from cic_eth.db import dsn_from_config
+from cic_eth.runnable.daemons.filters import (
+        CallbackFilter,
+        GasFilter,
+        TxFilter,
+        RegistrationFilter,
+        TransferAuthFilter,
+        )
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+
+logg = cic_base.log.create()
+argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
+#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+args = cic_base.argparse.parse(argparser, logg)
+config = cic_base.config.create(args.c, args, args.env_prefix)
+
+config.add(args.y, '_KEYSTORE_FILE', True)
+
+config.add(args.q, '_CELERY_QUEUE', True)
+
+cic_base.config.log(config)
+
+dsn = dsn_from_config(config)
+
+SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
+cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+
+
+def main():
+    # connect to celery
+    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+    # Connect to blockchain with chainlib
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    o = block_latest()
+    r = rpc.do(o)
+    block_offset = int(strip_0x(r), 16) + 1
+
+    logg.debug('starting at block {}'.format(block_offset))
+
+    syncers = []
+
+    #if SyncerBackend.first(chain_spec):
+    #    backend = SyncerBackend.initial(chain_spec, block_offset)
+    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
+
+    if len(syncer_backends) == 0:
+        logg.info('found no backends to resume')
+        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
+    else:
+        for syncer_backend in syncer_backends:
+            logg.info('resuming sync session {}'.format(syncer_backend))
+
+    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
+
+    for syncer_backend in syncer_backends:
+        try:
+            syncers.append(HistorySyncer(syncer_backend))
+            logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
+        except AttributeError:
+            logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend))
+
+    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+    if trusted_addresses_src == None:
+        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+        sys.exit(1)
+    trusted_addresses = trusted_addresses_src.split(',')
+    for address in trusted_addresses:
+        logg.info('using trusted address {}'.format(address))
+    CallbackFilter.trusted_addresses = trusted_addresses
+
+    callback_filters = []
+    for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
+        task_split = cb.split(':')
+        task_queue = config.get('_CELERY_QUEUE')
+        if len(task_split) > 1:
+            task_queue = task_split[0]
+        callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
+        callback_filters.append(callback_filter)
+
+    tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
+
+    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
+
+    gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
+
+    #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
+
+    i = 0
+    for syncer in syncers:
+        logg.debug('running syncer index {}'.format(i))
+        syncer.add_filter(gas_filter)
+        syncer.add_filter(registration_filter)
+        # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
+        syncer.add_filter(tx_filter)
+        #syncer.add_filter(transfer_auth_filter)
+        for cf in callback_filters:
+            syncer.add_filter(cf)
+
+        r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
+        sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
+
+        i += 1
+
+
+if __name__ == '__main__':
+    main()

apps/cic-eth/cic_eth/runnable/resend.py

@@ -22,7 +22,7 @@ logg = logging.getLogger()
 logging.getLogger('web3').setLevel(logging.WARNING)
 logging.getLogger('urllib3').setLevel(logging.WARNING)
 
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 
 argparser = argparse.ArgumentParser()

apps/cic-eth/cic_eth/runnable/tag.py

@@ -5,14 +5,14 @@ import logging
 import argparse
 import re
 
-# third-party imports
-import web3
-from web3 import HTTPProvider, WebsocketProvider
+# external imports
+import celery
 import confini
+from chainlib.chain import ChainSpec
+from xdg.BaseDirectory import xdg_config_home
 
 # local imports
 from cic_eth.api import AdminApi
-from cic_eth.eth import RpcClient
 from cic_eth.db import dsn_from_config
 from cic_eth.db.models.base import SessionBase
 
@@ -48,29 +48,14 @@ config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}\n{}'.format(args.c, config))
 
+chain_spec = ChainSpec.from_chain_str(args.i)
 
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-c = RpcClient(config.get('CIC_CHAIN_SPEC'))
 
 def main():
-    api = AdminApi(c)
-    api.tag_account(args.tag, args.address)
+    api = AdminApi(None)
+    api.tag_account(args.tag, args.address, chain_spec)
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/view.py

@@ -11,34 +11,34 @@ import sys
 import re
 import datetime
 
-# third-party imports
+# external imports
 import confini
 import celery
-import web3
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-from cic_registry.chain import ChainRegistry
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
+from chainlib.chain import ChainSpec
+from chainlib.eth.connection import EthHTTPConnection
+from hexathon import add_0x
 
 # local imports
 from cic_eth.api import AdminApi
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.db.enum import StatusEnum
-from cic_eth.db.enum import LockEnum
+from cic_eth.db.enum import (
+    StatusEnum,
+    status_str,
+    LockEnum,
+)
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-logging.getLogger('web3').setLevel(logging.WARNING)
-logging.getLogger('urllib3').setLevel(logging.WARNING)
-
-
-default_abi_dir = '/usr/share/local/cic/solidity/abi'
-default_config_dir = os.path.join('/usr/local/etc/cic-eth')
+default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
-argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format')
+argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
+argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
+argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -61,42 +61,38 @@ config.process()
 args_override = {
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         }
 # override args
+config.dict_override(args_override, 'cli args')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = web3.Web3.WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = web3.Web3.HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-
+try:
+    config.add(add_0x(args.query), '_QUERY', True)
+except:
+    config.add(args.query, '_QUERY', True)
 
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 queue = args.q
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-chain_str = str(chain_spec)
-c = RpcClient(chain_spec)
-admin_api = AdminApi(c)
 
-CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-chain_registry = ChainRegistry(chain_spec)
-CICRegistry.add_chain_registry(chain_registry)
-CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-CICRegistry.load_for(chain_spec)
+rpc = EthHTTPConnection(args.p)
+
+registry_address = config.get('CIC_REGISTRY_ADDRESS')
+
+admin_api = AdminApi(rpc)
+
+trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+if trusted_addresses_src == None:
+    logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+    sys.exit(1)
+trusted_addresses = trusted_addresses_src.split(',')
+for address in trusted_addresses:
+    logg.info('using trusted address {}'.format(address))
 
 fmt = 'terminal'
 if args.f[:1] == 'j':
@@ -114,18 +110,22 @@ def render_tx(o, **kwargs):
     
     for v in o.get('status_log', []):
         d = datetime.datetime.fromisoformat(v[0])
-        e = StatusEnum(v[1]).name
+        e = status_str(v[1], args.status_raw)
         content += '{}: {}\n'.format(d, e)
 
     return content
 
 def render_account(o, **kwargs):
-    return '{} {} {} {}'.format(
+    s = '{} {} {} {}'.format(
             o['date_updated'],
             o['nonce'],
             o['tx_hash'],
             o['status'],
             )
+    if len(o['errors']) > 0:
+        s += ' !{}'.format(','.join(o['errors']))
+
+    return s
 
 
 def render_lock(o, **kwargs):
@@ -146,32 +146,41 @@ def render_lock(o, **kwargs):
 
     return s
 
+
+def connect_registry(registry_address, chain_spec, rpc):
+    CICRegistry.address = registry_address
+    registry = CICRegistry(chain_spec, rpc)
+    declarator_address = registry.by_name('AddressDeclarator')
+    lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses)
+    registry.add_lookup(lookup)
+    return registry
+
+
 # TODO: move each command to submodule
 def main():
-    logg.debug('len {}'.format(len(args.query)))
     txs  = []
     renderer = render_tx
-    if len(args.query) > 66:
-        txs = [admin_api.tx(chain_spec, tx_raw=args.query)]
-    elif len(args.query) > 42:
-        txs = [admin_api.tx(chain_spec, tx_hash=args.query)]
-    elif len(args.query) == 42:
-        txs = admin_api.account(chain_spec, args.query, include_recipient=False)
-        renderer = render_account
-    elif len(args.query) >= 4 and args.query[:4] == 'lock':
-        txs = admin_api.get_lock()
-        renderer = render_lock
-    else:
-        raise ValueError('cannot parse argument {}'.format(args.query))
+    if len(config.get('_QUERY')) > 66:
+        registry = connect_registry(registry_address, chain_spec, rpc)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
+    elif len(config.get('_QUERY')) > 42:
+        registry = connect_registry(registry_address, chain_spec, rpc)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
 
-    if len(txs) == 0:
-        logg.info('no matches found')
+    elif len(config.get('_QUERY')) == 42:
+        registry = connect_registry(registry_address, chain_spec, rpc)
+        txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
+        renderer = render_account
+    elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
+        t = admin_api.get_lock()
+        txs = t.get()
+        renderer = render_lock
+        for tx in txs:
+            r = renderer(txs)
+            sys.stdout.write(r + '\n')
     else:
-        if fmt == 'json':
-            sys.stdout.write(json.dumps(txs))
-        else:
-            m = map(renderer, txs)
-            print(*m, sep="\n")
-                    
+        raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
+                   
+
 if __name__ == '__main__':
     main()

apps/cic-eth/cic_eth/sync/__init__.py

@@ -1 +0,0 @@
-from .base import Syncer

apps/cic-eth/cic_eth/sync/backend.py

@@ -1,201 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from cic_eth.db.models.sync import BlockchainSync
-from cic_eth.db.models.base import SessionBase
-
-logg = logging.getLogger()
-
-
-class SyncerBackend:
-    """Interface to block and transaction sync state.
-
-    :param chain_spec: Chain spec for the chain that syncer is running for.
-    :type chain_spec: cic_registry.chain.ChainSpec
-    :param object_id: Unique id for the syncer session.
-    :type object_id: number
-    """
-    def __init__(self, chain_spec, object_id):
-        self.db_session = None
-        self.db_object = None
-        self.chain_spec = chain_spec
-        self.object_id = object_id
-        self.connect()
-        self.disconnect()
-
-
-    def connect(self):
-        """Loads the state of the syncer session with the given id.
-        """
-        if self.db_session == None:
-            self.db_session = SessionBase.create_session()
-        q = self.db_session.query(BlockchainSync)
-        q = q.filter(BlockchainSync.id==self.object_id)
-        self.db_object = q.first()
-        if self.db_object == None:
-            self.disconnect()
-            raise ValueError('sync entry with id {} not found'.format(self.object_id))
-        return self.db_session
-
-
-    def disconnect(self):
-        """Commits state of sync to backend.
-        """
-        if self.db_session != None:
-            self.db_session.add(self.db_object)
-            self.db_session.commit()
-            self.db_session.close()
-            self.db_session = None
-       
-
-    def chain(self):
-        """Returns chain spec for syncer
-
-        :returns: Chain spec
-        :rtype chain_spec: cic_registry.chain.ChainSpec
-        """
-        return self.chain_spec
-   
-
-    def get(self):
-        """Get the current state of the syncer cursor.
-
-        :returns: Block and block transaction height, respectively
-        :rtype: tuple
-        """
-        self.connect()
-        pair = self.db_object.cursor()
-        self.disconnect()
-        return pair
-   
-
-    def set(self, block_height, tx_height):
-        """Update the state of the syncer cursor
-        :param block_height: Block height of cursor
-        :type block_height: number
-        :param tx_height: Block transaction height of cursor
-        :type tx_height: number
-        :returns: Block and block transaction height, respectively
-        :rtype: tuple
-        """
-        self.connect()
-        pair = self.db_object.set(block_height, tx_height)
-        self.disconnect()
-        return pair
-
-
-    def start(self):
-        """Get the initial state of the syncer cursor.
-
-        :returns: Initial block and block transaction height, respectively
-        :rtype: tuple
-        """
-        self.connect()
-        pair = self.db_object.start()
-        self.disconnect()
-        return pair
-
-    
-    def target(self):
-        """Get the target state (upper bound of sync) of the syncer cursor.
-
-        :returns: Target block height
-        :rtype: number
-        """
-        self.connect()
-        target = self.db_object.target()
-        self.disconnect()
-        return target
-
-
-    @staticmethod
-    def first(chain):
-        """Returns the model object of the most recent syncer in backend.
-
-        :param chain: Chain spec of chain that syncer is running for.
-        :type chain: cic_registry.chain.ChainSpec
-        :returns: Last syncer object 
-        :rtype: cic_eth.db.models.BlockchainSync
-        """
-        return BlockchainSync.first(chain)
-
-
-    @staticmethod
-    def initial(chain, block_height):
-        """Creates a new syncer session and commit its initial state to backend.
-
-        :param chain: Chain spec of chain that syncer is running for.
-        :type chain: cic_registry.chain.ChainSpec
-        :param block_height: Target block height
-        :type block_height: number
-        :returns: New syncer object 
-        :rtype: cic_eth.db.models.BlockchainSync
-        """
-        object_id = None
-        session = SessionBase.create_session()
-        o = BlockchainSync(chain, 0, 0, block_height)
-        session.add(o)
-        session.commit()
-        object_id = o.id
-        session.close()
-
-        return SyncerBackend(chain, object_id)
-
-
-    @staticmethod
-    def resume(chain, block_height):
-        """Retrieves and returns all previously unfinished syncer sessions.
-
-
-        :param chain: Chain spec of chain that syncer is running for.
-        :type chain: cic_registry.chain.ChainSpec
-        :param block_height: Target block height
-        :type block_height: number
-        :returns: Syncer objects of unfinished syncs
-        :rtype: list of cic_eth.db.models.BlockchainSync
-        """
-        syncers = []
-
-        session = SessionBase.create_session()
-
-        object_id = None
-
-        for object_id in BlockchainSync.get_unsynced(session=session):
-            logg.debug('block syncer resume added previously unsynced sync entry id {}'.format(object_id))
-            syncers.append(SyncerBackend(chain, object_id))
-
-        (block_resume, tx_resume) = BlockchainSync.get_last_live_height(block_height, session=session)
-        if block_height != block_resume:
-            o = BlockchainSync(chain, block_resume, tx_resume, block_height)
-            session.add(o)
-            session.commit()
-            object_id = o.id
-            syncers.append(SyncerBackend(chain, object_id))
-            logg.debug('block syncer resume added new sync entry from previous run id {}, start{}:{} target {}'.format(object_id, block_resume, tx_resume, block_height))
-
-        session.close()
-
-        return syncers
-
-
-    @staticmethod
-    def live(chain, block_height):
-        """Creates a new open-ended syncer session starting at the given block height.
-
-        :param chain: Chain spec of chain that syncer is running for.
-        :type chain: cic_registry.chain.ChainSpec
-        :param block_height: Target block height
-        :type block_height: number
-        :returns: "Live" syncer object
-        :rtype: cic_eth.db.models.BlockchainSync
-        """
-        object_id = None
-        session = SessionBase.create_session()
-        o = BlockchainSync(chain, block_height, 0, None)
-        session.add(o)
-        session.commit()
-        object_id = o.id
-        session.close()
-
-        return SyncerBackend(chain, object_id)

apps/cic-eth/cic_eth/sync/base.py

@@ -1,51 +0,0 @@
-# TODO: extend blocksync model
-class Syncer:
-    """Base class and interface for implementing a block sync poller routine.
-
-    :param bc_cache: Retrieves block cache cursors for chain head and latest processed block.
-    :type bc_cache: cic_eth.sync.SyncerBackend
-    """
-    w3 = None
-    running_global = True
-
-    def __init__(self, bc_cache):
-        self.cursor = None
-        self.bc_cache = bc_cache
-        self.filter = []
-        self.running = True
-
-
-    def chain(self):
-        """Returns the string representation of the chain spec for the chain the syncer is running on.
-
-        :returns: Chain spec string
-        :rtype: str
-        """
-        return self.bc_cache.chain()
-
-
-    def get(self):
-        """Get latest unprocessed blocks.
-
-        :returns: list of block hash strings
-        :rtype: list
-        """
-        raise NotImplementedError()
-
-
-    def process(self, w3, ref):
-        """Process transactions in a single block.
-
-        :param ref: Reference of object to process
-        :type ref: str, 0x-hex
-        """
-        raise NotImplementedError()
-
-
-    def loop(self, interval):
-        """Entry point for syncer loop
-
-        :param interval: Delay in seconds until next attempt if no new blocks are found.
-        :type interval: int
-        """
-        raise NotImplementedError()

apps/cic-eth/cic_eth/sync/error.py

@@ -1,4 +0,0 @@
-class LoopDone(Exception):
-    """Exception raised when a syncing is complete.
-    """
-    pass

apps/cic-eth/cic_eth/sync/head.py

@@ -1,51 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import web3
-
-# local imports
-from .mined import MinedSyncer
-from .base import Syncer
-
-logg = logging.getLogger()
-
-
-class HeadSyncer(MinedSyncer):
-    """Implements the get method in Syncer for retrieving every new mined block.
-
-    :param bc_cache: Retrieves block cache cursors for chain head and latest processed block.
-    :type bc_cache: Object implementing methods from cic_eth.sync.SyncerBackend 
-    """
-    def __init__(self, bc_cache):
-        super(HeadSyncer, self).__init__(bc_cache)
-        # TODO: filter not returning all blocks, at least with ganache. kind of defeats the point, then
-        #self.w3_filter = rpc.w3.eth.filter({
-        #    'fromBlock': block_offset,
-        #    }) #'latest')
-        #self.bc_cache.set(block_offset, 0)
-        logg.debug('initialized head syncer with offset {}'.format(bc_cache.start()))
-
-    """Implements Syncer.get
-
-    :param w3: Web3 object
-    :type w3: web3.Web3
-    :returns: Block hash of newly mined blocks. if any
-    :rtype: list of str, 0x-hex
-    """
-    def get(self, w3):
-        # Of course, the filter doesn't return the same block dict format as getBlock() so we'll just waste some cycles getting the hashes instead.
-        #hashes = []
-        #for block in self.w3_filter.get_new_entries():
-        #    hashes.append(block['blockHash'])
-        #logg.debug('blocks {}'.format(hashes))
-        #return hashes
-        (block_number, tx_number) = self.bc_cache.get()
-        block_hash = []
-        try:
-            block = w3.eth.getBlock(block_number)
-            block_hash.append(block.hash)
-        except web3.exceptions.BlockNotFound:
-            pass
-
-        return block_hash

apps/cic-eth/cic_eth/sync/history.py

@@ -1,74 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-from web3.exceptions import BlockNotFound
-from .error import LoopDone
-
-# local imports
-from .mined import MinedSyncer
-from .base import Syncer
-from cic_eth.db.models.base import SessionBase
-
-logg = logging.getLogger()
-
-
-class HistorySyncer(MinedSyncer):
-    """Implements the get method in Syncer for retrieving all blocks between last processed block before previous shutdown and block height at time of syncer start.
-
-    :param bc_cache: Retrieves block cache cursors for chain head and latest processed block.
-    :type bc_cache: Object implementing methods from cic_eth.sync.SyncerBackend 
-    :param mx: Maximum number of blocks to return in one call
-    :type mx: int
-    """
-    def __init__(self, bc_cache, mx=500):
-        super(HistorySyncer, self).__init__(bc_cache)
-        self.max = mx
-
-        self.target = bc_cache.target()
-        logg.info('History syncer target block number {}'.format(self.target))
-
-        session_offset = self.bc_cache.get()
-
-        self.block_offset = session_offset[0]
-        self.tx_offset = session_offset[1]
-        logg.info('History syncer starting at {}:{}'.format(session_offset[0], session_offset[1]))
-
-        self.filter = []
-
-
-    """Implements Syncer.get
-
-    BUG: Should also raise LoopDone when block array is empty after loop.
-
-    :param w3: Web3 object
-    :type w3: web3.Web3
-    :raises LoopDone: If a block is not found.
-    :return: Return a batch of blocks to process
-    :rtype: list of str, 0x-hex
-    """
-    def get(self, w3):
-        sync_db = self.bc_cache
-        height = self.bc_cache.get()
-        logg.debug('height {}'.format(height))
-        block_last = height[0]
-        tx_last = height[1]
-        if not self.running:
-            raise LoopDone((block_last, tx_last))
-        b = []
-        block_target = block_last + self.max
-        if block_target > self.target:
-            block_target = self.target
-        logg.debug('target {} last {} max {}'.format(block_target, block_last, self.max))
-        for i in range(block_last, block_target):
-            if i == self.target:
-                logg.info('reached target {}, exiting'.format(i))
-                self.running = False
-                break
-            bhash = w3.eth.getBlock(i).hash
-            b.append(bhash)
-            logg.debug('appending block {} {}'.format(i, bhash.hex()))
-        if block_last == block_target:
-            logg.info('aleady reached target {}, exiting'.format(self.target))
-            self.running = False
-        return b

apps/cic-eth/cic_eth/sync/mempool.py

@@ -1,50 +0,0 @@
-class MemPoolSyncer(Syncer):
-
-
-    def __init__(self, bc_cache):
-        raise NotImplementedError('incomplete, needs web3 tx to raw transaction conversion')
-        super(MemPoolSyncer, self).__init__(bc_cache)
-#        self.w3_filter = Syncer.w3.eth.filter('pending')
-#        for tx in tx_cache.txs:
-#            self.txs.append(tx)
-#            logg.debug('add tx {} to mempoolsyncer'.format(tx))
-#
-#
-#    def get(self):
-#        return self.w3_filter.get_new_entries()
-#
-#
-#    def process(self, tx_hash):
-#        tx_hash_hex = tx_hash.hex()
-#        if tx_hash_hex in self.txs:
-#            logg.debug('syncer already watching {}, skipping'.format(tx_hash_hex))
-#        tx = self.w3.eth.getTransaction(tx_hash_hex)
-#        serialized_tx = rlp.encode({
-#            'nonce': tx.nonce,
-#            'from': getattr(tx, 'from'),
-#            })
-#        logg.info('add {} to syncer: {}'.format(tx, serialized_tx))
-#        otx = Otx(
-#                nonce=tx.nonce,
-#                address=getattr(tx, 'from'),
-#                tx_hash=tx_hash_hex,
-#                signed_tx=serialized_tx,
-#                )
-#        Otx.session.add(otx)
-#        Otx.session.commit()
-#
-#
-#    def loop(self, interval):
-#        while Syncer.running:
-#            logg.debug('loop execute')
-#            txs = self.get()
-#            logg.debug('got txs {}'.format(txs))
-#            for tx in txs:
-#                #block_number = self.process(block.hex())
-#                self.process(tx)
-#                #if block_number > self.bc_cache.head():
-#                #    self.bc_cache.head(block_number)
-#            time.sleep(interval)
-#        logg.info("Syncer no longer set to run, gracefully exiting")
-
-

apps/cic-eth/cic_eth/sync/mined.py

@@ -1,109 +0,0 @@
-# standard imports
-import logging
-import time
-
-# third-party imports
-import celery
-
-# local impotes
-from .base import Syncer
-from cic_eth.queue.tx import set_final_status
-from cic_eth.eth import RpcClient
-
-app = celery.current_app
-logg = logging.getLogger()
-
-
-class MinedSyncer(Syncer):
-    """Base implementation of block processor for mined blocks.
-
-    Loops through all transactions, 
-
-    :param bc_cache: Retrieves block cache cursors for chain head and latest processed block.
-    :type bc_cache: Object implementing methods from cic_eth.sync.SyncerBackend 
-    """
-
-    yield_delay = 0.005
-
-    def __init__(self, bc_cache):
-        super(MinedSyncer, self).__init__(bc_cache)
-        self.block_offset = 0
-        self.tx_offset = 0
-
-
-    def process(self, w3, ref):
-        """Processes transactions in a single block, advancing transaction (and block) cursor accordingly.
-
-        :param w3: Web3 object
-        :type w3: web3.Web3
-        :param ref: Block reference (hash) to process
-        :type ref: str, 0x-hex
-        :returns: Block number of next unprocessed block
-        :rtype: number
-        """
-        b = w3.eth.getBlock(ref)
-        c = w3.eth.getBlockTransactionCount(ref)
-        s = 0
-        if self.block_offset == b.number:
-            s = self.tx_offset
-
-        logg.debug('processing {} (blocknumber {}, count {}, offset {})'.format(ref, b.number, c, s))
-
-        for i in range(s, c):
-            tx = w3.eth.getTransactionByBlock(ref, i)
-            tx_hash_hex = tx['hash'].hex()
-            rcpt = w3.eth.getTransactionReceipt(tx_hash_hex)
-            logg.debug('{}/{} processing tx {} from block {} {}'.format(i+1, c, tx_hash_hex, b.number, ref))
-            ours = False
-            # TODO: ensure filter loop can complete on graceful shutdown
-            for f in self.filter:
-                #try:
-                session = self.bc_cache.connect()
-                task_uuid = f(w3, tx, rcpt, self.chain(), session)
-                #except Exception as e:
-                #    logg.error('error in filter {} tx {}: {}'.format(f, tx_hash_hex, e))
-                #    continue
-                if task_uuid != None:
-                    logg.debug('tx {} passed to celery task {}'.format(tx_hash_hex, task_uuid))
-                    s = celery.signature(
-                            'set_final_status',
-                            [tx_hash_hex, rcpt['blockNumber'], not rcpt['status']],
-                            )
-                    s.apply_async()
-                    break
-            next_tx = i + 1
-            if next_tx == c:
-                self.bc_cache.set(b.number+1, 0)
-            else:
-                self.bc_cache.set(b.number, next_tx)
-        if c == 0:
-            logg.info('synced block {} has no transactions'.format(b.number))
-            #self.bc_cache.session(b.number+1, 0)
-            self.bc_cache.set(b.number+1, 0)
-        return b['number']
-
-
-
-    def loop(self, interval):
-        """Loop running until the "running" property of Syncer is set to False.
-
-        Retrieves latest unprocessed blocks and processes them.
-
-        :param interval: Delay in seconds until next attempt if no new blocks are found.
-        :type interval: int
-        """
-        while self.running and Syncer.running_global:
-            self.bc_cache.connect()
-            c = RpcClient(self.chain())
-            logg.debug('loop execute')
-            e = self.get(c.w3)
-            logg.debug('got blocks {}'.format(e))
-            for block in e:
-                block_number = self.process(c.w3, block.hex())
-                logg.info('processed block {} {}'.format(block_number, block.hex()))
-            self.bc_cache.disconnect()
-            if len(e) > 0:
-                time.sleep(self.yield_delay)
-            else:
-                time.sleep(interval)
-        logg.info("Syncer no longer set to run, gracefully exiting")

apps/cic-eth/cic_eth/sync/retry.py

@@ -1,71 +0,0 @@
-# standard imports
-import logging
-import datetime
-import time
-
-# third-party imports
-import celery
-
-# local imports
-from .base import Syncer
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.db.enum import StatusEnum
-from cic_eth.queue.tx import get_status_tx
-
-logg = logging.getLogger()
-
-celery_app = celery.current_app
-
-
-class noop_cache:
-
-    def __init__(self, chain_spec):
-        self.chain_spec = chain_spec
-
-
-    def chain(self):
-        return self.chain_spec
-
-
-class RetrySyncer(Syncer):
-
-    def __init__(self, chain_spec, stalled_grace_seconds, failed_grace_seconds=None, final_func=None):
-        cache = noop_cache(chain_spec)
-        super(RetrySyncer, self).__init__(cache)
-        if failed_grace_seconds == None:
-            failed_grace_seconds = stalled_grace_seconds
-        self.stalled_grace_seconds = stalled_grace_seconds
-        self.failed_grace_seconds = failed_grace_seconds
-        self.final_func = final_func
-
-
-    def get(self, w3):
-#            before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.failed_grace_seconds)
-#            failed_txs = get_status_tx(
-#                    StatusEnum.SENDFAIL.value,
-#                    before=before,
-#                    )
-            before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.stalled_grace_seconds)
-            stalled_txs = get_status_tx(
-                    StatusEnum.SENT.value,
-                    before=before,
-                    )
-       #     return list(failed_txs.keys()) + list(stalled_txs.keys())
-            return stalled_txs
-
-
-    def process(self, w3, ref):
-        logg.debug('tx {}'.format(ref))
-        for f in self.filter:
-            f(w3, ref, None, str(self.chain()))
-
-
-    def loop(self, interval):
-        chain_str = str(self.chain())
-        while self.running and Syncer.running_global:
-            c = RpcClient(self.chain())
-            for tx in self.get(c.w3):
-                self.process(c.w3, tx)
-            if self.final_func != None:
-                self.final_func(chain_str)
-            time.sleep(interval)

apps/cic-eth/cic_eth/task.py

@@ -0,0 +1,96 @@
+# import
+import time
+import requests
+import logging
+import uuid
+
+# external imports
+import celery
+import sqlalchemy
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import RPCGasOracle
+
+# local imports
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
+from cic_eth.db.models.base import SessionBase
+
+logg = logging.getLogger(__name__)
+
+celery_app = celery.current_app
+
+
+class BaseTask(celery.Task):
+
+    session_func = SessionBase.create_session
+    call_address = ZERO_ADDRESS
+    create_nonce_oracle = RPCNonceOracle
+    create_gas_oracle = RPCGasOracle
+
+    def create_session(self):
+        return BaseTask.session_func()
+
+
+    def log_banner(self):
+        logg.debug('task {} root uuid {}'.format(self.__class__.__name__, self.request.root_id))
+        return
+
+    
+class CriticalTask(BaseTask):
+    retry_jitter = True
+    retry_backoff = True
+    retry_backoff_max = 8
+
+
+class CriticalSQLAlchemyTask(CriticalTask):
+    autoretry_for = (
+        sqlalchemy.exc.DatabaseError,
+        sqlalchemy.exc.TimeoutError,
+        sqlalchemy.exc.ResourceClosedError,
+        ) 
+
+
+class CriticalWeb3Task(CriticalTask):
+    autoretry_for = (
+        requests.exceptions.ConnectionError,
+        )
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+
+
+class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
+    autoretry_for = (
+        sqlalchemy.exc.DatabaseError,
+        sqlalchemy.exc.TimeoutError,
+        requests.exceptions.ConnectionError,
+        sqlalchemy.exc.ResourceClosedError,
+        EthError,
+        )
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+
+
+class CriticalSQLAlchemyAndSignerTask(CriticalTask):
+     autoretry_for = (
+        sqlalchemy.exc.DatabaseError,
+        sqlalchemy.exc.TimeoutError,
+        sqlalchemy.exc.ResourceClosedError,
+        SignerError,
+        ) 
+
+class CriticalWeb3AndSignerTask(CriticalTask):
+    autoretry_for = (
+        requests.exceptions.ConnectionError,
+        SignerError,
+        )
+    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def hello(self):
+    time.sleep(0.1)
+    return id(SessionBase.create_session)

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         10,
-        0,
-        'alpha.30',
+        1,
+        'beta.1',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -1,5 +1,5 @@
 [cic]
 registry_address = 
-chain_spec = 
+chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =

apps/cic-eth/config/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=
+DEBUG=0

apps/cic-eth/config/docker/database.ini

@@ -6,4 +6,4 @@ HOST=localhost
 PORT=63432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=1
+DEBUG=0

apps/cic-eth/config/tasks.ini

@@ -1,3 +1,3 @@
 [tasks]
 transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
-trace_queue_status = 
+trace_queue_status = 1

apps/cic-eth/crypto_dev_signer_config/config.ini

@@ -1,4 +1,4 @@
 [signer]
 secret = deadbeef
 #database = crypto-dev-signer
-socket_path = /run/crypto-dev-signer/jsonrpc.ipc
+socket_path = ipc:///run/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/docker/Dockerfile

@@ -16,12 +16,20 @@ ARG root_requirement_file='requirements.txt'
 #RUN apk add linux-headers
 #RUN apk add libffi-dev
 RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
 # Copy shared requirements from top of mono-repo
 RUN echo "copying root req file ${root_requirement_file}"
-COPY $root_requirement_file .
-RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
+#COPY $root_requirement_file .
+#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
+RUN /usr/local/bin/python -m pip install --upgrade pip
+#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
+#	cd cic-base && \
+#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
+#	python merge_requirements.py | tee merged_requirements.txt 
+#RUN cd cic-base && \
+#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./
@@ -42,7 +50,6 @@ COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
-RUN apt-get install -y git && \
-	git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
 	mkdir -p /usr/local/share/cic/solidity && \
 	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi

apps/cic-eth/docker/db.sh

@@ -2,5 +2,5 @@
 
 set -e
 >&2 echo executing database migration
-migrate.py -c /usr/local/etc/cic-eth --migrations-dir /usr/local/share/cic-eth/alembic -vv
+python scripts/migrate.py -c /usr/local/etc/cic-eth --migrations-dir /usr/local/share/cic-eth/alembic -vv
 set +e

apps/cic-eth/docker/start_manager.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-. ./db.sh
-
-/usr/local/bin/cic-eth-managerd $@

apps/cic-eth/docker/start_tasker.sh

@@ -6,7 +6,7 @@ set -e
 # set CONFINI_ENV_PREFIX to override the env prefix to override env vars
 
 echo "!!! starting signer"
-python /usr/local/bin/crypto-dev-daemon -vv -c /usr/local/etc/crypto-dev-signer &
+python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
 echo "!!! starting tracker"
 /usr/local/bin/cic-eth-taskerd $@

apps/cic-eth/docker/start_tracker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. ./db.sh
+
+/usr/local/bin/cic-eth-trackerd $@

apps/cic-eth/requirements.txt

@@ -1,22 +1,24 @@
-web3==5.12.2
+cic-base~=0.1.2a46
 celery==4.4.7
-crypto-dev-signer~=0.4.13rc2
-confini~=0.3.6b1
-cic-registry~=0.5.3a20
-cic-bancor~=0.0.6
+crypto-dev-signer~=0.4.14a16
+confini~=0.3.6rc3
+cic-eth-registry~=0.5.4a7
+#cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.10a10
-erc20-approval-escrow~=0.3.0a5
-erc20-single-shot-faucet~=0.2.0a6
-rlp==2.0.1
+eth_accounts_index~=0.0.11a3
+erc20-transfer-authorization~=0.3.1a2
+#simple-rlp==0.1.2
 uWSGI==2.0.19.1
 semver==2.13.0
-eth-gas-proxy==0.0.1a4
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.0a8
-chainlib~=0.0.1a16
-hexathon~=0.0.1a3
+eth-address-index~=0.1.1a5
+chainlib~=0.0.1a42
+hexathon~=0.0.1a7
+chainsyncer~=0.0.1a20
+pysha3==1.0.2
+coincurve==15.0.0
+sarafu-faucet==0.0.2a13

apps/cic-eth/setup.cfg

@@ -32,7 +32,6 @@ packages =
 	cic_eth.db
 	cic_eth.db.models
 	cic_eth.queue
-	cic_eth.sync
 	cic_eth.ext
 	cic_eth.runnable
 	cic_eth.runnable.daemons
@@ -45,7 +44,7 @@ scripts =
 console_scripts =
 	# daemons
 	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main
-	cic-eth-managerd = cic_eth.runnable.daemons.manager:main
+	cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main
 	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main
 	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main
 	# tools	

apps/cic-eth/test_requirements.txt

@@ -1,7 +1,7 @@
 pytest==6.0.1
 pytest-celery==0.0.0a1
 pytest-mock==3.3.1
-py-eth==0.1.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
+giftable-erc20-token==0.0.8a4

apps/cic-eth/tests/conftest.py

@@ -1,47 +1,30 @@
 # standard imports
 import os
-import logging
 import sys
+import logging
 
-# third-party imports
-import pytest
-from cic_registry import CICRegistry
+# local imports
+from cic_eth.api import Api
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
-data_dir = os.path.join(script_dir, 'testdata', 'abi')
-CICRegistry.add_path(data_dir)
 
-# fixtures
-from tests.fixtures_registry import *
-from cic_registry.pytest import *
-from cic_bancor.pytest import *
+# assemble fixtures
 from tests.fixtures_config import *
-from tests.fixtures_celery import *
-from tests.fixtures_web3 import *
 from tests.fixtures_database import *
-from tests.fixtures_faucet import *
-from tests.fixtures_transferapproval import *
-from tests.fixtures_account import *
-
-logg = logging.getLogger()
-
-
-@pytest.fixture(scope='session')
-def init_registry(
-        init_w3_conn,
-        ):
-    return CICRegistry
+from tests.fixtures_celery import *
+from tests.fixtures_role import *
+from chainlib.eth.pytest import *
+from eth_contract_registry.pytest import *
+from cic_eth_registry.pytest.fixtures_contracts import *
+from cic_eth_registry.pytest.fixtures_tokens import *
 
 
 @pytest.fixture(scope='function')
-def eth_empty_accounts(
-        init_wallet_extension,
-        ):
-    a = []
-    for i in range(10):
-        address = init_wallet_extension.new_account()
-        a.append(address)
-        logg.info('added address {}'.format(a))
-    return a
+def api(
+    default_chain_spec,
+    custodial_roles,
+    ):
+    chain_str = str(default_chain_spec)
+    return Api(chain_str, queue=None, callback_param='foo')

apps/cic-eth/tests/fixtures_account.py

@@ -1,30 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import pytest
-from eth_accounts_index import AccountRegistry
-from cic_registry import CICRegistry
-
-logg = logging.getLogger(__name__)
-
-
-@pytest.fixture(scope='session')
-def accounts_registry(
-        default_chain_spec,
-        cic_registry,
-        w3,
-        ):
-    abi = AccountRegistry.abi()
-    constructor = w3.eth.contract(abi=abi, bytecode=AccountRegistry.bytecode())
-    tx_hash = constructor.constructor().transact()
-    r = w3.eth.getTransactionReceipt(tx_hash) 
-    logg.debug('accounts registry deployed {}'.format(r.contractAddress))
-    account_registry = AccountRegistry(w3, r.contractAddress)
-
-    c = w3.eth.contract(abi=abi, address=r.contractAddress)
-    c.functions.addWriter(w3.eth.accounts[0]).transact()
-
-    CICRegistry.add_contract(default_chain_spec, c, 'AccountRegistry')
-
-    return account_registry

apps/cic-eth/tests/fixtures_bancor.py

@@ -1,234 +0,0 @@
-# standard imports
-import os
-import logging
-import json
-
-# third-party imports
-import pytest
-from cic_registry.bancor import contract_ids
-from cic_registry import bancor
-
-# local imports
-from cic_eth.eth import rpc
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-
-logg = logging.getLogger(__file__)
-
-
-class BancorContractLoader:
-
-    bancor_path = os.path.join(root_dir, 'bancor')
-    registry_contract = None
-
-    @staticmethod
-    def build_path():
-        return BancorContractLoader.bancor_path
-#        return os.path.join(BancorContractLoader.bancor_path, 'solidity', 'build', 'contracts')
-
-
-    @staticmethod
-    def contract(w3, bundle_id, registry_id=None):
-        if registry_id == None:
-            registry_id = bundle_id
-        contract_id_hex = w3.toHex(text=registry_id)
-        contract_address = BancorContractLoader.registry_contract.functions.addressOf(contract_id_hex).call()
-        contract_build_file = os.path.join(
-                BancorContractLoader.build_path(),
-                '{}.json'.format(bundle_id),
-                )
-        f = open(os.path.join(contract_build_file))
-        j = json.load(f)
-        f.close()
-        contract_abi = j['abi']
-        logg.debug('creating contract interface {} ({}) at address {}'.format(registry_id, bundle_id, contract_address))
-        contract = w3.eth.contract(abi=contract_abi, address=contract_address)
-        return contract
-
-
-
-# TODO: DRY
-@pytest.fixture(scope='session')
-def bancor_deploy(
-        load_config,
-        init_w3_conn,
-        ):
-    bancor_dir_default = os.path.join(root_dir, 'bancor')
-    logg.debug('bancor deploy "{}"'.format(bancor_dir_default))
-    BancorContractLoader.bancor_path = load_config.get('BANCOR_DIR', bancor_dir_default)
-    bancor_build_dir = BancorContractLoader.build_path()
-
-    # deploy registry
-    registry_build_file = os.path.join(bancor_build_dir, 'ContractRegistry.json')
-    f = open(os.path.join(registry_build_file))
-    j = json.load(f)
-    f.close()
-    registry_constructor = init_w3_conn.eth.contract(abi=j['abi'], bytecode=j['bytecode'])
-    tx = registry_constructor.constructor().transact()
-    rcpt = init_w3_conn.eth.getTransactionReceipt(tx)
-    registry_address = rcpt['contractAddress']
-    registry_contract = init_w3_conn.eth.contract(abi=j['abi'], address=registry_address)
-    BancorContractLoader.registry_contract = registry_contract
-
-    # deply reserve token
-    reservetoken_build_file = os.path.join(bancor_build_dir, 'EtherToken.json')
-    f = open(os.path.join(reservetoken_build_file))
-    j = json.load(f)
-    f.close()
-    reservetoken_constructor = init_w3_conn.eth.contract(abi=j['abi'], bytecode=j['bytecode'])
-    tx = reservetoken_constructor.constructor('Reserve', 'RSV').transact()
-    rcpt = init_w3_conn.eth.getTransactionReceipt(tx)
-    reservetoken_address = rcpt['contractAddress']
-    reservetoken_contract = init_w3_conn.eth.contract(abi=j['abi'], address=reservetoken_address)
-
-    # register reserve token as bancor hub token
-    key_hex = init_w3_conn.toHex(text='BNTToken')
-    registry_contract.functions.registerAddress(key_hex, reservetoken_address).transact()
-
-    # deposit balances for minting liquid tokens with reserve
-    init_w3_conn.eth.sendTransaction({
-        'from': init_w3_conn.eth.accounts[1],
-        'to': reservetoken_address,
-        'value': init_w3_conn.toWei('101', 'ether'),
-        'nonce': 0,
-        })
-    init_w3_conn.eth.sendTransaction({
-        'from': init_w3_conn.eth.accounts[2],
-        'to': reservetoken_address,
-        'value': init_w3_conn.toWei('101', 'ether'),
-        'nonce': 0,
-        })
-
-    # deploy converter factory contract for creating liquid token exchanges
-    build_file = os.path.join(bancor_build_dir, 'LiquidTokenConverterFactory.json')
-    f = open(build_file)
-    j = json.load(f)
-    f.close()
-    converterfactory_constructor = init_w3_conn.eth.contract(abi=j['abi'], bytecode=j['bytecode'])
-    tx = converterfactory_constructor.constructor().transact()
-    rcpt = init_w3_conn.eth.getTransactionReceipt(tx)
-    converter_factory_address = rcpt['contractAddress']
-
-    # deploy the remaining contracts managed by the registry
-    for k in contract_ids.keys():
-        build_file = os.path.join(bancor_build_dir, '{}.json'.format(k))
-        f = open(build_file)
-        j = json.load(f)
-        f.close()
-        contract_constructor = init_w3_conn.eth.contract(abi=j['abi'], bytecode=j['bytecode'])
-        tx = None
-
-        # include the registry address as constructor parameters for the contracts that require it
-        if k in ['ConverterRegistry', 'ConverterRegistryData', 'BancorNetwork', 'ConversionPathFinder']:
-            tx = contract_constructor.constructor(registry_address).transact()
-        else:
-            tx = contract_constructor.constructor().transact()
-        rcpt = init_w3_conn.eth.getTransactionReceipt(tx)
-        contract_address = rcpt['contractAddress']
-
-        # register contract in registry
-        key_hex = init_w3_conn.toHex(text=contract_ids[k])
-        registry_contract.functions.registerAddress(key_hex, contract_address).transact()
-        contract = init_w3_conn.eth.contract(abi=j['abi'], address=contract_address)
-
-        # bancor formula needs to be initialized before use
-        if k == 'BancorFormula':
-            logg.debug('init bancor formula {}'.format(contract_address))
-            contract.functions.init().transact()
-
-        # converter factory needs liquid token converter factory to be able to issue our liquid tokens
-        if k == 'ConverterFactory':
-            logg.debug('register converter factory {}'.format(converter_factory_address))
-            contract.functions.registerTypedConverterFactory(converter_factory_address).transact()
-    
-    logg.info('deployed registry at address {}'.format(registry_address))
-    return registry_contract
-
-
-
-def __create_converter(w3, converterregistry_contract, reserve_address, owner_address, token_name, token_symbol):
-    converterregistry_contract.functions.newConverter(
-            0, 
-            token_name,
-            token_symbol,
-            18,
-            100000,
-            [reserve_address],
-            [250000],
-            ).transact({
-                'from': owner_address,
-                })
-
-
-@pytest.fixture(scope='session')
-def tokens_to_deploy(
-        ):
-    return [
-        (1, 'Bert Token', 'BRT'), # account_index, token name, token symbol
-        (2, 'Ernie Token', 'RNI'),
-            ]
-
-
-@pytest.fixture(scope='session')
-def bancor_tokens(
-    init_w3_conn,
-    bancor_deploy,
-    tokens_to_deploy,
-    ):
-
-    registry_contract = bancor_deploy
-
-    reserve_contract = BancorContractLoader.contract(init_w3_conn, 'ERC20Token', 'BNTToken')
-    reserve_address = reserve_contract.address
-
-    network_id = init_w3_conn.toHex(text='BancorNetwork')
-    network_address = registry_contract.functions.addressOf(network_id).call()
-
-    converterregistry_contract = BancorContractLoader.contract(init_w3_conn, 'ConverterRegistry', 'BancorConverterRegistry')
-
-    for p in tokens_to_deploy:
-        __create_converter(init_w3_conn, converterregistry_contract, reserve_address, init_w3_conn.eth.accounts[p[0]], p[1], p[2])
-
-    tokens = converterregistry_contract.functions.getAnchors().call()
-
-    network_contract = BancorContractLoader.contract(init_w3_conn, 'BancorNetwork')
-
-    mint_amount = init_w3_conn.toWei('100', 'ether')
-    i = 0
-    for token in tokens:
-        i += 1
-        owner = init_w3_conn.eth.accounts[i]
-        logg.debug('owner {} is {}'.format(owner, token))
-        reserve_contract.functions.approve(network_address, 0).transact({
-            'from': owner
-            })
-        reserve_contract.functions.approve(network_address, mint_amount).transact({
-            'from': owner
-            })
-        logg.debug('convert {} {} {} {}'.format(reserve_address, token, mint_amount, owner))
-        network_contract.functions.convert([
-            reserve_address,
-            token,
-            token,
-            ],
-            mint_amount,
-            mint_amount,
-            ).transact({
-                'from': owner,
-                })
-    
-    return tokens
-
-
-@pytest.fixture(scope='session')
-def bancor_load(
-        load_config,
-        init_w3_conn,
-        bancor_deploy,
-        bancor_tokens,
-        ):
-    registry_address = bancor_deploy.address
-    bancor_dir_default = os.path.join(root_dir, 'bancor')
-    bancor_dir = load_config.get('BANCOR_DIR', bancor_dir_default)
-    bancor.load(init_w3_conn, registry_address, bancor_dir)

apps/cic-eth/tests/fixtures_celery.py

@@ -1,25 +1,36 @@
-# third-party imports
+# external imports
 import pytest
 import tempfile
 import logging
 import shutil
 
-logg = logging.getLogger(__name__)
+# local impors
+from cic_eth.task import BaseTask
+
+#logg = logging.getLogger(__name__)
+logg = logging.getLogger()
+
+
+@pytest.fixture(scope='function')
+def init_celery_tasks(
+    contract_roles, 
+        ):
+    BaseTask.call_address = contract_roles['DEFAULT']
 
 
 # celery fixtures
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
-        'cic_eth.eth.bancor',
-        'cic_eth.eth.token',
-        'cic_eth.eth.request',
+#        'cic_eth.eth.bancor',
+        'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
         'cic_eth.queue.tx',
         'cic_eth.queue.balance',
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
+        'cic_eth.admin.debug',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
@@ -52,7 +63,7 @@ def celery_config():
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {
-#            'queues': ('cic-eth'),
+#            'queues': ('celery'),
             }
 
 @pytest.fixture(scope='session')

apps/cic-eth/tests/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, load_config.get('DATABASE_DEBUG') != None)
+    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
     return dsn
 
 

apps/cic-eth/tests/fixtures_faucet.py

@@ -1,74 +0,0 @@
-# third-party imports
-import pytest
-from cic_registry.pytest import *
-from erc20_single_shot_faucet import Faucet
-from cic_registry import zero_address
-
-
-@pytest.fixture(scope='session')
-def faucet_amount():
-    return 50
-
-
-@pytest.fixture(scope='session')
-def faucet(
-        faucet_amount,
-        config,
-        default_chain_spec,
-        cic_registry,
-        bancor_tokens,
-        w3_account_roles,
-        w3_account_token_owners,
-        solidity_abis,
-        w3,
-        #accounts_registry,
-        ):
-
-
-    abi = Faucet.abi('storage')
-    bytecode = Faucet.bytecode('storage')
-
-    cs = w3.eth.contract(abi=abi, bytecode=bytecode)
-    tx_hash = cs.constructor().transact({'from': w3_account_roles['eth_account_faucet_owner']})
-    rcpt = w3.eth.getTransactionReceipt(tx_hash)
-    cs_address = rcpt.contractAddress
-        
-    abi = Faucet.abi()
-    bytecode = Faucet.bytecode()
-    cf = w3.eth.contract(abi=abi, bytecode=bytecode)
-
-    tx_hash = cf.constructor(
-        [
-            w3_account_roles['eth_account_faucet_owner']
-            ],
-        bancor_tokens[0],
-        cs_address,
-        zero_address,
-        #accounts_registry,
-        ).transact({
-            'from': w3_account_roles['eth_account_faucet_owner']
-            }
-            )
-
-    rcpt = w3.eth.getTransactionReceipt(tx_hash)
-    cf_address = rcpt.contractAddress
-    
-    c = w3.eth.contract(abi=abi, address=cf_address)
-    c.functions.setAmount(50).transact({
-            'from': w3_account_roles['eth_account_faucet_owner']
-        }
-        )
-
-    logg.debug('foo {} bar {}'.format(cf_address, w3_account_roles))
-
-    # fund the faucet with token balance
-    token = w3.eth.contract(abi=solidity_abis['ERC20'], address=bancor_tokens[0])
-    token_symbol = token.functions.symbol().call()
-    tx_hash = token.functions.transfer(cf_address, 100000).transact({
-        'from': w3_account_token_owners[token_symbol],
-        })
-
-    CICRegistry.add_contract(default_chain_spec, c, 'Faucet')
-
-    return cf_address
- 

apps/cic-eth/tests/fixtures_keystore.py

@@ -1,52 +0,0 @@
-# standard imports
-import os
-import json
-import logging
-
-# third-party imports
-import pytest
-from cic_eth.eth.rpc import RpcClient
-from crypto_dev_signer.keystore import ReferenceKeystore
-#from crypto_dev_signer.eth.web3ext import Web3 as Web3ext
-
-logg = logging.getLogger(__file__)
-
-
-# TODO: need mock for deterministic signatures
-# depends on mock blockchain (ganache) where private key is passed directly to this module
-@pytest.fixture(scope='session')
-def init_mock_keystore(
-        ):
-    raise NotImplementedError
-
-
-@pytest.fixture(scope='session')
-def init_keystore(
-        load_config,
-        database_engine,
-        ):
-        #symkey_hex = os.environ.get('CIC_SIGNER_SECRET')
-        symkey_hex = load_config.get('SIGNER_SECRET')
-        symkey = bytes.fromhex(symkey_hex)
-        opt = {
-                'symmetric_key': symkey,
-                }
-        k = ReferenceKeystore(database_engine, **opt)
-        k.db_session.execute('DELETE from ethereum')
-        k.db_session.commit()
-        keys_file = load_config.get('SIGNER_DEV_KEYS_PATH')
-        addresses = []
-        if keys_file:
-            logg.debug('loading keys from {}'.format(keys_file))
-            f = open(keys_file, 'r')
-            j = json.load(f)
-            f.close()
-            signer_password = load_config.get('SIGNER_PASSWORD')
-            for pk in j['private']:
-                address_hex = k.import_raw_key(bytes.fromhex(pk[2:]), signer_password)
-                addresses.append(address_hex)
-    
-        RpcClient.set_provider_address(addresses[0])
-        return addresses
-
-

apps/cic-eth/tests/fixtures_registry.py

@@ -1,50 +0,0 @@
-# standard imports
-import os
-import json
-import logging
-
-# third-party imports
-import pytest
-from eth_address_declarator import AddressDeclarator
-
-# local imports
-from cic_registry import CICRegistry
-from cic_registry import to_identifier
-from cic_registry.contract import Contract
-from cic_registry.error import ChainExistsError
-
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(__file__)
-
-
-@pytest.fixture(scope='session')
-def local_cic_registry(
-        cic_registry,
-        ):
-    path = os.path.realpath(os.path.join(script_dir, 'testdata', 'abi'))
-    CICRegistry.add_path(path)
-    return cic_registry
-
-
-@pytest.fixture(scope='function')
-def address_declarator(
-        bloxberg_config,
-        default_chain_spec,
-        default_chain_registry,
-        local_cic_registry,
-        init_rpc,
-        init_w3,
-        ):
-
-    c = init_rpc.w3.eth.contract(abi=AddressDeclarator.abi(), bytecode=AddressDeclarator.bytecode())
-    default_description = '0x{:<064s}'.format(b'test'.hex()) 
-    logg.debug('default_ {}'.format(default_description))
-    tx_hash = c.constructor(default_description).transact()
-    rcpt = init_rpc.w3.eth.getTransactionReceipt(tx_hash)
-
-    registry = init_rpc.w3.eth.contract(abi=CICRegistry.abi(), address=local_cic_registry)
-    chain_identifier = to_identifier(default_chain_registry.chain())
-    registry.functions.set(to_identifier('AddressDeclarator'), rcpt.contractAddress, chain_identifier, bloxberg_config['digest']).transact()
-
-    return rcpt.contractAddress

apps/cic-eth/tests/fixtures_role.py

@@ -0,0 +1,62 @@
+# standard imports
+import logging
+
+# external imports
+import pytest
+from hexathon import add_0x
+from chainlib.eth.address import to_checksum_address
+
+# local imports
+from cic_eth.db.models.role import AccountRole
+from cic_eth.db.models.nonce import Nonce
+
+#logg = logging.getLogger(__name__)
+# what the actual fuck, debug is not being shown even though explicitly set
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+@pytest.fixture(scope='function')
+def init_custodial(
+    contract_roles,
+    token_roles,
+    agent_roles,
+    init_database,
+    ):
+    for roles in [contract_roles, token_roles, agent_roles]:
+        for role in roles.values():
+            Nonce.init(role, session=init_database)
+
+    init_database.commit()
+
+
+@pytest.fixture(scope='function')
+def custodial_roles(
+    init_custodial,
+    contract_roles,
+    token_roles,
+    agent_roles,
+    eth_accounts,
+    eth_keystore,
+    init_database,
+    ):
+    r = {}
+    r.update(contract_roles)
+    r.update(agent_roles)
+    r.update({
+        'GAS_GIFTER': eth_accounts[10],
+        'FOO_TOKEN_GIFTER': token_roles['FOO_TOKEN_OWNER'],
+            })
+    for k in r.keys():
+        role = AccountRole.set(k, r[k])
+        init_database.add(role)
+        logg.info('adding role {} -> {}'.format(k, r[k]))
+    init_database.commit()
+    return r
+
+
+@pytest.fixture(scope='function')
+def whoever(
+    init_eth_tester,
+    ):
+    return init_eth_tester.new_account()