Adds account management integration tests.

.gitlab-ci.yml

@@ -6,7 +6,6 @@ include:
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/cic_cache/cache.py

@@ -125,7 +125,6 @@ class DataCache(Cache):
                 'to_value': int(r['to_value']),
                 'source_token': r['source_token'],
                 'destination_token': r['destination_token'],
-                'success': r['success'],
                 'tx_type': tx_type,
             }
 

apps/cic-cache/cic_cache/db/list.py

@@ -42,7 +42,7 @@ def list_transactions_mined_with_data(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
 
     r = session.execute(s)
     return r

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -89,7 +89,7 @@ def process_transactions_all_data(session, env):
 
     offset = r[1]
     end = r[2]
-    if int(r[2]) < int(r[1]):
+    if r[2] < r[1]:
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)

apps/cic-cache/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[syncer]
-loop_interval = 1

apps/cic-eth/.coveragerc

@@ -5,5 +5,3 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
-	cic_eth/queue/state.py
-	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -5,29 +5,18 @@
 
 .cic_eth_changes_target:
     rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      #changes:
-          #- $CONTEXT/$APP_NAME/**/*
-      when: always
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-eth:
     extends:
-        - .cic_eth_variables
         - .cic_eth_changes_target
-        - .py_build_target_test
-
-test-mr-cic-eth:
-    extends:
+        - .py_build_merge_request
         - .cic_eth_variables
-        - .cic_eth_changes_target
-    stage: test
-    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
-    script:
-        - cd apps/$APP_NAME/
-        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
+        
+

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,18 +4,11 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
-from chainlib.eth.tx import (
-        unpack,
-        TxFactory,
-        )
-from chainlib.eth.gas import OverrideGasOracle
-from chainqueue.sql.query import get_tx
-from chainqueue.sql.state import set_cancel
+from chainlib.eth.tx import unpack
+from chainqueue.query import get_tx
+from chainqueue.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import strip_0x
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -28,14 +21,13 @@ from cic_eth.admin.ctrl import (
         )
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
-from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True, base=BaseTask)
-def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
+@celery_app.task(bind=True)
+def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -46,29 +38,25 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
-    chain_spec = ChainSpec.from_dict(chainspec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    session = BaseTask.session_func()
-    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    tx_brief = get_tx(tx_hash_orig_hex)
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
 
-    lock_queue(None, chain_spec.asdict(), address=address)
-    lock_send(None, chain_spec.asdict(), address=address)
-
-    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
+    lock_queue(None, chain_str, address)
+    lock_send(None, chain_str, address)
 
+    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -81,57 +69,49 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
-        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        tx_new['gas_price'] += 1
-        tx_new['gasPrice'] = tx_new['gas_price']
-        tx_new['nonce'] -= delta
-
-        logg.debug('tx_new {}'.format(tx_new))
-
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
-        del(tx_new['hashUnsigned'])
+        tx_new['nonce'] -= delta
 
-        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
-        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
+        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            tx_new['nonce'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            )
+            nonce=tx_new['nonce'],
+            address=tx_new['from'],
+            tx_hash=tx_hash_hex,
+            signed_tx=tx_signed_raw_hex,
+                )
         session.add(otx)
+        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
+        set_cancel(tx_previous_hash_hex, True)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
-        session.commit()
 
     session.close()
 
-    s = create_check_gas_task(
+    s = create_check_gas_and_send_task(
          txs, 
-         chain_spec,
+         chain_str,
          tx_new['from'],
-         gas=tx_new['gas'],
-         tx_hashes_hex=tx_hashes, 
-         queue=queue,
+         tx_new['gas'],
+         tx_hashes, 
+         queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_spec.asdict(),
+            chain_str,
             tx_new['from'],
             ],
         queue=queue,
@@ -139,7 +119,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_spec.asdict(),
+            chain_str,
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/cic_eth/api/api_admin.py

@@ -8,7 +8,6 @@ from chainlib.eth.constant import (
         ZERO_ADDRESS,
         )
 from cic_eth_registry import CICRegistry
-from cic_eth_registry.erc20 import ERC20Token
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.contract import code
@@ -31,14 +30,13 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx
 
 app = celery.current_app
 
@@ -190,7 +188,6 @@ class AdminApi:
         s_manual = celery.signature(
             'cic_eth.queue.state.set_manual',
             [
-                chain_spec.asdict(),
                 tx_hash_hex,
                 ],
             queue=self.queue,
@@ -209,9 +206,8 @@ class AdminApi:
             s.link(s_gas)
 
         return s_manual.apply_async()
-                    
-
-    def check_nonce(self, chain_spec, address):
+                        
+    def check_nonce(self, address):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -232,12 +228,13 @@ class AdminApi:
             s_get_tx = celery.signature(
                     'cic_eth.queue.query.get_tx',
                     [
-                        chain_spec.asdict(),
+                    chain_spec.asdict(),
                         k,
                         ],
                     queue=self.queue,
                     )
             tx = s_get_tx.apply_async().get()
+            #tx = get_tx(k)
             logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
             if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@@ -245,9 +242,7 @@ class AdminApi:
                 blocking_tx = k
                 blocking_nonce = nonce_otx
             elif nonce_otx - last_nonce > 1:
-                logg.debug('tx {}'.format(tx))
-                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
+                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from']))
                 blocking_tx = k
                 blocking_nonce = nonce_otx
                 break
@@ -261,13 +256,12 @@ class AdminApi:
                 'blocking': blocking_nonce,
             },
             'tx': {
-                'blocking': add_0x(blocking_tx),
+                'blocking': blocking_tx,
+                }
             }
-        }
 
 
-    # TODO: is risky since it does not validate that there is actually a nonce problem?
-    def fix_nonce(self, chain_spec, address, nonce):
+    def fix_nonce(self, address, nonce, chain_spec):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -281,17 +275,15 @@ class AdminApi:
         txs = s.apply_async().get()
 
         tx_hash_hex = None
-        session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx(k)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
-        session.close()
 
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    chain_spec.asdict(),
+                    self.rpc.chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -396,13 +388,12 @@ class AdminApi:
     
         t = s.apply_async()
         tx = t.get()
-
+  
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
-            source_token_declaration = None
             if registry != None:
                 try:
-                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
+                    source_token = registry.by_address(tx['source_token'])
                 except UnknownContractError:
                     logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
             else:
@@ -415,21 +406,16 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                source_token_declaration = t.get()
-
-            if source_token_declaration != None:
-                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
-                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
-                logg.debug('source token set tup {}'.format(source_token))
-
+                source_token = t.get()
+                if source_token == None:
+                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
 
 
         destination_token = None
         if tx['destination_token'] != ZERO_ADDRESS:
-            destination_token_declaration = None
             if registry != None:
                 try:
-                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
+                    destination_token = registry.by_address(tx['destination_token'])
                 except UnknownContractError:
                     logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
             else:
@@ -442,10 +428,10 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                destination_token_declaration = t.get()
-            if destination_token_declaration != None:
-                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
-                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
+                destination_token = t.get()
+                if destination_token == None:
+                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
+
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
@@ -557,19 +543,13 @@ class AdminApi:
                 if role != None:
                     tx['recipient_description'] = role
 
-        erc20_c = ERC20(chain_spec)
         if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol
-            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['source_token_symbol'] = source_token.symbol()
+            tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call()
 
         if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol
-            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
-            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
+            tx['destination_token_symbol'] = destination_token.symbol()
+            tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
         tx['network_status'] = 'Not in node' 

apps/cic-eth/cic_eth/api/api_task.py

@@ -74,134 +74,134 @@ class Api:
         return s_token.apply_async()
 
 
-#    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
-#
-#        :param from_address: Ethereum address of sender
-#        :type from_address: str, 0x-hex
-#        :param to_address: Ethereum address of receipient
-#        :type to_address: str, 0x-hex
-#        :param target_return: Estimated return from conversion
-#        :type  target_return: int
-#        :param minimum_return: The least value of destination token return to allow
-#        :type minimum_return: int
-#        :param from_token_symbol: ERC20 token symbol of token being converted
-#        :type from_token_symbol: str
-#        :param to_token_symbol: ERC20 token symbol of token to receive
-#        :type to_token_symbol: str
-#        :returns: uuid of root task
-#        :rtype: celery.Task
-#        """
-#        raise NotImplementedError('out of service until new DEX migration is done')
-#        s_check = celery.signature(
-#                'cic_eth.admin.ctrl.check_lock',
-#                [
-#                    [from_token_symbol, to_token_symbol],
-#                    self.chain_spec.asdict(),
-#                    LockEnum.QUEUE,
-#                    from_address,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce = celery.signature(
-#                'cic_eth.eth.nonce.reserve_nonce',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_tokens = celery.signature(
-#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-#                [
-#                    self.chain_str,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_convert = celery.signature(
-#                'cic_eth.eth.bancor.convert_with_default_reserve',
-#                [
-#                    from_address,
-#                    target_return,
-#                    minimum_return,
-#                    to_address,
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce.link(s_tokens)
-#        s_check.link(s_nonce)
-#        if self.callback_param != None:
-#            s_convert.link(self.callback_success)
-#            s_tokens.link(s_convert).on_error(self.callback_error)
-#        else:
-#            s_tokens.link(s_convert)
-#
-#        t = s_check.apply_async(queue=self.queue)
-#        return t
-#
-#
-#    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
-#
-#        :param from_address: Ethereum address of sender
-#        :type from_address: str, 0x-hex
-#        :param target_return: Estimated return from conversion
-#        :type  target_return: int
-#        :param minimum_return: The least value of destination token return to allow
-#        :type minimum_return: int
-#        :param from_token_symbol: ERC20 token symbol of token being converted
-#        :type from_token_symbol: str
-#        :param to_token_symbol: ERC20 token symbol of token to receive
-#        :type to_token_symbol: str
-#        :returns: uuid of root task
-#        :rtype: celery.Task
-#        """
-#        raise NotImplementedError('out of service until new DEX migration is done')
-#        s_check = celery.signature(
-#                'cic_eth.admin.ctrl.check_lock',
-#                [
-#                    [from_token_symbol, to_token_symbol],
-#                    self.chain_spec.asdict(),
-#                    LockEnum.QUEUE,
-#                    from_address,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce = celery.signature(
-#                'cic_eth.eth.nonce.reserve_nonce',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_tokens = celery.signature(
-#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_convert = celery.signature(
-#                'cic_eth.eth.bancor.convert_with_default_reserve',
-#                [
-#                    from_address,
-#                    target_return,
-#                    minimum_return,
-#                    from_address,
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce.link(s_tokens)
-#        s_check.link(s_nonce)
-#        if self.callback_param != None:
-#            s_convert.link(self.callback_success)
-#            s_tokens.link(s_convert).on_error(self.callback_error)
-#        else:
-#            s_tokens.link(s_convert)
-#
-#        t = s_check.apply_async(queue=self.queue)
-#        return t
+    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param to_address: Ethereum address of receipient
+        :type to_address: str, 0x-hex
+        :param target_return: Estimated return from conversion
+        :type  target_return: int
+        :param minimum_return: The least value of destination token return to allow
+        :type minimum_return: int
+        :param from_token_symbol: ERC20 token symbol of token being converted
+        :type from_token_symbol: str
+        :param to_token_symbol: ERC20 token symbol of token to receive
+        :type to_token_symbol: str
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        raise NotImplementedError('out of service until new DEX migration is done')
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [from_token_symbol, to_token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_str,
+                    ],
+                queue=self.queue,
+                )
+        s_convert = celery.signature(
+                'cic_eth.eth.bancor.convert_with_default_reserve',
+                [
+                    from_address,
+                    target_return,
+                    minimum_return,
+                    to_address,
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_convert.link(self.callback_success)
+            s_tokens.link(s_convert).on_error(self.callback_error)
+        else:
+            s_tokens.link(s_convert)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
+
+
+    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param target_return: Estimated return from conversion
+        :type  target_return: int
+        :param minimum_return: The least value of destination token return to allow
+        :type minimum_return: int
+        :param from_token_symbol: ERC20 token symbol of token being converted
+        :type from_token_symbol: str
+        :param to_token_symbol: ERC20 token symbol of token to receive
+        :type to_token_symbol: str
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        raise NotImplementedError('out of service until new DEX migration is done')
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [from_token_symbol, to_token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_convert = celery.signature(
+                'cic_eth.eth.bancor.convert_with_default_reserve',
+                [
+                    from_address,
+                    target_return,
+                    minimum_return,
+                    from_address,
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_convert.link(self.callback_success)
+            s_tokens.link(s_convert).on_error(self.callback_error)
+        else:
+            s_tokens.link(s_convert)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
 
 
     def transfer(self, from_address, to_address, value, token_symbol):

apps/cic-eth/cic_eth/db/util.py

@@ -0,0 +1,8 @@
+import math
+
+def num_serialize(n):
+    if n == 0:
+        return b'\x00'
+    binlog = math.log2(n)
+    bytelength = int(binlog / 8 + 1)
+    return n.to_bytes(bytelength, 'big')

apps/cic-eth/cic_eth/eth/gas.py

@@ -57,12 +57,10 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-MAXIMUM_FEE_UNITS = 8000000
-
 class MaxGasOracle:
 
     def gas(code=None):
-        return MAXIMUM_FEE_UNITS
+        return 8000000
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -152,7 +150,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -172,30 +170,24 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
-
-    addresspass = None
     if len(txs) == 0:
-        addresspass = []
         for i in range(len(tx_hashes)):
-            o = get_tx(chain_spec_dict, tx_hashes[i])
+            o = get_tx(tx_hashes[i])
             txs.append(o['signed_tx'])
-            logg.debug('sender {}'.format(o))
-            tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
             if address == None:
-                address = tx['from']
-            elif address != tx['from']:
-                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
-            addresspass.append(address)
+                address = o['address']
 
+    #if not web3.Web3.isChecksumAddress(address):
     if not is_checksum_address(address):
         raise ValueError('invalid address {}'.format(address))
 
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
+    # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     gas_balance = 0
     try:
         o = balance(address)
@@ -206,9 +198,6 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
         conn.disconnect()
         raise EthError('gas_balance call for {}: {}'.format(address, e))
 
-    if gas_required == None:
-        gas_required = MAXIMUM_FEE_UNITS
-
     logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
     session = SessionBase.create_session()
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@@ -279,8 +268,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
             queue=queue,
             )
         ready_tasks.append(s)
-    t = celery.group(ready_tasks)()
-    logg.debug('group {}'.format(t))
+    celery.group(ready_tasks)()
 
     return txs
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -21,7 +21,6 @@ from chainqueue.db.models.tx import Otx
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db import SessionBase
@@ -59,9 +58,6 @@ def hashes_to_txs(self, tx_hashes):
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
-    for i in range(len(tx_hashes)):
-        tx_hashes[i] = strip_0x(tx_hashes[i])
-
     queue = self.request.delivery_info['routing_key']
 
     session = SessionBase.create_session()
@@ -152,7 +148,7 @@ def send(self, txs, chain_spec_dict):
 
 @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
 def sync_tx(self, tx_hash_hex, chain_spec_dict):
-    """Force update of network status of a single transaction
+    """Force update of network status of a simgle transaction
 
     :param tx_hash_hex: Transaction hash
     :type tx_hash_hex: str, 0x-hex
@@ -177,14 +173,12 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
 
     # TODO: apply receipt in tx object to validate and normalize input
     if rcpt != None:
-        rcpt = snake_and_camel(rcpt)
         success = rcpt['status'] == 1
-        logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
+        logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
 
         s = celery.signature(
             'cic_eth.queue.state.set_final',
             [
-                chain_spec_dict,
                 tx_hash_hex,
                 rcpt['blockNumber'],
                 rcpt['transactionIndex'],
@@ -192,14 +186,12 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
                 ],
                 queue=queue,
             )
-    # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
     else:
         logg.debug('sync tx {} mempool'.format(tx_hash_hex))
 
         s = celery.signature(
             'cic_eth.queue.state.set_sent',
             [
-                chain_spec_dict,
                 tx_hash_hex,
                 ],
                 queue=queue,

apps/cic-eth/cic_eth/ext/tx.py

@@ -19,7 +19,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
-from chainqueue.sql.query import get_tx_cache
+from chainqueue.query import get_tx_cache
 from eth_erc20 import ERC20
 
 # local imports

apps/cic-eth/cic_eth/queue/query.py

@@ -5,7 +5,7 @@ import datetime
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.sql.query
+import chainqueue.query
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -28,7 +28,7 @@ celery_app = celery.current_app
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
+    r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
+    r =  chainqueue.query.get_tx(chain_spec, tx_hash)
     session.close()
     return r
 
@@ -46,7 +46,7 @@ def get_tx(chain_spec_dict, tx_hash):
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
+    r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
     session.close()
     return r
 
@@ -55,17 +55,17 @@ def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True,
 def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):

apps/cic-eth/cic_eth/queue/state.py

@@ -1,6 +1,6 @@
 # external imports
 from chainlib.chain import ChainSpec
-import chainqueue.sql.state
+import chainqueue.state
 
 # local imports
 import celery
@@ -14,7 +14,7 @@ celery_app = celery.current_app
 def set_sent(chain_spec_dict, tx_hash, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
+    r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session)
     session.close()
     return r
 
@@ -23,7 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
+    r = chainqueue.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
     session.close()
     return r
 
@@ -32,7 +32,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
+    r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
     session.close()
     return r
 
@@ -41,7 +41,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 def set_rejected(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -50,7 +50,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 def set_fubar(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -59,7 +59,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 def set_manual(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -68,7 +68,7 @@ def set_manual(chain_spec_dict, tx_hash):
 def set_ready(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -77,7 +77,7 @@ def set_ready(chain_spec_dict, tx_hash):
 def set_reserved(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -86,7 +86,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 def set_waitforgas(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
     session.close()
     return r
     
@@ -95,7 +95,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 def get_state_log(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -104,6 +104,6 @@ def get_state_log(chain_spec_dict, tx_hash):
 def obsolete(chain_spec_dict, tx_hash, final):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
+    r = chainqueue.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
     session.close()
     return r

apps/cic-eth/cic_eth/queue/tx.py

@@ -15,14 +15,14 @@ from sqlalchemy import tuple_
 from sqlalchemy import func
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.sql.state
+import chainqueue.state
 from chainqueue.db.enum import (
         StatusEnum,
         StatusBits,
         is_alive,
         dead,
         )
-from chainqueue.sql.tx import create
+from chainqueue.tx import create
 from chainqueue.error import NotLocalTxError
 from chainqueue.db.enum import status_str
 

apps/cic-eth/cic_eth/registry.py

@@ -5,30 +5,29 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
-from chainlib.eth.constant import ZERO_ADDRESS
 
 logg = logging.getLogger()
 
 
-def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
+def connect_token_registry(rpc, chain_spec):
     registry = CICRegistry(chain_spec, rpc)
-    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
+    token_registry_address = registry.by_name('TokenRegistry')
     logg.debug('using token registry address {}'.format(token_registry_address))
     lookup = TokenIndexLookup(chain_spec, token_registry_address)
     CICRegistry.add_lookup(lookup)
 
 
-def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
+def connect_declarator(rpc, chain_spec, trusted_addresses):
     registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    declarator_address = registry.by_name('AddressDeclarator')
     logg.debug('using declarator address {}'.format(declarator_address))
     lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
     CICRegistry.add_lookup(lookup)
 
 
-def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
+def connect(rpc, chain_spec, registry_address):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
+    registry_address = registry.by_name('ContractRegistry')
     return registry
 

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -21,7 +21,7 @@ from chainqueue.db.enum import (
     StatusBits,
     )
 from chainqueue.error import NotLocalTxError
-from chainqueue.sql.state import set_reserved
+from chainqueue.state import set_reserved
 
 # local imports
 import cic_eth

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,15 +10,14 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
+from chainqueue.query import get_paused_tx_cache as get_paused_tx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 
 class GasFilter(SyncFilter):
@@ -28,11 +27,11 @@ class GasFilter(SyncFilter):
         self.chain_spec = chain_spec
 
 
-    def filter(self, conn, block, tx, db_session):
+    def filter(self, conn, block, tx, session):
         if tx.value > 0:
             tx_hash_hex = add_0x(tx.hash)
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(db_session)
+            session = SessionBase.bind_session(session)
             q = session.query(TxCache.recipient)
             q = q.join(Otx)
             q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@@ -57,7 +56,7 @@ class GasFilter(SyncFilter):
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
-                return s.apply_async()
+                s.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -50,8 +50,7 @@ class RegistrationFilter(SyncFilter):
                     queue=self.queue,
                     )
                 s_nonce.link(s_gift)
-                t = s_nonce.apply_async()
-                return t
+                s_nonce.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/straggler.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from chainqueue.sql.state import obsolete_by_cache
+from chainqueue.state import obsolete_by_cache
 
 logg = logging.getLogger()
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
         self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
 
 
-    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
+    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
 
         if tx.payload == None:
             logg.debug('no payload')
@@ -45,17 +45,16 @@ class TransferAuthFilter(SyncFilter):
             return False
 
         recipient = tx.inputs[0]
-        #if recipient != self.transfer_request_contract.address():
-        if recipient != self.transfer_request_contract:
+        if recipient != self.transfer_request_contract.address():
             logg.debug('not our transfer auth contract address {}'.format(recipient))
             return False
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
-          
-        sender = r[0]
-        recipient = r[1]
-        token = r[2]
-        value = r[3]
+           
+        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
+        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
+        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
+        value = abi_decode_single(ABIContractType.UINT256, r[3])
 
         token_data = {
             'address': token,
@@ -65,7 +64,6 @@ class TransferAuthFilter(SyncFilter):
             'cic_eth.eth.nonce.reserve_nonce',
             [
                 [token_data],
-                self.chain_spec.asdict(), 
                 sender,
                 ],
             queue=self.queue,
@@ -82,7 +80,7 @@ class TransferAuthFilter(SyncFilter):
             )
         s_nonce.link(s_approve)
         t = s_nonce.apply_async()
-        return t
+        return True
 
 
     def __str__(self):

apps/cic-eth/cic_eth/stat.py

@@ -20,11 +20,7 @@ def init_chain_stat(rpc, block_start=0):
     if block_start == 0:
         o = block_latest()
         r = rpc.do(o)
-        try:
-            block_start = int(r, 16)
-        except TypeError:
-            block_start = int(r)
-        logg.debug('blockstart {}'.format(block_start))
+        block_start = int(r, 16)
 
     for i in range(BLOCK_SAMPLES):
         o = block_by_number(block_start-10+i)

apps/cic-eth/cic_eth/task.py

@@ -20,8 +20,7 @@ import liveness.linux
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
 
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 celery_app = celery.current_app
 
@@ -119,13 +118,12 @@ def registry():
     return CICRegistry.address
 
 
-@celery_app.task(bind=True, base=BaseTask)
-def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
+@celery_app.task()
+def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    r = registry.by_address(address, sender_address=self.call_address)
-    return r
+    return registry.by_address(address)
 
 
 @celery_app.task(throws=(UnknownContractError,))
@@ -133,7 +131,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name, sender_address=self.call_address)
+    return registry.by_name(name)
 
 
 @celery_app.task()

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.16',
+        'beta.13',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,62 +1,48 @@
-FROM python:3.8.6-slim-buster as compile
+# FROM grassrootseconomics:cic
+
+#FROM python:3.8.6-alpine
+FROM python:3.8.6-slim-buster
+
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
 
 WORKDIR /usr/src/cic-eth
 
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-#RUN python -m venv venv && . venv/bin/activate
-
 ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-RUN /usr/local/bin/python -m pip install --upgrade pip
-RUN pip install semver
-
-# TODO use a packaging style that lets us copy requirments only ie. pip-tools 
-COPY cic-eth/ .
-RUN	pip install $pip_extra_index_url_flag .
-
-# --- TEST IMAGE ---
-FROM python:3.8.6-slim-buster as test 
+ARG root_requirement_file='requirements.txt'
 
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-WORKDIR /usr/src/cic-eth
-
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file:  ${root_requirement_file}"
+#COPY $root_requirement_file .
+#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
 RUN /usr/local/bin/python -m pip install --upgrade pip
+#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
+#	cd cic-base && \
+#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
+#	python merge_requirements.py | tee merged_requirements.txt 
+#RUN cd cic-base && \
+#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
+RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-# TODO we could use venv inside container to isolate the system and app deps further
-# COPY --from=compile /usr/src/cic-eth/ .
-# RUN . venv/bin/activate
-
+COPY cic-eth/scripts/ scripts/
+COPY cic-eth/setup.cfg cic-eth/setup.py ./
+COPY cic-eth/cic_eth/ cic_eth/
+# Copy app specific requirements
+COPY cic-eth/requirements.txt .
 COPY cic-eth/test_requirements.txt .
-RUN	pip install $pip_extra_index_url_flag -r test_requirements.txt 
-
-COPY cic-eth . 
-
-ENV PYTHONPATH .
-
-ENTRYPOINT ["pytest"]
-
-# --- RUNTIME ---
-FROM python:3.8.6-slim-buster as runtime 
-
-RUN apt-get update && \
-	apt install -y gnupg libpq-dev procps
-
-WORKDIR /usr/src/cic-eth
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
+RUN	pip install $pip_extra_index_url_flag .
 
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
+COPY cic-eth/tests/ tests/
 
-COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -65,4 +51,3 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
-

apps/cic-eth/requirements.txt

@@ -1,25 +1,24 @@
-cic-base~=0.1.2b15
+cic-base~=0.1.2b11
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a7
+cic-eth-registry~=0.5.5a4
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
 eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a7
+erc20-transfer-authorization~=0.3.1a6
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.1a11
-chainlib~=0.0.3rc2
+chainlib~=0.0.3a2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a5
-chainqueue~=0.0.2b3
+chainsyncer[sql]~=0.0.2a4
+chainqueue~=0.0.2a2
 sarafu-faucet==0.0.3a3
 erc20-faucet==0.2.1a4
 coincurve==15.0.0
 potaahto~=0.0.1a2
-pycryptodome==3.10.1

apps/cic-eth/setup.py

@@ -11,6 +11,17 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
+test_requirements = []
+f = open('test_requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    test_requirements.append(l.rstrip())
+f.close()
+
+
 setup(
-    install_requires=requirements
+    install_requires=requirements,
+    tests_require=test_requirements,
         )

apps/cic-eth/test_requirements.txt

@@ -4,3 +4,4 @@ pytest-mock==3.3.1
 pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
+giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/check/test_check_db.py

@@ -1,8 +0,0 @@
-# local imports
-from cic_eth.check.db import health
-
-def test_check_health(
-    init_database,
-        ):
-
-    assert health()

apps/cic-eth/tests/check/test_check_gas.py

@@ -1,20 +0,0 @@
-# local imports
-from cic_eth.check.gas import health
-from cic_eth.db.models.role import AccountRole
-
-def test_check_gas(
-    config,
-    init_database,
-    default_chain_spec,
-    eth_rpc,
-    custodial_roles,
-    whoever,
-        ):
-
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
-    assert health(config=config)
-
-    AccountRole.set('GAS_GIFTER', whoever, session=init_database)
-    init_database.commit()
-    assert not health(config=config)

apps/cic-eth/tests/check/test_check_redis.py

@@ -1,16 +0,0 @@
-# external imports
-import pytest
-
-# local imports
-from cic_eth.check.redis import health
-
-
-def test_check_redis(
-        config,
-        have_redis,
-        ):
-
-    if have_redis != None:
-        pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
-
-    assert health(unit='test', config=config)

apps/cic-eth/tests/check/test_check_signer.py

@@ -1,13 +0,0 @@
-# local imports
-from cic_eth.check.signer import health
-
-
-def test_check_signer(
-        default_chain_spec,
-        config,
-        eth_signer,
-        eth_rpc,
-        ):
-
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    assert health(config=config)

apps/cic-eth/tests/conftest.py

@@ -2,11 +2,9 @@
 import os
 import sys
 import logging
-import uuid
 
 # external imports
 from eth_erc20 import ERC20
-import redis
 
 # local imports
 from cic_eth.api import Api
@@ -21,7 +19,6 @@ from tests.fixtures_config import *
 from tests.fixtures_database import *
 from tests.fixtures_celery import *
 from tests.fixtures_role import *
-from tests.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -58,28 +55,3 @@ def default_token(
         ):
     BaseTask.default_token_symbol = foo_token_symbol
     BaseTask.default_token_address = foo_token   
-
-
-
-@pytest.fixture(scope='session')
-def have_redis(
-        config,
-        ):
-
-    r = redis.Redis(
-            host = config.get('REDIS_HOST'),
-            port = config.get('REDIS_PORT'),
-            db = config.get('REDIS_DB'),
-            ) 
-    k = str(uuid.uuid4())
-    try:
-        r.set(k, 'foo')
-        r.delete(k)
-    except redis.exceptions.ConnectionError as e:
-        return e
-    except TypeError as e:
-        return e 
-
-    return None
-
-

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -1,38 +0,0 @@
-# local imports
-from cic_eth.runnable.daemons.filters.gas import GasFilter
-from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
-from cic_eth.runnable.daemons.filters.callback import CallbackFilter
-from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
-from cic_eth.runnable.daemons.filters.tx import TxFilter
-from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-
-
-# Hit tx mismatch paths on all filters
-def test_filter_bogus(
-        init_database,
-        bogus_tx_block,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        transfer_auth,
-        cic_registry,
-        contract_roles,
-        register_lookups,
-        ):
-
-    fltrs = [
-        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
-        GasFilter(default_chain_spec, queue=None),
-        TxFilter(default_chain_spec, None),
-        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
-        StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, queue=None),
-        ]
-      
-    for fltr in fltrs:
-        r = None
-        try:
-            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
-        except:
-            pass
-        assert not r

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -1,101 +0,0 @@
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainqueue.sql.state import (
-        set_waitforgas,
-        )
-from hexathon import strip_0x
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-
-# local imports
-from cic_eth.runnable.daemons.filters.gas import GasFilter
-from cic_eth.eth.gas import cache_gas_data
-
-
-def test_filter_gas(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        celery_session_worker,
-    ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
-    init_database.commit()
-    tx_hash_hex_wait = tx_hash_hex
-    otx = Otx.load(tx_hash_hex_wait, session=init_database)
-    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['BOB'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    fltr = GasFilter(default_chain_spec, queue=None)
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-    init_database.commit()
-
-    otx = Otx.load(tx_hash_hex_wait, session=init_database)
-    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,78 +0,0 @@
-# external imports
-from eth_accounts_index.registry import AccountRegistry
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.tx import(
-        receipt,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from erc20_faucet import Faucet
-from hexathon import strip_0x
-from chainqueue.sql.query import get_account_tx
-
-# local imports
-from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-
-
-def test_register_filter(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        account_registry,
-        faucet,
-        register_lookups,
-        contract_roles,
-        agent_roles,
-        cic_registry,
-        init_celery_tasks,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
-
-    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
-    r = eth_rpc.do(o)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
-
-    o = receipt(tx_hash_hex)
-    rcpt = eth_rpc.do(o)
-    assert rcpt['status'] == 1
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block, rcpt=rcpt)
-    tx.apply_receipt(rcpt)
-
-    fltr = RegistrationFilter(default_chain_spec, queue=None)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-
-    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
-    ks = list(gift_txs.keys())
-    assert len(ks) == 1
-
-    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
-    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
-    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
-
-    gift = Faucet.parse_give_to_request(gift_tx['data'])
-    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_straggler_filter.py

@@ -17,8 +17,8 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
+from chainqueue.tx import create as queue_create
+from chainqueue.state import (
         set_reserved,
         set_ready,
         set_sent,

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -1,79 +0,0 @@
-# external imports
-from erc20_transfer_authorization import TransferAuthorization
-from eth_erc20 import ERC20
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.tx import (
-        receipt,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from hexathon import strip_0x
-from chainqueue.sql.query import get_account_tx
-
-# local imports
-from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
-
-
-def test_filter_transferauth(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        transfer_auth,
-        foo_token,
-        celery_session_worker,
-        register_lookups,
-        init_custodial,
-        cic_registry,
-    ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
-    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
-    
-    r = rpc.do(o)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-
-    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-
-    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
-    ks = list(approve_txs.keys())
-    assert len(ks) == 1
-
-    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
-    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
-    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
-
-    c = ERC20(default_chain_spec)
-    approve = c.parse_approve_request(approve_tx['data']) 
-    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -17,12 +17,13 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
+from chainqueue.tx import create as queue_create
+from chainqueue.state import (
         set_reserved,
         set_ready,
         set_sent,
         )
+
 from hexathon import strip_0x
 
 # local imports
@@ -30,7 +31,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
 from cic_eth.eth.gas import cache_gas_data
 
 
-def test_filter_tx(
+def test_tx(
         default_chain_spec,
         init_database,
         eth_rpc,

apps/cic-eth/tests/fixtures_celery.py

@@ -22,6 +22,7 @@ def init_celery_tasks(
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
+#        'cic_eth.eth.bancor',
         'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
@@ -46,8 +47,8 @@ def celery_config():
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
     rq = tempfile.mkdtemp()
-    logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
-    logg.debug('celery backend session store {}'.format(rq))
+    logg.debug('celery broker queue {} processed {}'.format(bq, bp))
+    logg.debug('celery backend store {}'.format(rq))
     yield {
         'broker_url': 'filesystem://',
         'broker_transport_options': {
@@ -57,11 +58,12 @@ def celery_config():
             },
         'result_backend': 'file://{}'.format(rq),
             }
-    logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
+    logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq))
     shutil.rmtree(bq)
     shutil.rmtree(bp)
     shutil.rmtree(rq)
 
+
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {

apps/cic-eth/tests/fixtures_contract.py

@@ -1,77 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import pytest
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        ABIContractType,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        TxFactory,
-        TxFormat,
-        unpack,
-        Tx,
-        )
-from hexathon import strip_0x
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-
-
-@pytest.fixture(scope='function')
-def bogus_tx_block(
-    default_chain_spec,
-    eth_rpc,
-    eth_signer,
-    contract_roles,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
-
-    f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
-    bytecode = f.read()
-    f.close()
-
-    c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
-    tx = c.set_code(tx, bytecode)
-    (tx_hash_hex, o) = c.build(tx)
-
-    r = eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-
-    contract_address = r['contract_address']
-
-    enc = ABIContractEncoder()
-    enc.method('poke')
-    data = enc.get()
-    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
-    tx = c.set_code(tx, data)
-    (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
-    r = eth_rpc.do(o)
-    tx_signed_raw_hex = strip_0x(o['params'][0])
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-
-    return (block, tx)

apps/cic-eth/tests/task/api/test_admin.py

@@ -9,14 +9,8 @@ from chainlib.eth.tx import (
         unpack,
         TxFormat,
         )
-from chainlib.eth.nonce import (
-        RPCNonceOracle,
-        OverrideNonceOracle,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import Gas
 from chainlib.eth.address import to_checksum_address
 from hexathon import (
         strip_0x,
@@ -29,15 +23,7 @@ from chainqueue.db.enum import (
         StatusBits,
         status_str,
         )
-from chainqueue.sql.state import (
-        set_fubar,
-        set_ready,
-        set_reserved,
-        )
-from chainqueue.sql.query import (
-        get_tx,
-        get_nonce_tx_cache,
-        )
+from chainqueue.query import get_tx
 
 # local imports
 from cic_eth.api import AdminApi
@@ -50,6 +36,150 @@ from cic_eth.queue.tx import queue_create
 logg = logging.getLogger()
 
 
+#def test_resend_inplace(
+#    default_chain_spec,
+#    init_database,
+#    init_w3,
+#    celery_session_worker,
+#    ):
+#    
+#    chain_str = str(default_chain_spec)
+#    c = RpcClient(default_chain_spec)
+#    
+#    sigs = []
+#
+#    gas_provider = c.gas_provider()
+#
+#    s_nonce = celery.signature(
+#        'cic_eth.eth.nonce.reserve_nonce',
+#        [
+#            init_w3.eth.accounts[0],
+#            gas_provider,
+#            ],
+#        queue=None,
+#        )
+#    s_refill = celery.signature(
+#        'cic_eth.eth.gas.refill_gas',
+#        [
+#            chain_str,
+#            ],
+#        queue=None,
+#            )
+#    s_nonce.link(s_refill)
+#    t = s_nonce.apply_async()
+#    t.get()
+#    for r in t.collect():
+#        pass
+#    assert t.successful()
+#
+#    q = init_database.query(Otx)
+#    q = q.join(TxCache)
+#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
+#    o = q.first()
+#    tx_raw = o.signed_tx
+#
+#    tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
+#    gas_price_before = tx_dict['gasPrice'] 
+#
+#    s = celery.signature(
+#        'cic_eth.admin.ctrl.lock_send',
+#        [
+#            chain_str,
+#            init_w3.eth.accounts[0],
+#            ],
+#        queue=None,
+#            )
+#    t = s.apply_async()
+#    t.get()
+#    assert t.successful()
+#
+#    api = AdminApi(c, queue=None)
+#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
+#    t.get()
+#    i = 0
+#    tx_hash_new_hex = None
+#    for r in t.collect():
+#        tx_hash_new_hex = r[1]
+#    assert t.successful()
+#  
+#    tx_raw_new = get_tx(tx_hash_new_hex) 
+#    logg.debug('get {}'.format(tx_raw_new))
+#    tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
+#    assert tx_hash_new_hex != tx_dict['hash']
+#    assert tx_dict_new['gasPrice'] > gas_price_before
+#
+#    tx_dict_after = get_tx(tx_dict['hash'])
+#
+#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
+#    assert tx_dict_after['status'] & StatusBits.MANUAL
+
+
+#def test_check_fix_nonce(
+#    default_chain_spec,
+#    init_database,
+#    init_eth_account_roles,
+#    init_w3,
+#    eth_empty_accounts,
+#    celery_session_worker,
+#    ):
+#
+#    chain_str = str(default_chain_spec)
+#    
+#    sigs = []
+#    for i in range(5):
+#        s = celery.signature(
+#            'cic_eth.eth.gas.refill_gas',
+#            [
+#                eth_empty_accounts[i],
+#                chain_str,
+#                ],
+#            queue=None,
+#                )
+#        sigs.append(s)
+#
+#    t = celery.group(sigs)()
+#    txs = t.get()
+#    assert t.successful()
+#
+#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
+#    c = RpcClient(default_chain_spec)
+#    api = AdminApi(c, queue=None)
+#    address = init_eth_account_roles['eth_account_gas_provider']
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['network'] == 0
+#    assert nonce_spec['nonce']['queue'] == 4
+#    assert nonce_spec['nonce']['blocking'] == None
+#
+#    s_set = celery.signature(
+#            'cic_eth.queue.tx.set_rejected',
+#            [
+#                tx_hash.hex(),
+#                ],
+#            queue=None,
+#            )
+#    t = s_set.apply_async()
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['blocking'] == 2
+#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
+#
+#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#    for tx in txs[3:]:
+#        tx_hash = web3.Web3.keccak(hexstr=tx)
+#        tx_dict = get_tx(tx_hash.hex())
+#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
+#
+#
+
+
 def test_have_account(
     default_chain_spec,
     custodial_roles,
@@ -113,6 +243,28 @@ def test_tag_account(
     assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
+#def test_ready(
+#    init_database,
+#    agent_roles,
+#    eth_rpc,
+#    ):
+#
+#    api = AdminApi(eth_rpc)
+#   
+#    with pytest.raises(InitializationError):
+#        api.ready()
+#
+#    bogus_account = os.urandom(20)
+#    bogus_account_hex = '0x' + bogus_account.hex()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
+#    with pytest.raises(KeyError):
+#        api.ready()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
+#    api.ready()
+
+
 def test_tx(
     default_chain_spec,
     cic_registry,
@@ -134,168 +286,3 @@ def test_tx(
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
     tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
     logg.warning('code missing to verify tx contents {}'.format(tx))
-
-
-def test_check_nonce_gap(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    # NOTE: this only works as long as agents roles start at nonce 0
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    j = 0
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        if i == 3:
-            j = 1
-            nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
-
-        queue_create(
-                default_chain_spec,
-                i+j,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
-
-    assert r['nonce']['blocking'] == 4
-    assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
-
-
-def test_check_nonce_localfail(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    # NOTE: this only works as long as agents roles start at nonce 0
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    j = 0
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-        queue_create(
-                default_chain_spec,
-                i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    set_ready(default_chain_spec, tx_hashes[4], session=init_database)
-    set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
-    set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
-
-    assert r['nonce']['blocking'] == 4
-    assert r['tx']['blocking'] == tx_hashes[4]
-
-
-def test_fix_nonce(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        init_celery_tasks,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-        queue_create(
-                default_chain_spec,
-                i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
-    r = t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-    
-    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
-    ks = txs.keys()
-    assert len(ks) == 2
-    for k in ks:
-        hsh = add_0x(k)
-        otx = Otx.load(hsh, session=init_database)
-        init_database.refresh(otx)
-        logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
-        if add_0x(k) == tx_hashes[3]:
-            assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-        else:
-            assert otx.status == 1

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -1,373 +0,0 @@
-# standard imports
-import logging
-import io
-import json
-
-# external imports
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import (
-        nonce,
-        OverrideNonceOracle,
-        RPCNonceOracle,
-        )
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.tx import (
-        raw,
-        receipt,
-        TxFormat,
-        Tx,
-        )
-from chainlib.eth.block import block_latest
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        set_sent,
-        set_ready,
-        )
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-from chainqueue.sql.query import get_nonce_tx_cache
-from eth_erc20 import ERC20
-from cic_eth_registry import CICRegistry
-
-# local imports
-from cic_eth.api.api_admin import AdminApi
-from cic_eth.eth.gas import cache_gas_data
-from cic_eth.eth.erc20 import cache_transfer_data
-
-logg = logging.getLogger()
-
-
-def test_admin_api_tx(
-        default_chain_spec,
-        init_database,
-        init_celery_tasks,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        custodial_roles,
-        celery_session_worker,
-        foo_token,
-        address_declarator,
-        cic_registry,
-        register_tokens,
-        register_lookups,
-        caplog,
-        ):
-
-    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
-
-    o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
-    r = eth_rpc.do(o)
-    gifter_nonce = int(r, 16)
-
-    #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-    queue_create(
-            default_chain_spec,
-            gifter_nonce, # will only work if agent starts at 0
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_transfer_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    o = raw(tx_signed_raw_hex)
-    eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    # lookup by transaction hash, without registry
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
-    logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by RLP transaction, without registry
-    tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by transaction hash, with registry
-    registry = CICRegistry(default_chain_spec, eth_rpc)
-    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by transaction hash, using writer
-    buf = io.StringIO()
-    api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
-    tx = json.loads(buf.getvalue())
-    assert tx['tx_hash'] == tx_hash_hex 
-
-
-def test_admin_api_account(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes_alice = []
-    txs_alice = []
-
-    for i in range(3):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                42+i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes_alice.append(tx_hash_hex)
-        txs_alice.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
-    tx_hashes_bob = []
-    txs_bob = []
-
-    for i in range(2):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                13+i,
-                agent_roles['BOB'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes_bob.append(tx_hash_hex)
-        txs_bob.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'])
-    assert len(r) == 5
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
-    assert len(r) == 2
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
-    assert len(r) == 3
-
-
-def test_admin_api_account_writer(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    buf = io.StringIO()
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
-
-    # TODO: improve eval
-    tx = json.loads(buf.getvalue())
-    assert tx['tx_hash'] == tx_hash_hex
-
-
-def test_registry(
-        eth_rpc,
-        cic_registry,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.registry()
-    r = t.get_leaf()
-    assert r == cic_registry
-
-
-def test_proxy_do(
-        default_chain_spec,
-        eth_rpc,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.proxy_do(default_chain_spec, o)
-    rr = t.get_leaf()
-
-    assert r == rr
-
-
-def test_resend_inplace(
-        init_database,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
-    r = t.get_leaf()
-    assert t.successful()
-
-
-    otx = Otx.load(tx_hash_hex, session=init_database)
-    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 2
-
-
-
-@pytest.mark.xfail()
-def test_resend_clone(
-        init_database,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
-    r = t.get_leaf()
-    assert t.successful()
-
-    otx = Otx.load(tx_hash_hex, session=init_database)
-    assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
-    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 1
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 1
-
-    otx = Otx.load(txs[0], session=init_database)
-    assert otx.status == 0

apps/cic-eth/tests/task/api/test_app.py

@@ -8,20 +8,11 @@ import pytest
 import celery
 from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
-from eth_accounts_index import AccountsIndex
-from chainlib.eth.tx import (
-        transaction,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        )
 
 # local imports
 from cic_eth.api import Api
-from cic_eth.queue.query import get_tx
 
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 def test_account_api(
@@ -38,47 +29,6 @@ def test_account_api(
     assert t.successful()
 
 
-def test_account_api_register(
-        default_chain_spec,
-        init_database,
-        account_registry,
-        faucet,
-        custodial_roles,
-        cic_registry,
-        register_lookups,
-        eth_rpc,
-        celery_session_worker,
-        ):
-    api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.create_account('')
-    register_tx_hash = t.get_leaf()
-    assert t.successful()
-
-    set_reserved(default_chain_spec, register_tx_hash, session=init_database)
-
-    tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
-    s = celery.signature(
-            'cic_eth.eth.tx.send',
-            [
-                [tx['signed_tx']],
-                default_chain_spec.asdict(),
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    o = transaction(register_tx_hash)
-    tx_src = eth_rpc.do(o)
-
-    c = AccountsIndex(default_chain_spec)
-    address = c.parse_add_request(tx_src['data'])
-    o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    assert c.parse_have(r)
-
-
 def test_transfer_api(
         default_chain_spec,
         eth_rpc,
@@ -87,15 +37,16 @@ def test_transfer_api(
         custodial_roles,
         agent_roles,
         cic_registry,
-        token_registry,
+        register_tokens,
         register_lookups,
         celery_session_worker,
-        register_tokens,
-        foo_token_symbol,
         ):
 
+    #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
+    foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
+
     api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) 
+    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol)
     t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -1,19 +0,0 @@
-# local imports
-from cic_eth.api.api_task import Api
-from cic_eth.task import BaseTask
-
-def test_default_token(
-        default_chain_spec,
-        foo_token,
-        default_token,
-        token_registry,
-        register_tokens,
-        register_lookups,
-        cic_registry,
-        celery_session_worker,
-        ):
-
-    api = Api(str(default_chain_spec), queue=None)     
-    t = api.default_token()
-    r = t.get_leaf()
-    assert r['address'] == foo_token

apps/cic-eth/tests/task/test_task_account.py

@@ -156,7 +156,6 @@ def test_gift(
     eth_signer,
     init_celery_tasks,
     cic_registry,
-    register_lookups,
     celery_session_worker,
     ):
 

apps/cic-eth/tests/task/test_task_admin.py

@@ -1,88 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import (
-        create as queue_create,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainlib.eth.tx import TxFormat
-from chainqueue.sql.query import get_nonce_tx_cache
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-from hexathon import add_0x
-
-# local imports
-from cic_eth.admin.nonce import shift_nonce
-from cic_eth.eth.gas import cache_gas_data
-
-logg = logging.getLogger()
-
-
-def test_shift_nonce(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                42+i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.admin.nonce.shift_nonce',
-            [
-                default_chain_spec.asdict(),
-                tx_hashes[3],
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    init_database.commit()
-
-
-    for i in range(42+3, 42+10):
-        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
-        for k in txs.keys():
-            hsh = add_0x(k)
-            otx = Otx.load(hsh, session=init_database)
-            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
-            if add_0x(k) == tx_hashes[i-42]:
-                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-            else:
-                assert otx.status == 1

apps/cic-eth/tests/task/test_task_gas.py

@@ -1,286 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import (
-        OverrideNonceOracle,
-        RPCNonceOracle,
-        )
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
-from chainlib.eth.tx import (
-        unpack,
-        TxFormat,
-        )
-from chainlib.eth.constant import (
-        MINIMUM_FEE_UNITS,
-        MINIMUM_FEE_PRICE,
-        )
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.query import get_tx
-from chainqueue.db.enum import StatusBits
-from chainqueue.sql.state import (
-        set_ready,
-        set_reserved,
-        set_sent,
-        )
-from chainqueue.db.models.otx import Otx
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.eth.gas import cache_gas_data
-from cic_eth.error import OutOfGasError
-
-logg = logging.getLogger()
-
-
-def test_task_check_gas_ok(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                [],
-                None,
-                8000000,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
-
-
-def test_task_check_gas_insufficient(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        whoever,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(whoever, 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            whoever,
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                [],
-                None,
-                None,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    try:
-        r = t.get_leaf()
-    except OutOfGasError:
-        pass
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
-
-
-def test_task_check_gas_low(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        whoever,
-        ):
-
-    gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
-    nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
-    r = eth_rpc.do(o)
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) 
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            whoever,
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                ],
-                [],
-                None,
-                None,
-            queue=None
-            )
-    t = s.apply_async()
-    t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
-
-
-@pytest.mark.parametrize(
-        '_gas_price,_gas_factor',
-        [
-        (None, 1.1),
-        (MINIMUM_FEE_PRICE * 1.1, 0.9),
-        (None, 1.3),
-        ]
-        )
-def test_task_resend_explicit(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        _gas_price,
-        _gas_factor,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
-
-    init_database.commit()
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
-            [
-                tx_hash_hex,
-                default_chain_spec.asdict(),
-                _gas_price,
-                _gas_factor,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    q = init_database.query(Otx)
-    q = q.filter(Otx.tx_hash==strip_0x(r))
-    otx = q.first()
-    if otx == None:
-        raise NotLocalTxError(r)
-
-    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
-    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
-    assert tx_after['gasPrice'] > tx_before['gasPrice']
-

apps/cic-eth/tests/task/test_task_tx.py

@@ -4,27 +4,16 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
+from chainlib.eth.gas import Gas
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
         transaction,
         receipt,
-        raw,
         )
 from hexathon import strip_0x
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
-        set_reserved,
-        set_ready,
-        set_sent,
-        )
-from chainqueue.db.enum import StatusBits
 
 # local imports
 from cic_eth.queue.tx import register_tx
@@ -71,6 +60,15 @@ def test_tx_send(
 
 
 def test_sync_tx(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+    pass
+
+
+def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,
         eth_rpc,
@@ -79,48 +77,31 @@ def test_sync_tx(
         celery_session_worker,
         ):
 
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    o = raw(tx_signed_raw_hex)
-    r = eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
+    register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
+    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
+    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
 
     s = celery.signature(
-            'cic_eth.eth.tx.sync_tx',
+            'cic_eth.eth.gas.resend_with_higher_gas',
             [
                 tx_hash_hex,
                 default_chain_spec.asdict(),
                 ],
-            queue=None
+            queue=None,
             )
     t = s.apply_async()
     r = t.get_leaf()
-    assert t.successful() 
 
-    init_database.commit()
+    q = init_database.query(Otx)
+    q = q.filter(Otx.tx_hash==strip_0x(r))
+    otx = q.first()
+    if otx == None:
+        raise NotLocalTxError(r)
+
+    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
+    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
+    assert tx_after['gasPrice'] > tx_before['gasPrice']
 
-    o = Otx.load(tx_hash_hex, session=init_database)
-    assert o.status & StatusBits.FINAL == StatusBits.FINAL

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -1,170 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-import pytest
-import celery
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.nonce import (
-        RPCNonceOracle,
-        OverrideNonceOracle,
-        )
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        receipt,
-        )
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        set_ready,
-        )
-
-logg = logging.getLogger()
-
-
-def test_hashes_to_txs(
-        init_database,
-        default_chain_spec,
-        agent_roles,            
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex_one,
-            tx_signed_raw_hex_one,
-            session=init_database,
-            )
-
-    #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['ALICE'],
-            tx_hash_hex_two,
-            tx_signed_raw_hex_two,
-            session=init_database,
-            )
-
-    init_database.commit()
-
-    bogus_one = add_0x(os.urandom(32).hex())
-    bogus_two = add_0x(os.urandom(32).hex())
-
-    yarrgs = [
-                bogus_one,
-                tx_hash_hex_two,
-                bogus_two,
-                tx_hash_hex_one,
-            ]   
-    s = celery.signature(
-        'cic_eth.eth.tx.hashes_to_txs',
-        [
-            yarrgs,
-                ],
-        queue=None,
-        )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    assert len(r) == 2
-
-    logg.debug('r {}'.format(r))
-    txs = [
-        tx_signed_raw_hex_two,
-        tx_signed_raw_hex_one,
-            ]
-    for tx in r:
-        txs.remove(add_0x(tx))
-    assert len(txs) == 0
-
-
-
-def test_double_send(
-        init_database,
-        default_chain_spec,
-        agent_roles,            
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex_one,
-            tx_signed_raw_hex_one,
-            session=init_database,
-            )
-    set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['ALICE'],
-            tx_hash_hex_two,
-            tx_signed_raw_hex_two,
-            session=init_database,
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
-    init_database.commit()
-
-    yarrgs = [
-        tx_signed_raw_hex_one,
-        tx_signed_raw_hex_two,
-            ]
-    s = celery.signature(
-            'cic_eth.eth.tx.send',
-            [
-                yarrgs,
-                default_chain_spec.asdict(),
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    o = receipt(tx_hash_hex_one)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = receipt(tx_hash_hex_two)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-
-

apps/cic-eth/tests/testdata/Bogus.bin

@@ -1 +0,0 @@
-60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

apps/cic-eth/tests/testdata/Bogus.sol

@@ -1,10 +0,0 @@
-pragma solidity ^0.8.0;
-
-contract Bogus {
-
-	uint256 public a = 0;
-
-	function poke() public {
-		a++;
-	}
-}

apps/cic-eth/tests/unit/ext/test_address.py

@@ -19,7 +19,6 @@ def test_translate(
         agent_roles,
         cic_registry,
         init_celery_tasks,
-        register_lookups,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)

apps/cic-eth/tests/unit/test_registry_connect.py

@@ -1,22 +0,0 @@
-# local imports
-from cic_eth.registry import *
-
-def test_registry_connect(
-        eth_rpc,
-        default_chain_spec,
-        address_declarator,
-        token_registry,
-        contract_roles,
-        purge_lookups,
-        registry,
-        agent_roles,
-        ):
-
-    r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])
-

apps/cic-eth/tests/unit/test_stat.py

@@ -1,18 +0,0 @@
-# standard imports
-import datetime
-
-# local imports
-from cic_eth.stat import init_chain_stat
-
-
-def test_chain_stat(
-        eth_rpc,
-        init_eth_tester,
-        ):
-
-    now = int(datetime.datetime.now().timestamp()) + 1
-    for i in range(11):
-        init_eth_tester.time_travel(now + (i * 2))
-    
-    s = init_chain_stat(eth_rpc, block_start=0)
-    assert s.block_average() == 2

apps/cic-meta/bin/get.js

@@ -1,51 +0,0 @@
-#!/usr/bin/env node
-const colors = require('colors');
-const {Meta} = require("../dist");
-
-let { argv } = require('yargs')
-    .usage('Usage: $0 -m http://localhost:63380 -n publickeys')
-    .example(
-        '$0 -m http://localhost:63380 -n publickeys',
-        'Fetches the public keys blob from the meta server'
-    )
-    .option('m', {
-        alias: 'metaurl',
-        describe: 'The URL for the meta service',
-        demandOption: 'The meta url is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('n', {
-        alias: 'name',
-        describe: 'The name of the resource to be fetched from the meta service',
-        demandOption: 'The name of the resource is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('t', {
-        alias: 'type',
-        describe: 'The type of resource to be fetched from the meta service\n' +
-            'Options: `user`, `phone` and `custom`\n' +
-            'Defaults to `custom`',
-        type: 'string',
-        nargs: 1,
-    })
-    .epilog('Grassroots Economics (c) 2021')
-    .wrap(null);
-
-const metaUrl = argv.m;
-const resourceName = argv.n;
-let type = argv.t;
-if (type === undefined) {
-    type = 'custom'
-}
-
-(async () => {
-    const identifier = await Meta.getIdentifier(resourceName, type);
-    console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
-    const metaResponse = await Meta.get(identifier, metaUrl);
-    if (typeof metaResponse !== "object") {
-        console.error(colors.red('Metadata get failed!'));
-    }
-    console.log(colors.green(metaResponse));
-})();

apps/cic-meta/bin/set.js

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-const fs = require("fs");
-const colors = require('colors');
-const {Meta} = require("../dist");
-
-let { argv } = require('yargs')
-    .usage('Usage: $0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc')
-    .example(
-        '$0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc',
-        'Updates the public keys blob to the meta server'
-    )
-    .option('m', {
-        alias: 'metaurl',
-        describe: 'The URL for the meta service',
-        demandOption: 'The meta url is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('k', {
-        alias: 'privatekey',
-        describe: 'The PGP private key blob file used to sign the changes to the meta service',
-        demandOption: 'The private key file is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('n', {
-        alias: 'name',
-        describe: 'The name of the resource to be set or updated to the meta service',
-        demandOption: 'The name of the resource is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('r', {
-        alias: 'resource',
-        describe: 'The resource file to be set or updated to the meta service',
-        demandOption: 'The resource file is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('t', {
-        alias: 'type',
-        describe: 'The type of resource to be set or updated to the meta service\n' +
-            'Options: `user`, `phone` and `custom`\n' +
-            'Defaults to `custom`',
-        type: 'string',
-        nargs: 1,
-    })
-    .epilog('Grassroots Economics (c) 2021')
-    .wrap(null);
-
-const metaUrl = argv.m;
-const privateKeyFile = argv.k;
-const resourceName = argv.n;
-const resourceFile = argv.r;
-let type = argv.t;
-if (type === undefined) {
-    type = 'custom'
-}
-
-const privateKey = readFile(privateKeyFile);
-const resource = readFile(resourceFile);
-
-(async () => {
-    if (privateKey && resource) {
-        const identifier = await Meta.getIdentifier(resourceName, type);
-        console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
-        const meta = new Meta(metaUrl, privateKey);
-        meta.onload = async (status) => {
-            const response = await meta.set(identifier, resource)
-            console.log(colors.green(response));
-        }
-    }
-})();
-
-function readFile(filename) {
-    if(!fs.existsSync(filename)) {
-        console.log(colors.red(`File ${filename} not found`));
-        return;
-    }
-    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
-}

apps/cic-meta/package.json

@@ -1,14 +1,9 @@
 {
-	"name": "@cicnet/cic-client-meta",
-	"version": "0.0.11",
+	"name": "cic-client-meta",
+	"version": "0.0.7-alpha.8",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
-	"bin": {
-		"meta-set": "bin/set.js",
-		"meta-get": "bin/get.js"
-	},
-	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
@@ -16,14 +11,12 @@
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
 		"prepare": "npm run build && npm run build-server",
-		"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts",
-		"publish": "npm publish --access public"
+                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
 	},
 	"dependencies": {
-		"@cicnet/crdt-meta": "^0.0.10",
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
-		"colors": "^1.4.0",
+		"crdt-meta": "0.0.8",
 		"ethereumjs-wallet": "^1.0.1",
 		"ini": "^1.3.8",
 		"openpgp": "^4.10.8",

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-import { Config } from '@cicnet/crdt-meta';
+import { Config } from 'crdt-meta';
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {

apps/cic-meta/scripts/server/handlers.ts

@@ -1,7 +1,7 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
 
-import { Envelope, Syncable } from '@cicnet/crdt-meta';
+import { Envelope, Syncable } from 'crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {

apps/cic-meta/scripts/server/server.ts

@@ -3,8 +3,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import * as handlers from './handlers';
-import { PGPKeyStore, PGPSigner, Config } from '@cicnet/crdt-meta';
-import { SqliteAdapter, PostgresAdapter } from '../../src/db';
+import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
 
 import { standardArgs } from './args';
 

apps/cic-meta/src/custom.ts

@@ -1,27 +0,0 @@
-import {Addressable, mergeKey, Syncable} from "@cicnet/crdt-meta";
-
-class Custom extends Syncable implements Addressable {
-
-    name:	string
-    value:	Object
-
-    constructor(name:string, v:Object={}) {
-        super('', v);
-        Custom.toKey(name).then((cid) => {
-            this.id = cid;
-            this.value = v;
-        });
-    }
-
-    public static async toKey(item:string, identifier: string = ':cic.custom') {
-        return await mergeKey(Buffer.from(item), Buffer.from(identifier));
-    }
-
-    public key(): string {
-        return this.id;
-    }
-}
-
-export {
-    Custom,
-}

apps/cic-meta/src/db.ts

@@ -1,90 +0,0 @@
-import * as pg from 'pg';
-import * as sqlite from 'sqlite3';
-
-type DbConfig = {
-	name:		string
-	host:		string
-	port:		number
-	user:		string
-	password:	string	
-}
-
-interface DbAdapter {
-	query:	(s:string, callback:(e:any, rs:any) => void) => void
-	close: 	() => void
-}
-
-const re_creatematch = /^(CREATE)/i
-const re_getmatch = /^(SELECT)/i;
-const re_setmatch = /^(INSERT|UPDATE)/i;
-
-class SqliteAdapter implements DbAdapter {
-
-	db:		any
-
-	constructor(dbConfig:DbConfig, callback?:(any) => void) {
-		this.db = new sqlite.Database(dbConfig.name); //, callback);
-	}
-
-	public query(s:string, callback:(e:any, rs?:any) => void): void {
-		const local_callback = (e, rs) => {
-			let r = undefined;
-			if (rs !== undefined) {
-				r = {
-					rowCount: rs.length,
-					rows: rs,
-				}
-			}
-			callback(e, r);
-		};
-		if (s.match(re_getmatch)) {
-			this.db.all(s, local_callback);
-		} else if (s.match(re_setmatch)) {
-			this.db.run(s, local_callback);
-		} else if (s.match(re_creatematch)) {
-			this.db.run(s, callback);
-		} else {
-			throw 'unhandled query';
-		}
-	}
-
-	public close() {
-		this.db.close();
-	}
-}
-
-class PostgresAdapter implements DbAdapter {
-
-	db:		any
-
-	constructor(dbConfig:DbConfig) {
-		let o = dbConfig;
-		o['database'] = o.name;
-		this.db = new pg.Pool(o);
-		return this.db;
-	}
-
-	public query(s:string, callback:(e:any, rs:any) => void): void {
-		this.db.query(s, (e, rs) => {
-			let r = {
-				length: rs.rowCount,
-			}
-			rs.length = rs.rowCount;
-			if (e === undefined) {
-				e = null;
-			}
-			console.debug(e, rs);
-			callback(e, rs);
-		});
-	}
-
-	public close() {
-		this.db.end();
-	}
-}
-
-export {
-	DbConfig,
-	SqliteAdapter,
-	PostgresAdapter,
-}

apps/cic-meta/src/index.ts

@@ -1,4 +1,2 @@
 export { User } from './user';
 export { Phone } from './phone';
-export { Custom } from './custom';
-export { Meta } from './meta';

apps/cic-meta/src/meta.ts

@@ -1,126 +0,0 @@
-import {ArgPair, Envelope, Syncable, MutablePgpKeyStore, PGPSigner} from "@cicnet/crdt-meta";
-import {User} from "./user";
-import {Phone} from "./phone";
-import {Custom} from "./custom";
-const fetch = require("node-fetch");
-
-const headers =  {
-    'Content-Type': 'application/json;charset=utf-8',
-    'x-cic-automerge': 'client'
-};
-const options = {
-    headers: headers,
-};
-
-class Meta {
-    keystore: MutablePgpKeyStore = new MutablePgpKeyStore();
-    signer: PGPSigner = new PGPSigner(this.keystore);
-    metaUrl: string;
-    private privateKey: string;
-    onload: (status: boolean) => void;
-
-    constructor(metaUrl: string, privateKey: any) {
-        this.metaUrl = metaUrl;
-        this.privateKey = privateKey;
-        this.keystore.loadKeyring().then(() => {
-            this.keystore.importPrivateKey(privateKey).then(() => this.onload(true));
-        });
-    }
-
-    async set(identifier: string, data: Object): Promise<any> {
-        let syncable: Syncable;
-        const response = await Meta.get(identifier, this.metaUrl);
-        if (response === `Request to ${this.metaUrl}/${identifier} failed. Connection error.`) {
-            return response;
-        } else if (typeof response !== "object" || typeof data !== "object") {
-            syncable = new Syncable(identifier, data);
-            const res = await this.updateMeta(syncable, identifier);
-            return `${res.status}: ${res.statusText}`;
-        } else {
-            syncable = await Meta.get(identifier, this.metaUrl);
-            let update: Array<ArgPair> = [];
-            for (const prop in data) {
-                update.push(new ArgPair(prop, data[prop]));
-            }
-            syncable.update(update, 'client-branch');
-            const res = await this.updateMeta(syncable, identifier);
-            return `${res.status}: ${res.statusText}`;
-        }
-    }
-
-    async updateMeta(syncable: Syncable, identifier: string): Promise<any> {
-        const envelope: Envelope = await this.wrap(syncable);
-        const reqBody: string = envelope.toJSON();
-        const putOptions = {
-            method: 'PUT',
-            headers: headers,
-            body: reqBody
-        };
-        return await fetch(`${this.metaUrl}/${identifier}`, putOptions).then(async response => {
-            if (response.ok) {
-                return Promise.resolve({
-                    status: response.status,
-                    statusText: response.statusText + ', Metadata updated successfully!'
-                });
-            } else {
-                return Promise.reject({
-                    status: response.status,
-                    statusText: response.statusText
-                });
-            }
-        });
-    }
-
-    static async get(identifier: string, metaUrl: string): Promise<any> {
-        const response = await fetch(`${metaUrl}/${identifier}`, options).then(response => {
-            if (response.ok) {
-                return (response.json());
-            } else {
-                return Promise.reject({
-                    status: response.status,
-                    statusText: response.statusText
-                });
-            }
-        }).catch(error => {
-            if (error.code === 'ECONNREFUSED') {
-                return `Request to ${metaUrl}/${identifier} failed. Connection error.`
-            }
-            return `${error.status}: ${error.statusText}`;
-        });
-        if (typeof response !== "object") {
-            return response;
-        }
-        return Envelope.fromJSON(JSON.stringify(response)).unwrap();
-    }
-
-    static async getIdentifier(name: string, type: string = 'custom'): Promise<string> {
-        let identifier: string;
-        type = type.toLowerCase();
-        if (type === 'user') {
-            identifier = await User.toKey(name);
-        } else if (type === 'phone') {
-            identifier = await Phone.toKey(name);
-        } else {
-            identifier = await Custom.toKey(name);
-        }
-        return identifier;
-    }
-
-    private wrap(syncable: Syncable): Promise<Envelope> {
-        return new Promise<Envelope>(async (resolve, reject) => {
-            syncable.setSigner(this.signer);
-            syncable.onwrap = async (env) => {
-                if (env === undefined) {
-                    reject();
-                    return;
-                }
-                resolve(env);
-            };
-            syncable.sign();
-        });
-    }
-}
-
-export {
-    Meta,
-}

apps/cic-meta/src/phone.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, mergeKey } from '@cicnet/crdt-meta';
+import { Syncable, Addressable, mergeKey } from 'crdt-meta';
 
 class Phone extends Syncable implements Addressable {
 

apps/cic-meta/src/user.ts

@@ -1,4 +1,4 @@
-import { Syncable, Addressable, toAddressKey } from '@cicnet/crdt-meta';
+import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
 
 const keySalt = new TextEncoder().encode(':cic.person');
 class User extends Syncable implements Addressable {

apps/cic-meta/tests/server.ts

@@ -4,8 +4,7 @@ import pgp = require('openpgp');
 import sqlite = require('sqlite3');
 
 import * as handlers from '../scripts/server/handlers';
-import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer } from '@cicnet/crdt-meta';
-import { SqliteAdapter } from '../src/db';
+import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
 
 function createKeystore() {
 	const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

apps/cic-meta/tsconfig.json

@@ -19,7 +19,6 @@
   "include": [
     "src/**/*",
     "scripts/server/*",
-    "index.ts",
-    "bin"
+    "index.ts"
   ]
 }

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b14
-cic-eth~=0.11.0b15
+cic_base[full_graph]~=0.1.2b2
+cic-eth~=0.11.0b9
 cic-notify~=0.4.0a5
 cic-types~=0.1.0a10

apps/contract-migration/docker/Dockerfile

@@ -54,8 +54,8 @@ ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 ARG pip_extra_args=""
 ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2b15
-ARG cic_eth_version=0.11.0b16
+ARG cic_base_version=0.1.2b11
+ARG cic_eth_version=0.11.0b14
 ARG sarafu_token_version=0.0.1a8
 ARG sarafu_faucet_version=0.0.3a3
 RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
@@ -87,6 +87,7 @@ COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
+COPY contract-migration/scripts scripts/ 
 
 # A shared output dir for environment configs
 RUN mkdir -p /tmp/cic/config

apps/contract-migration/scripts/README.md

@@ -89,7 +89,12 @@ After this step is run, you can find top-level ethereum addresses (like the cic
 
 
 #### Custodial provisions
-
+response_data = send_ussd_request(address, self.data_dir)
+        state = response_data[:3]
+        out = response_data[4:]
+        m = '{} {}'.format(state, out[:7])
+        if m != 'CON Welcome':
+            raise VerifierError(response_data, 'ussd')
 This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
 
 `RUN_MASK=2 docker-compose up contract-migration`
@@ -123,16 +128,12 @@ The keystore file used for transferring external opening balances tracker is rel
 
 All external balance transactions are saved in raw wire format in `<datadir>/txs`, with transaction hash as file name.
 
-If the contract migrations have been executed with the default "giftable" token contract, then the `token_symbol` in the `import_balance` scripts should be set to `GFT`.
-
 
 
 #### Alternative 1 - Sovereign wallet import - `eth` 
 
 
-First, make a note of the **block height** before running anything:
-
-`eth-info -p http://localhost:63545`
+First, make a note of the **block height** before running anything.
 
 To import, run to _completion_:
 
@@ -142,7 +143,7 @@ After the script completes, keystore files for all generated accouts will be fou
 
 Then run:
 
-`python eth/import_balance.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
+`python eth/import_balance.py -v -c config -r <cic_registry_address> -p <eth_provider> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
 
 
 
@@ -150,7 +151,7 @@ Then run:
 
 Run in sequence, in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
 
 In another terminal:
 
@@ -167,40 +168,14 @@ If you have previously run the `cic_ussd` import incompletely, it could be a goo
 
 Then, in sequence, run in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
 
 In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
 
-
-### Step 4 - Metadata import (optional)
-
-The metadata import scripts can be run at any time after step 1 has been completed.
-
-
-#### Importing user metadata
-
-To import the main user metadata structs, run:
-
-`node cic_meta/import_meta.js <datadir> <number_of_users>`
-
-Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service.
-
-If _number of users_ is omitted the script will run until manually interrupted.
-
-
-
-#### Importing phone pointer
-
-`node cic_meta/import_meta_phone.js <datadir> <number_of_users>`
-
-If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
-
-
 ##### Importing pins and ussd data (optional)
-
 Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
 
 In one terminal run:
@@ -224,6 +199,29 @@ The balance script is a celery task worker, and will not exit by itself in its c
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
 
 
+### Step 4 - Metadata import (optional)
+
+The metadata import scripts can be run at any time after step 1 has been completed.
+
+
+#### Importing user metadata
+
+To import the main user metadata structs, run:
+
+`node cic_meta/import_meta.js <datadir> <number_of_users>`
+
+Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service.
+
+If _number of users_ is omitted the script will run until manually interrupted.
+
+
+#### Importing phone pointer
+
+`node cic_meta/import_meta_phone.js <datadir> <number_of_users>`
+
+If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
+
+
 ### Step 5 - Verify
 
 `python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>` 

apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py

@@ -163,9 +163,9 @@ class TrafficProvisioner:
     """Aux parameter template to be passed to the traffic generator module"""
 
 
-    def __init__(self, conn):
-        self.tokens = self.oracles['token'].get_tokens(conn)
-        self.accounts = self.oracles['account'].get_accounts(conn)
+    def __init__(self):
+        self.tokens = self.oracles['token'].get_tokens()
+        self.accounts = self.oracles['account'].get_accounts()
         self.aux = copy.copy(self.default_aux)
         self.__balances = {}
         for a in self.accounts:
@@ -277,14 +277,13 @@ class TrafficSyncHandler:
     :type traffic_router: TrafficRouter
     :raises Exception: Any Exception redis may raise on connection attempt.
     """
-    def __init__(self, config, traffic_router, conn):
+    def __init__(self, config, traffic_router):
         self.traffic_router = traffic_router
         self.redis_channel = str(uuid.uuid4())
         self.pubsub = self.__connect_redis(self.redis_channel, config)
         self.traffic_items = {}
         self.config = config
         self.init = False
-        self.conn = conn
 
 
     # connects to redis
@@ -308,7 +307,7 @@ class TrafficSyncHandler:
         :param tx_index: Syncer block transaction index at time of call.
         :type tx_index: number
         """
-        traffic_provisioner = TrafficProvisioner(self.conn)
+        traffic_provisioner = TrafficProvisioner()
         traffic_provisioner.add_aux('redis_channel', self.redis_channel)
 
         refresh_accounts = None
@@ -344,7 +343,7 @@ class TrafficSyncHandler:
             sender = traffic_provisioner.accounts[sender_index]
             #balance_full = balances[sender][token_pair[0].symbol()]
             if len(sender_indices) == 1:
-                sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
+                sender_indices[m] = sender_sender_indices[len(senders)-1]
             sender_indices = sender_indices[:len(sender_indices)-1]
 
             balance_full = traffic_provisioner.balance(sender, token_pair[0])
@@ -352,14 +351,7 @@ class TrafficSyncHandler:
             recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
             recipient = traffic_provisioner.accounts[recipient_index]
           
-            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
-                traffic_item,
-                token_pair,
-                sender,
-                recipient,
-                balance_full,
-                )
-                )
+            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
             (e, t, balance_result,) = traffic_item.method(
                     token_pair,
                     sender,
@@ -367,6 +359,7 @@ class TrafficSyncHandler:
                     balance_full,
                     traffic_provisioner.aux,
                     block_number,
+                    tx_index,
                     )
             traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
             sender_indices.append(recipient_index)

apps/contract-migration/scripts/cic_eth/traffic/common/registry.py

@@ -3,7 +3,7 @@ import logging
 import copy
 
 # external imports
-from cic_registry.registry import Registry
+from cic_registry import CICRegistry
 from eth_token_index import TokenUniqueSymbolIndex
 from eth_accounts_index import AccountRegistry
 from chainlib.chain import ChainSpec

apps/contract-migration/scripts/cic_eth/traffic/common/signer.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports 
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from crypto_dev_signer.keystore import DictKeystore
 
 logg = logging.getLogger(__name__)
 

apps/contract-migration/scripts/cic_eth/traffic/local/account.py

@@ -11,7 +11,7 @@ queue = 'cic-eth'
 name = 'account'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Triggers creation and registration of new account through the custodial cic-eth component.
 
     It expects the following aux parameters to exist:

apps/contract-migration/scripts/cic_eth/traffic/local/noop.py

@@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
 
     If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@@ -26,10 +26,12 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     :type aux: dict
     :param block_number: Syncer block number position at time of method call
     :type block_number: number
+    :param tx_index: Syncer block transaction index position at time of method call
+    :type tx_index: number
     :raises KeyError: Missing required aux element
     :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
     :rtype: tuple
     """
-    logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
+    logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
 
     return (None, None, sender_balance, )

apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py

@@ -12,7 +12,7 @@ queue = 'cic-eth'
 name = 'erc20_transfer'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
 
     It expects the following aux parameters to exist:
@@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     balance_units = int(sender_balance_value / decimals)
 
     if balance_units <= 0:
-        return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
+        return (AttributeError('sender {} has zero balance'), None, 0,)
 
     spend_units = random.randint(1, balance_units)
     spend_value = spend_units * decimals

apps/contract-migration/scripts/cic_eth/traffic/traffic.py

@@ -0,0 +1,101 @@
+# standard imports
+import os
+import logging
+import re
+import sys
+import json
+
+# external imports
+import redis
+import celery
+from chainsyncer.backend import MemBackend
+from chainsyncer.driver import HeadSyncer
+from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.gas import DefaultGasOracle
+from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.block import block_latest
+from hexathon import strip_0x
+
+# local imports
+import common
+from cmd.traffic import (
+        TrafficItem,
+        TrafficRouter,
+        TrafficProvisioner,
+        TrafficSyncHandler,
+        )
+from cmd.traffic import add_args as add_traffic_args
+
+
+# common basics
+script_dir = os.path.realpath(os.path.dirname(__file__))
+logg = common.log.create()
+argparser = common.argparse.create(script_dir, common.argparse.full_template)
+argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
+args = common.argparse.parse(argparser, logg)
+config = common.config.create(args.c, args, args.env_prefix)
+
+# map custom args to local config entries
+batchsize = args.batch_size
+if batchsize < 1:
+    batchsize = 1
+logg.info('batch size {}'.format(batchsize))
+config.add(batchsize, '_BATCH_SIZE', True)
+
+config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
+config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
+
+config.add(args.y, '_KEYSTORE_FILE', True)
+
+config.add(args.q, '_CELERY_QUEUE', True)
+
+common.config.log(config)
+
+
+def main():
+    # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
+    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
+
+    # connect to celery
+    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+    # set up registry
+    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    registry = common.registry.init_legacy(config, w3)
+
+    # Connect to blockchain with chainlib
+    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = DefaultGasOracle(conn)
+    nonce_oracle = DefaultNonceOracle(signer_address, conn)
+
+    # Set up magic traffic handler
+    traffic_router = TrafficRouter()
+    traffic_router.apply_import_dict(config.all(), config)
+    handler = TrafficSyncHandler(config, traffic_router)
+
+    # Set up syncer
+    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
+    o = block_latest()
+    r = conn.do(o)
+    block_offset = int(strip_0x(r), 16) + 1
+    syncer_backend.set(block_offset, 0)
+
+    # Set up provisioner for common task input data
+    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.default_aux = {
+            'chain_spec': config.get('CIC_CHAIN_SPEC'),
+            'registry': registry,
+            'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
+            'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
+            'redis_db': config.get('REDIS_DB'),
+            'api_queue': config.get('_CELERY_QUEUE'),
+            }
+
+    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
+    syncer.add_filter(handler)
+    syncer.loop(1, conn)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/cic_meta/import_meta.js

@@ -2,8 +2,8 @@ const fs = require('fs');
 const path = require('path');
 const http = require('http');
 
-const cic = require('@cicnet/cic-client-meta');
-const crdt = require('@cicnet/crdt-meta');
+const cic = require('cic-client-meta');
+const crdt = require('crdt-meta');
 
 //const conf = JSON.parse(fs.readFileSync('./cic.conf'));
 

apps/contract-migration/scripts/cic_meta/import_meta_custom.js

@@ -2,12 +2,12 @@ const fs = require('fs');
 const path = require('path');
 const http = require('http');
 
-const cic = require('@cicnet/cic-client-meta');
-const crdt = require('@cicnet/crdt-meta');
+const cic = require('cic-client-meta');
+const vcfp = require('vcard-parser');
 
 //const conf = JSON.parse(fs.readFileSync('./cic.conf'));
 
-const config = new crdt.Config('./config');
+const config = new cic.Config('./config'); 
 config.process();
 console.log(config);
 
@@ -42,18 +42,24 @@ function sendit(uid, envelope) {
 }
 
 function doOne(keystore, filePath, identifier) {
-	const signer = new crdt.PGPSigner(keystore);
+	const signer = new cic.PGPSigner(keystore);
 
 	const o = JSON.parse(fs.readFileSync(filePath).toString());
+	//const b = Buffer.from(j['vcard'], 'base64');
+	//const s = b.toString();
+	//const o = vcfp.parse(s);
+	//const phone = o.tel[0].value;
 
-	cic.Custom.toKey(identifier).then((uid) => {
-		const s = new crdt.Syncable(uid, o);
-		s.setSigner(signer);
-		s.onwrap = (env) => {
-			sendit(identifier, env);
-		};
-		s.sign();
-	});
+	//cic.Phone.toKey(phone).then((uid) => {
+	//const o = fs.readFileSync(filePath, 'utf-8');
+
+	const s = new cic.Syncable(identifier, o);
+	s.setSigner(signer);
+	s.onwrap = (env) => {
+		sendit(identifier, env);
+	};
+	s.sign();
+	//});
 }
 
 const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
@@ -61,7 +67,7 @@ const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_P
 pk = fs.readFileSync(privateKeyPath);
 pubk = fs.readFileSync(publicKeyPath);
 
-new crdt.PGPKeyStore(
+new cic.PGPKeyStore(
 	config.get('PGP_PASSPHRASE'),
 	pk,
 	pubk,
@@ -88,7 +94,7 @@ function importMetaCustom(keystore) {
 		err, files = fs.readdirSync(workDir);
 	} catch {
 		console.error('source directory not yet ready', workDir);
-		setTimeout(importMetaCustom, batchDelay, keystore);
+		setTimeout(importMetaPhone, batchDelay, keystore);
 		return;
 	}
 	let limit = batchSize;
@@ -122,7 +128,7 @@ function importMetaCustom(keystore) {
 		if (batchCount == batchSize) {
 			console.debug('reached batch size, breathing');
 			batchCount=0;
-			setTimeout(importMetaCustom, batchDelay, keystore);
+			setTimeout(importMeta, batchDelay, keystore);
 			return;
 		}
 	}

apps/contract-migration/scripts/cic_meta/import_meta_phone.js

@@ -2,9 +2,9 @@ const fs = require('fs');
 const path = require('path');
 const http = require('http');
 
-const cic = require('@cicnet/cic-client-meta');
+const cic = require('cic-client-meta');
 const vcfp = require('vcard-parser');
-const crdt = require('@cicnet/crdt-meta');
+const crdt = require('crdt-meta');
 
 //const conf = JSON.parse(fs.readFileSync('./cic.conf'));