Merge branch 'master' into lash/update-contracts-in-migration-2

.gitignore

@@ -8,8 +8,3 @@ gmon.out
 *.egg-info
 dist/
 build/
-**/*sqlite
-**/.nyc_output
-**/coverage
-**/.venv
-.idea

.gitlab-ci.yml

@@ -6,7 +6,6 @@ include:
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/cic_cache/cache.py

@@ -125,7 +125,6 @@ class DataCache(Cache):
                 'to_value': int(r['to_value']),
                 'source_token': r['source_token'],
                 'destination_token': r['destination_token'],
-                'success': r['success'],
                 'tx_type': tx_type,
             }
 

apps/cic-cache/cic_cache/db/list.py

@@ -42,7 +42,7 @@ def list_transactions_mined_with_data(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
 
     r = session.execute(s)
     return r

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -89,7 +89,7 @@ def process_transactions_all_data(session, env):
 
     offset = r[1]
     end = r[2]
-    if int(r[2]) < int(r[1]):
+    if r[2] < r[1]:
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,7 +16,6 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -29,8 +28,10 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -112,10 +113,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
+        syncers.append(HistorySyncer(syncer_backend))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
+    syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[syncer]
-loop_interval = 1

apps/cic-cache/requirements.txt

@@ -1,13 +1,12 @@
-cic-base==0.1.3a3+build.984b5cff
+cic-base~=0.1.2b10
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/scripts/migrate.py

@@ -2,7 +2,6 @@
 import os
 import argparse
 import logging
-import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -24,8 +23,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -56,10 +53,4 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,6 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
-sarafu-faucet~=0.0.4a1
+web3==5.12.2
+cic-eth-registry~=0.5.5a3
+cic-base[full]==0.1.2b8

apps/cic-eth/.coveragerc

@@ -5,5 +5,3 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
-	cic_eth/queue/state.py
-	*redis*.py

apps/cic-eth/.gitlab-ci.yml

@@ -3,29 +3,20 @@
         APP_NAME: cic-eth
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
 
-.cic_eth_mr_changes_target:
+.cic_eth_changes_target:
     rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - $CONTEXT/$APP_NAME/**/*
-      when: always
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-eth:
     extends:
+        - .cic_eth_changes_target
+        - .py_build_merge_request
         - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-        - .py_build_target_test
-
-test-mr-cic-eth:
-    extends:
-        - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-    stage: test
-    image: $IMAGE_TAG_BASE
-    script:
-        - cd apps/$APP_NAME/
-        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
 
 build-push-cic-eth:
     extends:
         - .py_build_push
+        - .cic_eth_variables
+        
+

apps/cic-eth/MANIFEST.in

@@ -1,2 +0,0 @@
-include *requirements.txt
-

apps/cic-eth/admin_requirements.txt

@@ -1,5 +0,0 @@
-SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a1
-hexathon~=0.0.1a7
-chainqueue~=0.0.2b5
-eth-erc20==0.0.10a2

apps/cic-eth/cic_eth/admin/nonce.py

@@ -4,18 +4,11 @@ import logging
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
-from chainlib.eth.tx import (
-        unpack,
-        TxFactory,
-        )
-from chainlib.eth.gas import OverrideGasOracle
-from chainqueue.sql.query import get_tx
-from chainqueue.sql.state import set_cancel
+from chainlib.eth.tx import unpack
+from chainqueue.query import get_tx
+from chainqueue.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import strip_0x
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -28,14 +21,13 @@ from cic_eth.admin.ctrl import (
         )
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
-from cic_eth.task import BaseTask
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True, base=BaseTask)
-def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
+@celery_app.task(bind=True)
+def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -46,29 +38,25 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
-    chain_spec = ChainSpec.from_dict(chainspec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    session = BaseTask.session_func()
-    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    tx_brief = get_tx(tx_hash_orig_hex)
+    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:]))
     tx = unpack(tx_raw, chain_spec)
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
 
-    lock_queue(None, chain_spec.asdict(), address=address)
-    lock_send(None, chain_spec.asdict(), address=address)
-
-    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
+    lock_queue(None, chain_str, address)
+    lock_send(None, chain_str, address)
 
+    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==address)
@@ -81,57 +69,49 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
-        tx_new = snake_and_camel(tx_new)
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        tx_new['gas_price'] += 1
-        tx_new['gasPrice'] = tx_new['gas_price']
-        tx_new['nonce'] -= delta
-
-        logg.debug('tx_new {}'.format(tx_new))
-
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
-        del(tx_new['hashUnsigned'])
+        tx_new['nonce'] -= delta
 
-        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
-        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
+        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            tx_new['nonce'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            )
+            nonce=tx_new['nonce'],
+            address=tx_new['from'],
+            tx_hash=tx_hash_hex,
+            signed_tx=tx_signed_raw_hex,
+                )
         session.add(otx)
+        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
+        set_cancel(tx_previous_hash_hex, True)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
-        session.commit()
 
     session.close()
 
-    s = create_check_gas_task(
+    s = create_check_gas_and_send_task(
          txs, 
-         chain_spec,
+         chain_str,
          tx_new['from'],
-         gas=tx_new['gas'],
-         tx_hashes_hex=tx_hashes, 
-         queue=queue,
+         tx_new['gas'],
+         tx_hashes, 
+         queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_spec.asdict(),
+            chain_str,
             tx_new['from'],
             ],
         queue=queue,
@@ -139,7 +119,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_spec.asdict(),
+            chain_str,
             tx_new['from'],
             ],
         queue=queue,

apps/cic-eth/cic_eth/api/__init__.py

@@ -5,3 +5,4 @@
 """
 
 from .api_task import Api
+from .api_admin import AdminApi

apps/cic-eth/cic_eth/api/api_admin.py

@@ -8,7 +8,6 @@ from chainlib.eth.constant import (
         ZERO_ADDRESS,
         )
 from cic_eth_registry import CICRegistry
-from cic_eth_registry.erc20 import ERC20Token
 from cic_eth_registry.error import UnknownContractError
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.contract import code
@@ -31,14 +30,13 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx
 
 app = celery.current_app
 
@@ -190,7 +188,6 @@ class AdminApi:
         s_manual = celery.signature(
             'cic_eth.queue.state.set_manual',
             [
-                chain_spec.asdict(),
                 tx_hash_hex,
                 ],
             queue=self.queue,
@@ -209,9 +206,8 @@ class AdminApi:
             s.link(s_gas)
 
         return s_manual.apply_async()
-                    
-
-    def check_nonce(self, chain_spec, address):
+                        
+    def check_nonce(self, address):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -232,12 +228,13 @@ class AdminApi:
             s_get_tx = celery.signature(
                     'cic_eth.queue.query.get_tx',
                     [
-                        chain_spec.asdict(),
+                    chain_spec.asdict(),
                         k,
                         ],
                     queue=self.queue,
                     )
             tx = s_get_tx.apply_async().get()
+            #tx = get_tx(k)
             logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
             if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@@ -245,9 +242,7 @@ class AdminApi:
                 blocking_tx = k
                 blocking_nonce = nonce_otx
             elif nonce_otx - last_nonce > 1:
-                logg.debug('tx {}'.format(tx))
-                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
+                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from']))
                 blocking_tx = k
                 blocking_nonce = nonce_otx
                 break
@@ -261,13 +256,12 @@ class AdminApi:
                 'blocking': blocking_nonce,
             },
             'tx': {
-                'blocking': add_0x(blocking_tx),
+                'blocking': blocking_tx,
+                }
             }
-        }
 
 
-    # TODO: is risky since it does not validate that there is actually a nonce problem?
-    def fix_nonce(self, chain_spec, address, nonce):
+    def fix_nonce(self, address, nonce, chain_spec):
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
                 [
@@ -281,17 +275,15 @@ class AdminApi:
         txs = s.apply_async().get()
 
         tx_hash_hex = None
-        session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx(k)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
-        session.close()
 
         s_nonce = celery.signature(
                 'cic_eth.admin.nonce.shift_nonce',
                 [
-                    chain_spec.asdict(),
+                    self.rpc.chain_spec.asdict(),
                     tx_hash_hex, 
                 ],
                 queue=self.queue
@@ -396,13 +388,12 @@ class AdminApi:
     
         t = s.apply_async()
         tx = t.get()
-
+  
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
-            source_token_declaration = None
             if registry != None:
                 try:
-                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
+                    source_token = registry.by_address(tx['source_token'])
                 except UnknownContractError:
                     logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
             else:
@@ -415,21 +406,16 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                source_token_declaration = t.get()
-
-            if source_token_declaration != None:
-                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
-                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
-                logg.debug('source token set tup {}'.format(source_token))
-
+                source_token = t.get()
+                if source_token == None:
+                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
 
 
         destination_token = None
         if tx['destination_token'] != ZERO_ADDRESS:
-            destination_token_declaration = None
             if registry != None:
                 try:
-                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
+                    destination_token = registry.by_address(tx['destination_token'])
                 except UnknownContractError:
                     logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
             else:
@@ -442,10 +428,10 @@ class AdminApi:
                         queue=self.queue
                         )
                 t = s.apply_async()
-                destination_token_declaration = t.get()
-            if destination_token_declaration != None:
-                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
-                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
+                destination_token = t.get()
+                if destination_token == None:
+                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
+
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
@@ -557,19 +543,13 @@ class AdminApi:
                 if role != None:
                     tx['recipient_description'] = role
 
-        erc20_c = ERC20(chain_spec)
         if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol
-            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance(r)
+            tx['source_token_symbol'] = source_token.symbol()
+            tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call()
 
         if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol
-            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
-            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
+            tx['destination_token_symbol'] = destination_token.symbol()
+            tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
         tx['network_status'] = 'Not in node' 

apps/cic-eth/cic_eth/api/api_task.py

@@ -8,10 +8,11 @@ import logging
 
 # external imports 
 import celery
+from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 
 # local imports
-from cic_eth.enum import LockEnum
+from cic_eth.db.enum import LockEnum
 
 app = celery.current_app
 
@@ -73,156 +74,29 @@ class Api:
         return s_token.apply_async()
 
 
-#    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
-#
-#        :param from_address: Ethereum address of sender
-#        :type from_address: str, 0x-hex
-#        :param to_address: Ethereum address of receipient
-#        :type to_address: str, 0x-hex
-#        :param target_return: Estimated return from conversion
-#        :type  target_return: int
-#        :param minimum_return: The least value of destination token return to allow
-#        :type minimum_return: int
-#        :param from_token_symbol: ERC20 token symbol of token being converted
-#        :type from_token_symbol: str
-#        :param to_token_symbol: ERC20 token symbol of token to receive
-#        :type to_token_symbol: str
-#        :returns: uuid of root task
-#        :rtype: celery.Task
-#        """
-#        raise NotImplementedError('out of service until new DEX migration is done')
-#        s_check = celery.signature(
-#                'cic_eth.admin.ctrl.check_lock',
-#                [
-#                    [from_token_symbol, to_token_symbol],
-#                    self.chain_spec.asdict(),
-#                    LockEnum.QUEUE,
-#                    from_address,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce = celery.signature(
-#                'cic_eth.eth.nonce.reserve_nonce',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_tokens = celery.signature(
-#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-#                [
-#                    self.chain_str,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_convert = celery.signature(
-#                'cic_eth.eth.bancor.convert_with_default_reserve',
-#                [
-#                    from_address,
-#                    target_return,
-#                    minimum_return,
-#                    to_address,
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce.link(s_tokens)
-#        s_check.link(s_nonce)
-#        if self.callback_param != None:
-#            s_convert.link(self.callback_success)
-#            s_tokens.link(s_convert).on_error(self.callback_error)
-#        else:
-#            s_tokens.link(s_convert)
-#
-#        t = s_check.apply_async(queue=self.queue)
-#        return t
-#
-#
-#    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
-#        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
-#
-#        :param from_address: Ethereum address of sender
-#        :type from_address: str, 0x-hex
-#        :param target_return: Estimated return from conversion
-#        :type  target_return: int
-#        :param minimum_return: The least value of destination token return to allow
-#        :type minimum_return: int
-#        :param from_token_symbol: ERC20 token symbol of token being converted
-#        :type from_token_symbol: str
-#        :param to_token_symbol: ERC20 token symbol of token to receive
-#        :type to_token_symbol: str
-#        :returns: uuid of root task
-#        :rtype: celery.Task
-#        """
-#        raise NotImplementedError('out of service until new DEX migration is done')
-#        s_check = celery.signature(
-#                'cic_eth.admin.ctrl.check_lock',
-#                [
-#                    [from_token_symbol, to_token_symbol],
-#                    self.chain_spec.asdict(),
-#                    LockEnum.QUEUE,
-#                    from_address,
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce = celery.signature(
-#                'cic_eth.eth.nonce.reserve_nonce',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_tokens = celery.signature(
-#                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-#                [
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_convert = celery.signature(
-#                'cic_eth.eth.bancor.convert_with_default_reserve',
-#                [
-#                    from_address,
-#                    target_return,
-#                    minimum_return,
-#                    from_address,
-#                    self.chain_spec.asdict(),
-#                    ],
-#                queue=self.queue,
-#                )
-#        s_nonce.link(s_tokens)
-#        s_check.link(s_nonce)
-#        if self.callback_param != None:
-#            s_convert.link(self.callback_success)
-#            s_tokens.link(s_convert).on_error(self.callback_error)
-#        else:
-#            s_tokens.link(s_convert)
-#
-#        t = s_check.apply_async(queue=self.queue)
-#        return t
-
-
-    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
-        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
+    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 
         :param from_address: Ethereum address of sender
         :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of recipient
+        :param to_address: Ethereum address of receipient
         :type to_address: str, 0x-hex
-        :param value: Estimated return from conversion
-        :type  value: int
-        :param token_symbol: ERC20 token symbol of token to send
-        :type token_symbol: str
-        :param spender_address: Ethereum address of recipient
-        :type spender_address: str, 0x-hex
+        :param target_return: Estimated return from conversion
+        :type  target_return: int
+        :param minimum_return: The least value of destination token return to allow
+        :type minimum_return: int
+        :param from_token_symbol: ERC20 token symbol of token being converted
+        :type from_token_symbol: str
+        :param to_token_symbol: ERC20 token symbol of token to receive
+        :type to_token_symbol: str
         :returns: uuid of root task
         :rtype: celery.Task
         """
+        raise NotImplementedError('out of service until new DEX migration is done')
         s_check = celery.signature(
                 'cic_eth.admin.ctrl.check_lock',
                 [
-                    [token_symbol],
+                    [from_token_symbol, to_token_symbol],
                     self.chain_spec.asdict(),
                     LockEnum.QUEUE,
                     from_address,
@@ -233,7 +107,70 @@ class Api:
                 'cic_eth.eth.nonce.reserve_nonce',
                 [
                     self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_str,
+                    ],
+                queue=self.queue,
+                )
+        s_convert = celery.signature(
+                'cic_eth.eth.bancor.convert_with_default_reserve',
+                [
                     from_address,
+                    target_return,
+                    minimum_return,
+                    to_address,
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_convert.link(self.callback_success)
+            s_tokens.link(s_convert).on_error(self.callback_error)
+        else:
+            s_tokens.link(s_convert)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
+
+
+    def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
+        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param target_return: Estimated return from conversion
+        :type  target_return: int
+        :param minimum_return: The least value of destination token return to allow
+        :type minimum_return: int
+        :param from_token_symbol: ERC20 token symbol of token being converted
+        :type from_token_symbol: str
+        :param to_token_symbol: ERC20 token symbol of token to receive
+        :type to_token_symbol: str
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        raise NotImplementedError('out of service until new DEX migration is done')
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [from_token_symbol, to_token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
                     ],
                 queue=self.queue,
                 )
@@ -244,41 +181,29 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_allow = celery.signature(
-                'cic_eth.eth.erc20.check_allowance',
+        s_convert = celery.signature(
+                'cic_eth.eth.bancor.convert_with_default_reserve',
                 [
                     from_address,
-                    value,
+                    target_return,
+                    minimum_return,
+                    from_address,
                     self.chain_spec.asdict(),
-                    spender_address,
                     ],
                 queue=self.queue,
                 )
-        s_transfer = celery.signature(
-                'cic_eth.eth.erc20.transfer_from',
-                [
-                    from_address,
-                    to_address,
-                    value,
-                    self.chain_spec.asdict(),
-                    spender_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens.link(s_allow)
         s_nonce.link(s_tokens)
         s_check.link(s_nonce)
         if self.callback_param != None:
-            s_transfer.link(self.callback_success)
-            s_allow.link(s_transfer).on_error(self.callback_error)
+            s_convert.link(self.callback_success)
+            s_tokens.link(s_convert).on_error(self.callback_error)
         else:
-            s_allow.link(s_transfer)
+            s_tokens.link(s_convert)
 
         t = s_check.apply_async(queue=self.queue)
         return t
 
 
-
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/db/enum.py

@@ -1 +1,158 @@
-from cic_eth.enum import *
+# standard imports
+import enum
+
+
+@enum.unique
+class StatusBits(enum.IntEnum):
+    """Individual bit flags that are combined to define the state and legacy of a queued transaction
+
+    """
+    QUEUED = 0x01 # transaction should be sent to network
+    IN_NETWORK = 0x08 # transaction is in network
+   
+    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
+    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
+
+    LOCAL_ERROR = 0x100 # errors that originate internally from the component
+    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
+    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
+    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
+
+    FINAL = 0x1000 # transaction processing has completed
+    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
+    MANUAL = 0x8000 # transaction processing has been manually overridden
+
+
+@enum.unique
+class StatusEnum(enum.IntEnum):
+    """
+
+    - Inactive, not finalized. (<0)
+        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
+        * SENDFAIL: The transaction was not received by the node.
+        * RETRY: The transaction is queued for a new send attempt after previously failing.
+        * READYSEND: The transaction is queued for its first send attempt
+        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
+        * WAITFORGAS: The transaction is on hold pending gas funding.
+    - Active state: (==0)
+        * SENT: The transaction has been sent to the mempool.
+    - Inactive, finalized. (>0)
+        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
+        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
+        * OVERRIDDEN: Transaction has been manually overriden.
+        * REJECTED: The transaction was rejected by the node.
+        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
+        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
+
+    """
+    PENDING = 0
+
+    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
+    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
+    READYSEND = StatusBits.QUEUED
+
+    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
+
+    WAITFORGAS = StatusBits.GAS_ISSUES
+
+    SENT = StatusBits.IN_NETWORK
+    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
+    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
+    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
+
+    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
+    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
+    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
+
+
+@enum.unique
+class LockEnum(enum.IntEnum):
+    """
+    STICKY: When set, reset is not possible
+    CREATE: Disable creation of accounts
+    SEND: Disable sending to network
+    QUEUE: Disable queueing new or modified transactions
+    """
+    STICKY=1
+    INIT=2
+    CREATE=4
+    SEND=8
+    QUEUE=16
+    QUERY=32
+    ALL=int(0xfffffffffffffffe)
+
+
+def status_str(v, bits_only=False):
+    """Render a human-readable string describing the status
+
+    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
+
+    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
+
+    :param v: Status bit field
+    :type v: number
+    :param bits_only: Only render individual bit labels.
+    :type bits_only: bool
+    :returns: Status string
+    :rtype: str
+    """
+    s = ''
+    if not bits_only:
+        try:
+            s = StatusEnum(v).name
+            return s
+        except ValueError:
+            pass
+
+    if v == 0:
+        return 'NONE'
+
+    for i in range(16):
+        b = (1 << i)
+        if (b & 0xffff) & v:
+            n = StatusBits(b).name
+            if len(s) > 0:
+                s += ','
+            s += n
+    if not bits_only:
+        s += '*'
+    return s
+
+
+def all_errors():
+    """Bit mask of all error states
+
+    :returns: Error flags
+    :rtype: number
+    """
+    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
+
+
+def is_error_status(v):
+    """Check if value is an error state
+
+    :param v: Status bit field
+    :type v: number
+    :returns: True if error
+    :rtype: bool
+    """
+    return bool(v & all_errors())
+
+
+def dead():
+    """Bit mask defining whether a transaction is still likely to be processed on the network.
+
+    :returns: Bit mask
+    :rtype: number
+    """
+    return StatusBits.FINAL | StatusBits.OBSOLETE
+
+
+def is_alive(v):
+    """Check if transaction is still likely to be processed on the network.
+
+    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
+
+    :returns: 
+    """
+    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/db/util.py

@@ -0,0 +1,8 @@
+import math
+
+def num_serialize(n):
+    if n == 0:
+        return b'\x00'
+    binlog = math.log2(n)
+    bytelength = int(binlog / 8 + 1)
+    return n.to_bytes(bytelength, 'big')

apps/cic-eth/cic_eth/enum.py

@@ -1,158 +0,0 @@
-# standard imports
-import enum
-
-
-@enum.unique
-class StatusBits(enum.IntEnum):
-    """Individual bit flags that are combined to define the state and legacy of a queued transaction
-
-    """
-    QUEUED = 0x01 # transaction should be sent to network
-    IN_NETWORK = 0x08 # transaction is in network
-   
-    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
-    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
-
-    LOCAL_ERROR = 0x100 # errors that originate internally from the component
-    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
-    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
-    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
-
-    FINAL = 0x1000 # transaction processing has completed
-    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
-    MANUAL = 0x8000 # transaction processing has been manually overridden
-
-
-@enum.unique
-class StatusEnum(enum.IntEnum):
-    """
-
-    - Inactive, not finalized. (<0)
-        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
-        * SENDFAIL: The transaction was not received by the node.
-        * RETRY: The transaction is queued for a new send attempt after previously failing.
-        * READYSEND: The transaction is queued for its first send attempt
-        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
-        * WAITFORGAS: The transaction is on hold pending gas funding.
-    - Active state: (==0)
-        * SENT: The transaction has been sent to the mempool.
-    - Inactive, finalized. (>0)
-        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
-        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
-        * OVERRIDDEN: Transaction has been manually overriden.
-        * REJECTED: The transaction was rejected by the node.
-        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
-        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
-
-    """
-    PENDING = 0
-
-    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
-    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
-    READYSEND = StatusBits.QUEUED
-
-    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
-
-    WAITFORGAS = StatusBits.GAS_ISSUES
-
-    SENT = StatusBits.IN_NETWORK
-    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
-    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
-    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
-
-    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
-    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
-    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
-
-
-@enum.unique
-class LockEnum(enum.IntEnum):
-    """
-    STICKY: When set, reset is not possible
-    CREATE: Disable creation of accounts
-    SEND: Disable sending to network
-    QUEUE: Disable queueing new or modified transactions
-    """
-    STICKY=1
-    INIT=2
-    CREATE=4
-    SEND=8
-    QUEUE=16
-    QUERY=32
-    ALL=int(0xfffffffffffffffe)
-
-
-def status_str(v, bits_only=False):
-    """Render a human-readable string describing the status
-
-    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
-
-    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
-
-    :param v: Status bit field
-    :type v: number
-    :param bits_only: Only render individual bit labels.
-    :type bits_only: bool
-    :returns: Status string
-    :rtype: str
-    """
-    s = ''
-    if not bits_only:
-        try:
-            s = StatusEnum(v).name
-            return s
-        except ValueError:
-            pass
-
-    if v == 0:
-        return 'NONE'
-
-    for i in range(16):
-        b = (1 << i)
-        if (b & 0xffff) & v:
-            n = StatusBits(b).name
-            if len(s) > 0:
-                s += ','
-            s += n
-    if not bits_only:
-        s += '*'
-    return s
-
-
-def all_errors():
-    """Bit mask of all error states
-
-    :returns: Error flags
-    :rtype: number
-    """
-    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
-
-
-def is_error_status(v):
-    """Check if value is an error state
-
-    :param v: Status bit field
-    :type v: number
-    :returns: True if error
-    :rtype: bool
-    """
-    return bool(v & all_errors())
-
-
-def dead():
-    """Bit mask defining whether a transaction is still likely to be processed on the network.
-
-    :returns: Bit mask
-    :rtype: number
-    """
-    return StatusBits.FINAL | StatusBits.OBSOLETE
-
-
-def is_alive(v):
-    """Check if transaction is still likely to be processed on the network.
-
-    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
-
-    :returns: 
-    """
-    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/error.py

@@ -80,8 +80,3 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
-
-
-class YouAreBrokeError(Exception):
-    """Exception raised when a value transfer is attempted without access to sufficient funds
-    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,7 +24,6 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
-        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -72,117 +71,6 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
-@celery_app.task(bind=True)
-def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
-    """Best-effort verification that the allowance for a transfer from spend is sufficient.
-
-    :raises YouAreBrokeError: If allowance is insufficient
-    
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :return: Token list as passed to task
-    :rtype: dict
-    """
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    caller_address = ERC20Token.caller_address 
-    c = ERC20(chain_spec)
-    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
-    r = rpc.do(o)
-    allowance = c.parse_allowance(r)
-    if allowance < value:
-        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
-        logg.error(errstr)
-        raise YouAreBrokeError(errstr)
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
-def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
-    """Transfer ERC20 tokens between addresses
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Token receiver address
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    :return: Transaction hash for tranfer operation
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    queue = self.request.delivery_info.get('routing_key')
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
-
-    session = self.create_session()
-    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
-    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
-
-    rpc_signer.disconnect()
-    rpc.disconnect()
-
-    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
-
-    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
-    session.commit()
-    session.close()
-    
-    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
-    gas_budget = gas_pair[0] * gas_pair[1]
-    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
-
-    s = create_check_gas_task(
-             [tx_signed_raw_hex],
-             chain_spec,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return tx_hash_hex
-
-
-
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -344,7 +232,6 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
-            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -392,48 +279,6 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def cache_transfer_from_data(
-    tx_hash_hex,
-    tx_signed_raw_hex,
-    chain_spec_dict,
-        ):
-    """Helper function for otx_cache_transfer_from
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
-
-    tx_data = ERC20.parse_transfer_from_request(tx['data'])
-    spender_address = tx_data[0]
-    recipient_address = tx_data[1]
-    token_value = tx_data[2]
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)
-
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/eth/gas.py

@@ -57,12 +57,10 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-MAXIMUM_FEE_UNITS = 8000000
-
 class MaxGasOracle:
 
     def gas(code=None):
-        return MAXIMUM_FEE_UNITS
+        return 8000000
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -152,7 +150,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -172,30 +170,24 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
-
-    addresspass = None
     if len(txs) == 0:
-        addresspass = []
         for i in range(len(tx_hashes)):
-            o = get_tx(chain_spec_dict, tx_hashes[i])
+            o = get_tx(tx_hashes[i])
             txs.append(o['signed_tx'])
-            logg.debug('sender {}'.format(o))
-            tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
             if address == None:
-                address = tx['from']
-            elif address != tx['from']:
-                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
-            addresspass.append(address)
+                address = o['address']
 
+    #if not web3.Web3.isChecksumAddress(address):
     if not is_checksum_address(address):
         raise ValueError('invalid address {}'.format(address))
 
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
 
+    # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     gas_balance = 0
     try:
         o = balance(address)
@@ -206,9 +198,6 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
         conn.disconnect()
         raise EthError('gas_balance call for {}: {}'.format(address, e))
 
-    if gas_required == None:
-        gas_required = MAXIMUM_FEE_UNITS
-
     logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
     session = SessionBase.create_session()
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@@ -279,8 +268,7 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
             queue=queue,
             )
         ready_tasks.append(s)
-    t = celery.group(ready_tasks)()
-    logg.debug('group {}'.format(t))
+    celery.group(ready_tasks)()
 
     return txs
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -21,7 +21,6 @@ from chainqueue.db.models.tx import Otx
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.db import SessionBase
@@ -59,9 +58,6 @@ def hashes_to_txs(self, tx_hashes):
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
-    for i in range(len(tx_hashes)):
-        tx_hashes[i] = strip_0x(tx_hashes[i])
-
     queue = self.request.delivery_info['routing_key']
 
     session = SessionBase.create_session()
@@ -152,7 +148,7 @@ def send(self, txs, chain_spec_dict):
 
 @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
 def sync_tx(self, tx_hash_hex, chain_spec_dict):
-    """Force update of network status of a single transaction
+    """Force update of network status of a simgle transaction
 
     :param tx_hash_hex: Transaction hash
     :type tx_hash_hex: str, 0x-hex
@@ -177,14 +173,12 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
 
     # TODO: apply receipt in tx object to validate and normalize input
     if rcpt != None:
-        rcpt = snake_and_camel(rcpt)
         success = rcpt['status'] == 1
-        logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
+        logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success))
 
         s = celery.signature(
             'cic_eth.queue.state.set_final',
             [
-                chain_spec_dict,
                 tx_hash_hex,
                 rcpt['blockNumber'],
                 rcpt['transactionIndex'],
@@ -192,14 +186,12 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
                 ],
                 queue=queue,
             )
-    # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
     else:
         logg.debug('sync tx {} mempool'.format(tx_hash_hex))
 
         s = celery.signature(
             'cic_eth.queue.state.set_sent',
             [
-                chain_spec_dict,
                 tx_hash_hex,
                 ],
                 queue=queue,

apps/cic-eth/cic_eth/ext/tx.py

@@ -19,7 +19,7 @@ from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
-from chainqueue.sql.query import get_tx_cache
+from chainqueue.query import get_tx_cache
 from eth_erc20 import ERC20
 
 # local imports

apps/cic-eth/cic_eth/pytest/fixtures_contract.py

@@ -1,77 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import pytest
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        ABIContractType,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        TxFactory,
-        TxFormat,
-        unpack,
-        Tx,
-        )
-from hexathon import strip_0x
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-
-
-@pytest.fixture(scope='function')
-def bogus_tx_block(
-    default_chain_spec,
-    eth_rpc,
-    eth_signer,
-    contract_roles,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
-
-    f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
-    bytecode = f.read()
-    f.close()
-
-    c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
-    tx = c.set_code(tx, bytecode)
-    (tx_hash_hex, o) = c.build(tx)
-
-    r = eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-
-    contract_address = r['contract_address']
-
-    enc = ABIContractEncoder()
-    enc.method('poke')
-    data = enc.get()
-    tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
-    tx = c.set_code(tx, data)
-    (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
-    r = eth_rpc.do(o)
-    tx_signed_raw_hex = strip_0x(o['params'][0])
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-
-    return (block, tx)

apps/cic-eth/cic_eth/pytest/testdata/Bogus.bin

@@ -1 +0,0 @@
-60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

apps/cic-eth/cic_eth/pytest/testdata/Bogus.sol

@@ -1,10 +0,0 @@
-pragma solidity ^0.8.0;
-
-contract Bogus {
-
-	uint256 public a = 0;
-
-	function poke() public {
-		a++;
-	}
-}

apps/cic-eth/cic_eth/queue/query.py

@@ -5,7 +5,7 @@ import datetime
 import celery
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.sql.query
+import chainqueue.query
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -28,7 +28,7 @@ celery_app = celery.current_app
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
+    r = chainqueue.query.get_tx_cache(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
+    r =  chainqueue.query.get_tx(chain_spec, tx_hash)
     session.close()
     return r
 
@@ -46,7 +46,7 @@ def get_tx(chain_spec_dict, tx_hash):
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
+    r = chainqueue.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
     session.close()
     return r
 
@@ -55,17 +55,17 @@ def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True,
 def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
+    r = chainqueue.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
+    return chainqueue.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
+    return chainqueue.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):

apps/cic-eth/cic_eth/queue/state.py

@@ -1,6 +1,6 @@
 # external imports
 from chainlib.chain import ChainSpec
-import chainqueue.sql.state
+import chainqueue.state
 
 # local imports
 import celery
@@ -14,7 +14,7 @@ celery_app = celery.current_app
 def set_sent(chain_spec_dict, tx_hash, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
+    r = chainqueue.state.set_sent(chain_spec, tx_hash, fail, session=session)
     session.close()
     return r
 
@@ -23,7 +23,7 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
+    r = chainqueue.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
     session.close()
     return r
 
@@ -32,7 +32,7 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
+    r = chainqueue.state.set_cancel(chain_spec, tx_hash, manual, session=session)
     session.close()
     return r
 
@@ -41,7 +41,7 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 def set_rejected(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_rejected(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -50,7 +50,7 @@ def set_rejected(chain_spec_dict, tx_hash):
 def set_fubar(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_fubar(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -59,7 +59,7 @@ def set_fubar(chain_spec_dict, tx_hash):
 def set_manual(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_manual(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -68,7 +68,7 @@ def set_manual(chain_spec_dict, tx_hash):
 def set_ready(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_ready(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -77,7 +77,7 @@ def set_ready(chain_spec_dict, tx_hash):
 def set_reserved(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_reserved(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -86,7 +86,7 @@ def set_reserved(chain_spec_dict, tx_hash):
 def set_waitforgas(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.set_waitforgas(chain_spec, tx_hash, session=session)
     session.close()
     return r
     
@@ -95,7 +95,7 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 def get_state_log(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
+    r = chainqueue.state.get_state_log(chain_spec, tx_hash, session=session)
     session.close()
     return r
 
@@ -104,6 +104,6 @@ def get_state_log(chain_spec_dict, tx_hash):
 def obsolete(chain_spec_dict, tx_hash, final):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
-    r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
+    r = chainqueue.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)
     session.close()
     return r

apps/cic-eth/cic_eth/queue/tx.py

@@ -15,14 +15,14 @@ from sqlalchemy import tuple_
 from sqlalchemy import func
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
-import chainqueue.sql.state
+import chainqueue.state
 from chainqueue.db.enum import (
         StatusEnum,
         StatusBits,
         is_alive,
         dead,
         )
-from chainqueue.sql.tx import create
+from chainqueue.tx import create
 from chainqueue.error import NotLocalTxError
 from chainqueue.db.enum import status_str
 

apps/cic-eth/cic_eth/registry.py

@@ -5,30 +5,29 @@ import logging
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
-from chainlib.eth.constant import ZERO_ADDRESS
 
 logg = logging.getLogger()
 
 
-def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
+def connect_token_registry(rpc, chain_spec):
     registry = CICRegistry(chain_spec, rpc)
-    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
+    token_registry_address = registry.by_name('TokenRegistry')
     logg.debug('using token registry address {}'.format(token_registry_address))
     lookup = TokenIndexLookup(chain_spec, token_registry_address)
     CICRegistry.add_lookup(lookup)
 
 
-def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
+def connect_declarator(rpc, chain_spec, trusted_addresses):
     registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    declarator_address = registry.by_name('AddressDeclarator')
     logg.debug('using declarator address {}'.format(declarator_address))
     lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
     CICRegistry.add_lookup(lookup)
 
 
-def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
+def connect(rpc, chain_spec, registry_address):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
+    registry_address = registry.by_name('ContractRegistry')
     return registry
 

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -12,7 +12,7 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.address import is_checksum_address
 
 # local imports
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import AdminApi
 from cic_eth.db.enum import LockEnum
 
 logging.basicConfig(level=logging.WARNING)

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -21,7 +21,7 @@ from chainqueue.db.enum import (
     StatusBits,
     )
 from chainqueue.error import NotLocalTxError
-from chainqueue.sql.state import set_reserved
+from chainqueue.state import set_reserved
 
 # local imports
 import cic_eth

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -72,9 +72,7 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        c = Faucet(self.chain_spec)
-
-        o = c.token(faucet_contract, sender_address=self.caller_address)
+        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,15 +10,14 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
+from chainqueue.query import get_paused_tx_cache as get_paused_tx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
 from .base import SyncFilter
 
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 
 class GasFilter(SyncFilter):
@@ -28,11 +27,11 @@ class GasFilter(SyncFilter):
         self.chain_spec = chain_spec
 
 
-    def filter(self, conn, block, tx, db_session):
+    def filter(self, conn, block, tx, session):
         if tx.value > 0:
             tx_hash_hex = add_0x(tx.hash)
             logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(db_session)
+            session = SessionBase.bind_session(session)
             q = session.query(TxCache.recipient)
             q = q.join(Otx)
             q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@@ -57,7 +56,7 @@ class GasFilter(SyncFilter):
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,
                 )
-                return s.apply_async()
+                s.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -50,8 +50,7 @@ class RegistrationFilter(SyncFilter):
                     queue=self.queue,
                     )
                 s_nonce.link(s_gift)
-                t = s_nonce.apply_async()
-                return t
+                s_nonce.apply_async()
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/filters/straggler.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from chainqueue.sql.state import obsolete_by_cache
+from chainqueue.state import obsolete_by_cache
 
 logg = logging.getLogger()
 

apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py

@@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
         self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
 
 
-    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
+    def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
 
         if tx.payload == None:
             logg.debug('no payload')
@@ -45,17 +45,16 @@ class TransferAuthFilter(SyncFilter):
             return False
 
         recipient = tx.inputs[0]
-        #if recipient != self.transfer_request_contract.address():
-        if recipient != self.transfer_request_contract:
+        if recipient != self.transfer_request_contract.address():
             logg.debug('not our transfer auth contract address {}'.format(recipient))
             return False
 
         r = TransferAuthorization.parse_create_request_request(tx.payload) 
-          
-        sender = r[0]
-        recipient = r[1]
-        token = r[2]
-        value = r[3]
+           
+        sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) 
+        recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
+        token = abi_decode_single(ABIContractType.ADDRESS, r[2]) 
+        value = abi_decode_single(ABIContractType.UINT256, r[3])
 
         token_data = {
             'address': token,
@@ -65,7 +64,6 @@ class TransferAuthFilter(SyncFilter):
             'cic_eth.eth.nonce.reserve_nonce',
             [
                 [token_data],
-                self.chain_spec.asdict(), 
                 sender,
                 ],
             queue=self.queue,
@@ -82,7 +80,7 @@ class TransferAuthFilter(SyncFilter):
             )
         s_nonce.link(s_approve)
         t = s_nonce.apply_async()
-        return t
+        return True
 
 
     def __str__(self):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -0,0 +1,136 @@
+# standard imports
+import os
+import re
+import logging
+import argparse
+import json
+
+# third-party imports
+import web3
+import confini
+import celery
+from json.decoder import JSONDecodeError
+from cic_registry.chain import ChainSpec
+
+# local imports
+from cic_eth.db import dsn_from_config
+from cic_eth.db.models.base import SessionBase
+from cic_eth.eth.util import unpack_signed_raw_tx
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_eth', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
+
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+args = argparser.parse_args()
+
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn)
+
+celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+queue = args.q
+
+re_something = r'^/something/?'
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+
+def process_something(session, env):
+    r = re.match(re_something, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    #if env.get('CONTENT_TYPE') != 'application/json':
+    #    raise AttributeError('content type')
+
+    #if env.get('REQUEST_METHOD') != 'POST':
+    #    raise AttributeError('method')
+
+    #post_data = json.load(env.get('wsgi.input'))
+    
+    #return ('text/plain', 'foo'.encode('utf-8'),)
+
+
+# uwsgi application
+def application(env, start_response):
+    
+    for k in env.keys():
+        logg.debug('env {} {}'.format(k, env[k]))
+
+    headers = []
+    content = b''
+    err = None
+
+    session = SessionBase.create_session()
+    for handler in [
+            process_something,
+            ]:
+        try:
+            r = handler(session, env)
+        except AttributeError as e:
+            logg.error('handler fail attribute {}'.format(e))
+            err = '400 Impertinent request'
+            break
+        except JSONDecodeError as e:
+            logg.error('handler fail json {}'.format(e))
+            err = '400 Invalid data format'
+            break
+        except KeyError as e:
+            logg.error('handler fail key {}'.format(e))
+            err = '400 Invalid JSON'
+            break
+        except ValueError as e:
+            logg.error('handler fail value {}'.format(e))
+            err = '400 Invalid data'
+            break
+        except RuntimeError as e:
+            logg.error('task fail value {}'.format(e))
+            err = '500 Task failed, sorry I cannot tell you more'
+            break
+        if r != None:
+            (mime_type, content) = r
+            break
+    session.close()
+
+    if err != None:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response(err, headers)
+        session.close()
+        return [content]
+
+    headers.append(('Content-Length', str(len(content))),)
+    headers.append(('Access-Control-Allow-Origin', '*',));
+
+    if len(content) == 0:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response('404 Looked everywhere, sorry', headers)
+    else:
+        headers.append(('Content-Type', mime_type,))
+        start_response('200 OK', headers)
+
+    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,7 +194,6 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
-    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,7 +15,6 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -27,8 +26,10 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -79,7 +80,6 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
-
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend, chain_interface))
+            syncers.append(HistorySyncer(syncer_backend))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend, chain_interface))
+            syncers.append(HeadSyncer(syncer_backend))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,8 +12,10 @@ import confini
 import celery
 
 # local imports
-from cic_eth.api import Api
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import (
+        Api,
+        AdminApi,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

apps/cic-eth/cic_eth/runnable/resend.py

@@ -11,7 +11,7 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
 
 # local imports
-from cic_eth.api.admin import AdminApi
+from cic_eth.api.api_admin import AdminApi
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

apps/cic-eth/cic_eth/runnable/tag.py

@@ -12,7 +12,7 @@ from chainlib.chain import ChainSpec
 from xdg.BaseDirectory import xdg_config_home
 
 # local imports
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import AdminApi
 from cic_eth.db import dsn_from_config
 from cic_eth.db.models.base import SessionBase
 

apps/cic-eth/cic_eth/runnable/view.py

@@ -19,7 +19,7 @@ from chainlib.eth.connection import EthHTTPConnection
 from hexathon import add_0x
 
 # local imports
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import AdminApi
 from cic_eth.db.enum import (
     StatusEnum,
     status_str,

apps/cic-eth/cic_eth/stat.py

@@ -20,11 +20,7 @@ def init_chain_stat(rpc, block_start=0):
     if block_start == 0:
         o = block_latest()
         r = rpc.do(o)
-        try:
-            block_start = int(r, 16)
-        except TypeError:
-            block_start = int(r)
-        logg.debug('blockstart {}'.format(block_start))
+        block_start = int(r, 16)
 
     for i in range(BLOCK_SAMPLES):
         o = block_by_number(block_start-10+i)

apps/cic-eth/cic_eth/task.py

@@ -20,8 +20,7 @@ import liveness.linux
 from cic_eth.error import SeppukuError
 from cic_eth.db.models.base import SessionBase
 
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger().getChild(__name__)
 
 celery_app = celery.current_app
 
@@ -119,13 +118,12 @@ def registry():
     return CICRegistry.address
 
 
-@celery_app.task(bind=True, base=BaseTask)
-def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
+@celery_app.task()
+def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    r = registry.by_address(address, sender_address=self.call_address)
-    return r
+    return registry.by_address(address)
 
 
 @celery_app.task(throws=(UnknownContractError,))
@@ -133,7 +131,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     conn = RPCConnection.connect(chain_spec, tag=connection_tag)
     registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name, sender_address=self.call_address)
+    return registry.by_name(name)
 
 
 @celery_app.task()

apps/cic-eth/cic_eth/version.py

@@ -8,9 +8,9 @@ import semver
 
 version = (
         0,
-        12,
+        11,
         0,
-        'alpha.1',
+        'beta.13',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,68 +1,48 @@
-FROM python:3.8.6-slim-buster as compile
+# FROM grassrootseconomics:cic
+
+#FROM python:3.8.6-alpine
+FROM python:3.8.6-slim-buster
+
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
 
 WORKDIR /usr/src/cic-eth
 
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-#RUN python -m venv venv && . venv/bin/activate
-
 ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN /usr/local/bin/python -m pip install --upgrade pip
-RUN pip install semver
-
-COPY cic-eth/ .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL .
-
-# --- TEST IMAGE ---
-FROM python:3.8.6-slim-buster as test 
+ARG root_requirement_file='requirements.txt'
 
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
-WORKDIR /usr/src/cic-eth
-
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file:  ${root_requirement_file}"
+#COPY $root_requirement_file .
+#RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
 RUN /usr/local/bin/python -m pip install --upgrade pip
+#RUN git clone https://gitlab.com/grassrootseconomics/cic-base.git && \
+#	cd cic-base && \
+#	git checkout 7ae1f02efc206b13a65873567b0f6d1c3b7f9bc0 && \
+#	python merge_requirements.py | tee merged_requirements.txt 
+#RUN cd cic-base && \
+#	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
+RUN  pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-# TODO we could use venv inside container to isolate the system and app deps further
-# COPY --from=compile /usr/src/cic-eth/ .
-# RUN . venv/bin/activate
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-
+COPY cic-eth/scripts/ scripts/
+COPY cic-eth/setup.cfg cic-eth/setup.py ./
+COPY cic-eth/cic_eth/ cic_eth/
+# Copy app specific requirements
+COPY cic-eth/requirements.txt .
 COPY cic-eth/test_requirements.txt .
-RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
-    	--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
-
-COPY cic-eth . 
-
-ENV PYTHONPATH .
-
-ENTRYPOINT ["pytest"]
-
-# --- RUNTIME ---
-FROM python:3.8.6-slim-buster as runtime 
-
-RUN apt-get update && \
-	apt install -y gnupg libpq-dev procps
-
-WORKDIR /usr/src/cic-eth
-
-COPY --from=compile /usr/local/bin/ /usr/local/bin/
-COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
+RUN	pip install $pip_extra_index_url_flag .
 
 COPY cic-eth/docker/* ./ 
 RUN chmod 755 *.sh
+COPY cic-eth/tests/ tests/
 
-COPY cic-eth/scripts/ scripts/
 # # ini files in config directory defines the configurable parameters for the application
 # # they can all be overridden by environment variables
 # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
@@ -71,4 +51,3 @@ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
 
 COPY util/liveness/health.sh /usr/local/bin/health.sh
-

apps/cic-eth/requirements.txt

@@ -1,3 +1,24 @@
+cic-base~=0.1.2b11
 celery==4.4.7
-chainlib~=0.0.5a1
+crypto-dev-signer~=0.4.14b3
+confini~=0.3.6rc3
+cic-eth-registry~=0.5.5a4
+redis==3.5.3
+alembic==1.4.2
+websockets==8.1
+requests~=2.24.0
+eth_accounts_index~=0.0.11a12
+erc20-transfer-authorization~=0.3.1a6
+uWSGI==2.0.19.1
 semver==2.13.0
+websocket-client==0.57.0
+moolb~=0.1.1b2
+eth-address-index~=0.1.1a11
+chainlib~=0.0.3a2
+hexathon~=0.0.1a7
+chainsyncer[sql]~=0.0.2a4
+chainqueue~=0.0.2a2
+sarafu-faucet==0.0.3a3
+erc20-faucet==0.2.1a4
+coincurve==15.0.0
+potaahto~=0.0.1a2

apps/cic-eth/scripts/migrate.py

@@ -2,8 +2,6 @@
 import os
 import argparse
 import logging
-import re
-import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -25,8 +23,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -57,10 +53,4 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/services_requirements.txt

@@ -1,7 +0,0 @@
-chainsyncer[sql]~=0.0.3a3
-chainqueue~=0.0.2b5
-alembic==1.4.2
-confini~=0.3.6rc4
-redis==3.5.3
-hexathon~=0.0.1a7
-pycryptodome==3.10.1

apps/cic-eth/setup.cfg

@@ -45,16 +45,16 @@ scripts =
 [options.entry_points]
 console_scripts =
 	# daemons
-	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main [services]
-	cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main [services]
-	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main [services]
-	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main [services]
+	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main
+	cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main
+	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main
+	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main
 	# tools	
-	cic-eth-create = cic_eth.runnable.create:main [tools]
-	cic-eth-inspect = cic_eth.runnable.view:main [tools]
-	cic-eth-ctl = cic_eth.runnable.ctrl:main [tools]
-	cic-eth-info = cic_eth.runnable.info:main [tools]
+	cic-eth-create = cic_eth.runnable.create:main 
+	cic-eth-inspect = cic_eth.runnable.view:main
+	cic-eth-ctl = cic_eth.runnable.ctrl:main
+	cic-eth-info = cic_eth.runnable.info:main
 	# TODO: Merge this with ctl when subcmds sorted to submodules
-	cic-eth-tag = cic_eth.runnable.tag:main [tools]
-	cic-eth-resend = cic_eth.runnable.resend:main [tools]
-	cic-eth-transfer = cic_eth.runnable.transfer:main [tools]
+	cic-eth-tag = cic_eth.runnable.tag:main
+	cic-eth-resend = cic_eth.runnable.resend:main
+	cic-eth-transfer = cic_eth.runnable.transfer:main

apps/cic-eth/setup.py

@@ -11,41 +11,17 @@ while True:
     requirements.append(l.rstrip())
 f.close()
 
-admin_requirements = []
-f = open('admin_requirements.txt', 'r')
+test_requirements = []
+f = open('test_requirements.txt', 'r')
 while True:
     l = f.readline()
     if l == '':
         break
-    admin_requirements.append(l.rstrip())
+    test_requirements.append(l.rstrip())
 f.close()
 
 
-
-tools_requirements = []
-f = open('tools_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    tools_requirements.append(l.rstrip())
-f.close()
-
-
-services_requirements = []
-f = open('services_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    services_requirements.append(l.rstrip())
-f.close()
-
 setup(
     install_requires=requirements,
-    extras_require = {
-        'tools': tools_requirements,
-        'admin_api': admin_requirements,
-        'services': services_requirements,
-        }
+    tests_require=test_requirements,
         )

apps/cic-eth/test_requirements.txt

@@ -2,8 +2,6 @@ pytest==6.0.1
 pytest-celery==0.0.0a1
 pytest-mock==3.3.1
 pytest-cov==2.10.1
-pytest-redis==2.0.0
-redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.10a2
+giftable-erc20-token==0.0.8a9

apps/cic-eth/tests/check/test_check_db.py

@@ -1,8 +0,0 @@
-# local imports
-from cic_eth.check.db import health
-
-def test_check_health(
-    init_database,
-        ):
-
-    assert health()

apps/cic-eth/tests/check/test_check_gas.py

@@ -1,20 +0,0 @@
-# local imports
-from cic_eth.check.gas import health
-from cic_eth.db.models.role import AccountRole
-
-def test_check_gas(
-    config,
-    init_database,
-    default_chain_spec,
-    eth_rpc,
-    custodial_roles,
-    whoever,
-        ):
-
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
-    assert health(config=config)
-
-    AccountRole.set('GAS_GIFTER', whoever, session=init_database)
-    init_database.commit()
-    assert not health(config=config)

apps/cic-eth/tests/check/test_check_redis.py

@@ -1,16 +0,0 @@
-# external imports
-import pytest
-
-# local imports
-from cic_eth.check.redis import health
-
-
-def test_check_redis(
-        config,
-        have_redis,
-        ):
-
-    if have_redis != None:
-        pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
-
-    assert health(unit='test', config=config)

apps/cic-eth/tests/check/test_check_signer.py

@@ -1,13 +0,0 @@
-# local imports
-from cic_eth.check.signer import health
-
-
-def test_check_signer(
-        default_chain_spec,
-        config,
-        eth_signer,
-        eth_rpc,
-        ):
-
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    assert health(config=config)

apps/cic-eth/tests/conftest.py

@@ -2,11 +2,9 @@
 import os
 import sys
 import logging
-import uuid
 
 # external imports
-from eth_erc20 import ERC20
-import redis
+from chainlib.eth.erc20 import ERC20
 
 # local imports
 from cic_eth.api import Api
@@ -17,11 +15,10 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from cic_eth.pytest.fixtures_config import *
-from cic_eth.pytest.fixtures_celery import *
-from cic_eth.pytest.fixtures_database import *
-from cic_eth.pytest.fixtures_role import *
-from cic_eth.pytest.fixtures_contract import *
+from tests.fixtures_config import *
+from tests.fixtures_database import *
+from tests.fixtures_celery import *
+from tests.fixtures_role import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -58,28 +55,3 @@ def default_token(
         ):
     BaseTask.default_token_symbol = foo_token_symbol
     BaseTask.default_token_address = foo_token   
-
-
-
-@pytest.fixture(scope='session')
-def have_redis(
-        config,
-        ):
-
-    r = redis.Redis(
-            host = config.get('REDIS_HOST'),
-            port = config.get('REDIS_PORT'),
-            db = config.get('REDIS_DB'),
-            ) 
-    k = str(uuid.uuid4())
-    try:
-        r.set(k, 'foo')
-        r.delete(k)
-    except redis.exceptions.ConnectionError as e:
-        return e
-    except TypeError as e:
-        return e 
-
-    return None
-
-

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -14,9 +14,9 @@ from chainlib.eth.tx import (
         Tx,
         )
 from chainlib.eth.block import Block
-from eth_erc20 import ERC20
+from chainlib.eth.erc20 import ERC20
 from sarafu_faucet import MinterFaucet
-from eth_accounts_index.registry import AccountRegistry
+from eth_accounts_index import AccountRegistry
 from potaahto.symbols import snake_and_camel
 from hexathon import add_0x
 
@@ -26,6 +26,7 @@ from cic_eth.runnable.daemons.filters.callback import CallbackFilter
 logg = logging.getLogger()
 
 
+@pytest.mark.skip()
 def test_transfer_tx(
         default_chain_spec,
         init_database,
@@ -65,6 +66,7 @@ def test_transfer_tx(
     assert transfer_type == 'transfer'
 
 
+@pytest.mark.skip()
 def test_transfer_from_tx(
         default_chain_spec,
         init_database,

apps/cic-eth/tests/filters/test_filter_bogus.py

@@ -1,38 +0,0 @@
-# local imports
-from cic_eth.runnable.daemons.filters.gas import GasFilter
-from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
-from cic_eth.runnable.daemons.filters.callback import CallbackFilter
-from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
-from cic_eth.runnable.daemons.filters.tx import TxFilter
-from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-
-
-# Hit tx mismatch paths on all filters
-def test_filter_bogus(
-        init_database,
-        bogus_tx_block,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        transfer_auth,
-        cic_registry,
-        contract_roles,
-        register_lookups,
-        ):
-
-    fltrs = [
-        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
-        GasFilter(default_chain_spec, queue=None),
-        TxFilter(default_chain_spec, None),
-        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
-        StragglerFilter(default_chain_spec, None),
-        RegistrationFilter(default_chain_spec, queue=None),
-        ]
-      
-    for fltr in fltrs:
-        r = None
-        try:
-            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
-        except:
-            pass
-        assert not r

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -1,101 +0,0 @@
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from chainqueue.sql.state import (
-        set_waitforgas,
-        )
-from hexathon import strip_0x
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-
-# local imports
-from cic_eth.runnable.daemons.filters.gas import GasFilter
-from cic_eth.eth.gas import cache_gas_data
-
-
-def test_filter_gas(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        celery_session_worker,
-    ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
-    init_database.commit()
-    tx_hash_hex_wait = tx_hash_hex
-    otx = Otx.load(tx_hash_hex_wait, session=init_database)
-    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['BOB'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    fltr = GasFilter(default_chain_spec, queue=None)
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-    init_database.commit()
-
-    otx = Otx.load(tx_hash_hex_wait, session=init_database)
-    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

apps/cic-eth/tests/filters/test_register_filter.py

@@ -1,78 +0,0 @@
-# external imports
-from eth_accounts_index.registry import AccountRegistry
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.tx import(
-        receipt,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from erc20_faucet import Faucet
-from hexathon import strip_0x
-from chainqueue.sql.query import get_account_tx
-
-# local imports
-from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-
-
-def test_register_filter(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        account_registry,
-        faucet,
-        register_lookups,
-        contract_roles,
-        agent_roles,
-        cic_registry,
-        init_celery_tasks,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
-
-    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
-    r = eth_rpc.do(o)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
-
-    o = receipt(tx_hash_hex)
-    rcpt = eth_rpc.do(o)
-    assert rcpt['status'] == 1
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block, rcpt=rcpt)
-    tx.apply_receipt(rcpt)
-
-    fltr = RegistrationFilter(default_chain_spec, queue=None)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-
-    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
-    ks = list(gift_txs.keys())
-    assert len(ks) == 1
-
-    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
-    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
-    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
-
-    gift = Faucet.parse_give_to_request(gift_tx['data'])
-    assert gift[0] == agent_roles['ALICE']

apps/cic-eth/tests/filters/test_straggler_filter.py

@@ -17,8 +17,8 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
+from chainqueue.tx import create as queue_create
+from chainqueue.state import (
         set_reserved,
         set_ready,
         set_sent,

apps/cic-eth/tests/filters/test_transferauth_filter.py

@@ -1,79 +0,0 @@
-# external imports
-from erc20_transfer_authorization import TransferAuthorization
-from eth_erc20 import ERC20
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import OverrideGasOracle
-from chainlib.eth.tx import (
-        receipt,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.block import (
-        block_latest,
-        block_by_number,
-        Block,
-        )
-from hexathon import strip_0x
-from chainqueue.sql.query import get_account_tx
-
-# local imports
-from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
-
-
-def test_filter_transferauth(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        transfer_auth,
-        foo_token,
-        celery_session_worker,
-        register_lookups,
-        init_custodial,
-        cic_registry,
-    ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
-    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
-    
-    r = rpc.do(o)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
-
-    o = receipt(tx_hash_hex)
-    r = rpc.do(o)
-    assert r['status'] == 1
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-
-    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-    t.get_leaf()
-    assert t.successful()
-
-    approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
-    ks = list(approve_txs.keys())
-    assert len(ks) == 1
-
-    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
-    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
-    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
-
-    c = ERC20(default_chain_spec)
-    approve = c.parse_approve_request(approve_tx['data']) 
-    assert approve[0] == agent_roles['BOB']

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -17,12 +17,13 @@ from chainlib.eth.block import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
+from chainqueue.tx import create as queue_create
+from chainqueue.state import (
         set_reserved,
         set_ready,
         set_sent,
         )
+
 from hexathon import strip_0x
 
 # local imports
@@ -30,7 +31,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
 from cic_eth.eth.gas import cache_gas_data
 
 
-def test_filter_tx(
+def test_tx(
         default_chain_spec,
         init_database,
         eth_rpc,

apps/cic-eth/tests/fixtures_celery.py

@@ -22,6 +22,7 @@ def init_celery_tasks(
 @pytest.fixture(scope='session')
 def celery_includes():
     return [
+#        'cic_eth.eth.bancor',
         'cic_eth.eth.erc20',
         'cic_eth.eth.tx',
         'cic_eth.ext.tx',
@@ -46,8 +47,8 @@ def celery_config():
     bq = tempfile.mkdtemp()
     bp = tempfile.mkdtemp()
     rq = tempfile.mkdtemp()
-    logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
-    logg.debug('celery backend session store {}'.format(rq))
+    logg.debug('celery broker queue {} processed {}'.format(bq, bp))
+    logg.debug('celery backend store {}'.format(rq))
     yield {
         'broker_url': 'filesystem://',
         'broker_transport_options': {
@@ -57,11 +58,12 @@ def celery_config():
             },
         'result_backend': 'file://{}'.format(rq),
             }
-    logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
+    logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq))
     shutil.rmtree(bq)
     shutil.rmtree(bp)
     shutil.rmtree(rq)
 
+
 @pytest.fixture(scope='session')
 def celery_worker_parameters():
     return {

apps/cic-eth/tests/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# external imports
+# third-party imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(os.path.dirname(script_dir))
-logg = logging.getLogger(__name__)
+root_dir = os.path.dirname(script_dir)
+logg = logging.getLogger(__file__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/tests/fixtures_database.py

@@ -37,8 +37,7 @@ def init_database(
         database_engine,
         ):
 
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-    rootdir = os.path.dirname(os.path.dirname(script_dir))
+    rootdir = os.path.dirname(os.path.dirname(__file__))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/tests/task/api/test_admin.py

@@ -9,14 +9,8 @@ from chainlib.eth.tx import (
         unpack,
         TxFormat,
         )
-from chainlib.eth.nonce import (
-        RPCNonceOracle,
-        OverrideNonceOracle,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import Gas
 from chainlib.eth.address import to_checksum_address
 from hexathon import (
         strip_0x,
@@ -29,18 +23,10 @@ from chainqueue.db.enum import (
         StatusBits,
         status_str,
         )
-from chainqueue.sql.state import (
-        set_fubar,
-        set_ready,
-        set_reserved,
-        )
-from chainqueue.sql.query import (
-        get_tx,
-        get_nonce_tx_cache,
-        )
+from chainqueue.query import get_tx
 
 # local imports
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import AdminApi
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import InitializationError
@@ -50,6 +36,150 @@ from cic_eth.queue.tx import queue_create
 logg = logging.getLogger()
 
 
+#def test_resend_inplace(
+#    default_chain_spec,
+#    init_database,
+#    init_w3,
+#    celery_session_worker,
+#    ):
+#    
+#    chain_str = str(default_chain_spec)
+#    c = RpcClient(default_chain_spec)
+#    
+#    sigs = []
+#
+#    gas_provider = c.gas_provider()
+#
+#    s_nonce = celery.signature(
+#        'cic_eth.eth.nonce.reserve_nonce',
+#        [
+#            init_w3.eth.accounts[0],
+#            gas_provider,
+#            ],
+#        queue=None,
+#        )
+#    s_refill = celery.signature(
+#        'cic_eth.eth.gas.refill_gas',
+#        [
+#            chain_str,
+#            ],
+#        queue=None,
+#            )
+#    s_nonce.link(s_refill)
+#    t = s_nonce.apply_async()
+#    t.get()
+#    for r in t.collect():
+#        pass
+#    assert t.successful()
+#
+#    q = init_database.query(Otx)
+#    q = q.join(TxCache)
+#    q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
+#    o = q.first()
+#    tx_raw = o.signed_tx
+#
+#    tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
+#    gas_price_before = tx_dict['gasPrice'] 
+#
+#    s = celery.signature(
+#        'cic_eth.admin.ctrl.lock_send',
+#        [
+#            chain_str,
+#            init_w3.eth.accounts[0],
+#            ],
+#        queue=None,
+#            )
+#    t = s.apply_async()
+#    t.get()
+#    assert t.successful()
+#
+#    api = AdminApi(c, queue=None)
+#    t = api.resend(tx_dict['hash'], chain_str, unlock=True)
+#    t.get()
+#    i = 0
+#    tx_hash_new_hex = None
+#    for r in t.collect():
+#        tx_hash_new_hex = r[1]
+#    assert t.successful()
+#  
+#    tx_raw_new = get_tx(tx_hash_new_hex) 
+#    logg.debug('get {}'.format(tx_raw_new))
+#    tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
+#    assert tx_hash_new_hex != tx_dict['hash']
+#    assert tx_dict_new['gasPrice'] > gas_price_before
+#
+#    tx_dict_after = get_tx(tx_dict['hash'])
+#
+#    logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
+#    assert tx_dict_after['status'] & StatusBits.MANUAL
+
+
+#def test_check_fix_nonce(
+#    default_chain_spec,
+#    init_database,
+#    init_eth_account_roles,
+#    init_w3,
+#    eth_empty_accounts,
+#    celery_session_worker,
+#    ):
+#
+#    chain_str = str(default_chain_spec)
+#    
+#    sigs = []
+#    for i in range(5):
+#        s = celery.signature(
+#            'cic_eth.eth.gas.refill_gas',
+#            [
+#                eth_empty_accounts[i],
+#                chain_str,
+#                ],
+#            queue=None,
+#                )
+#        sigs.append(s)
+#
+#    t = celery.group(sigs)()
+#    txs = t.get()
+#    assert t.successful()
+#
+#    tx_hash = web3.Web3.keccak(hexstr=txs[2])
+#    c = RpcClient(default_chain_spec)
+#    api = AdminApi(c, queue=None)
+#    address = init_eth_account_roles['eth_account_gas_provider']
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['network'] == 0
+#    assert nonce_spec['nonce']['queue'] == 4
+#    assert nonce_spec['nonce']['blocking'] == None
+#
+#    s_set = celery.signature(
+#            'cic_eth.queue.tx.set_rejected',
+#            [
+#                tx_hash.hex(),
+#                ],
+#            queue=None,
+#            )
+#    t = s_set.apply_async()
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#
+#    nonce_spec = api.check_nonce(address)
+#    assert nonce_spec['nonce']['blocking'] == 2
+#    assert nonce_spec['tx']['blocking'] == tx_hash.hex()
+#
+#    t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
+#    t.get()
+#    t.collect()
+#    assert t.successful()
+#
+#    for tx in txs[3:]:
+#        tx_hash = web3.Web3.keccak(hexstr=tx)
+#        tx_dict = get_tx(tx_hash.hex())
+#        assert tx_dict['status'] == StatusEnum.OVERRIDDEN
+#
+#
+
+
 def test_have_account(
     default_chain_spec,
     custodial_roles,
@@ -113,6 +243,28 @@ def test_tag_account(
     assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
 
 
+#def test_ready(
+#    init_database,
+#    agent_roles,
+#    eth_rpc,
+#    ):
+#
+#    api = AdminApi(eth_rpc)
+#   
+#    with pytest.raises(InitializationError):
+#        api.ready()
+#
+#    bogus_account = os.urandom(20)
+#    bogus_account_hex = '0x' + bogus_account.hex()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
+#    with pytest.raises(KeyError):
+#        api.ready()
+#
+#    api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
+#    api.ready()
+
+
 def test_tx(
     default_chain_spec,
     cic_registry,
@@ -134,168 +286,3 @@ def test_tx(
     api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
     tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
     logg.warning('code missing to verify tx contents {}'.format(tx))
-
-
-def test_check_nonce_gap(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    # NOTE: this only works as long as agents roles start at nonce 0
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    j = 0
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        if i == 3:
-            j = 1
-            nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
-
-        queue_create(
-                default_chain_spec,
-                i+j,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
-
-    assert r['nonce']['blocking'] == 4
-    assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
-
-
-def test_check_nonce_localfail(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    # NOTE: this only works as long as agents roles start at nonce 0
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    j = 0
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-        queue_create(
-                default_chain_spec,
-                i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    set_ready(default_chain_spec, tx_hashes[4], session=init_database)
-    set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
-    set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
-
-    assert r['nonce']['blocking'] == 4
-    assert r['tx']['blocking'] == tx_hashes[4]
-
-
-def test_fix_nonce(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        init_celery_tasks,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-        queue_create(
-                default_chain_spec,
-                i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
-    t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
-    r = t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-    
-    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
-    ks = txs.keys()
-    assert len(ks) == 2
-    for k in ks:
-        hsh = add_0x(k)
-        otx = Otx.load(hsh, session=init_database)
-        init_database.refresh(otx)
-        logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
-        if add_0x(k) == tx_hashes[3]:
-            assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-        else:
-            assert otx.status == 1

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -1,373 +0,0 @@
-# standard imports
-import logging
-import io
-import json
-
-# external imports
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import (
-        nonce,
-        OverrideNonceOracle,
-        RPCNonceOracle,
-        )
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.tx import (
-        raw,
-        receipt,
-        TxFormat,
-        Tx,
-        )
-from chainlib.eth.block import block_latest
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        set_sent,
-        set_ready,
-        )
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-from chainqueue.sql.query import get_nonce_tx_cache
-from eth_erc20 import ERC20
-from cic_eth_registry import CICRegistry
-
-# local imports
-from cic_eth.api.admin import AdminApi
-from cic_eth.eth.gas import cache_gas_data
-from cic_eth.eth.erc20 import cache_transfer_data
-
-logg = logging.getLogger()
-
-
-def test_admin_api_tx(
-        default_chain_spec,
-        init_database,
-        init_celery_tasks,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        custodial_roles,
-        celery_session_worker,
-        foo_token,
-        address_declarator,
-        cic_registry,
-        register_tokens,
-        register_lookups,
-        caplog,
-        ):
-
-    nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
-
-    o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
-    r = eth_rpc.do(o)
-    gifter_nonce = int(r, 16)
-
-    #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-    queue_create(
-            default_chain_spec,
-            gifter_nonce, # will only work if agent starts at 0
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_transfer_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    o = raw(tx_signed_raw_hex)
-    eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    # lookup by transaction hash, without registry
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
-    logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by RLP transaction, without registry
-    tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by transaction hash, with registry
-    registry = CICRegistry(default_chain_spec, eth_rpc)
-    tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
-    assert tx['tx_hash'] == tx_hash_hex 
-
-    # lookup by transaction hash, using writer
-    buf = io.StringIO()
-    api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
-    tx = json.loads(buf.getvalue())
-    assert tx['tx_hash'] == tx_hash_hex 
-
-
-def test_admin_api_account(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes_alice = []
-    txs_alice = []
-
-    for i in range(3):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                42+i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes_alice.append(tx_hash_hex)
-        txs_alice.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
-    tx_hashes_bob = []
-    txs_bob = []
-
-    for i in range(2):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                13+i,
-                agent_roles['BOB'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes_bob.append(tx_hash_hex)
-        txs_bob.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'])
-    assert len(r) == 5
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
-    assert len(r) == 2
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
-    assert len(r) == 3
-
-
-def test_admin_api_account_writer(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    buf = io.StringIO()
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
-
-    # TODO: improve eval
-    tx = json.loads(buf.getvalue())
-    assert tx['tx_hash'] == tx_hash_hex
-
-
-def test_registry(
-        eth_rpc,
-        cic_registry,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.registry()
-    r = t.get_leaf()
-    assert r == cic_registry
-
-
-def test_proxy_do(
-        default_chain_spec,
-        eth_rpc,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.proxy_do(default_chain_spec, o)
-    rr = t.get_leaf()
-
-    assert r == rr
-
-
-def test_resend_inplace(
-        init_database,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
-    r = t.get_leaf()
-    assert t.successful()
-
-
-    otx = Otx.load(tx_hash_hex, session=init_database)
-    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 2
-
-
-
-@pytest.mark.xfail()
-def test_resend_clone(
-        init_database,
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        contract_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    init_database.commit()
-
-    api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
-    t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
-    r = t.get_leaf()
-    assert t.successful()
-
-    otx = Otx.load(tx_hash_hex, session=init_database)
-    assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
-    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 1
-
-    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
-    assert len(txs) == 1
-
-    otx = Otx.load(txs[0], session=init_database)
-    assert otx.status == 0

apps/cic-eth/tests/task/api/test_app.py

@@ -8,20 +8,11 @@ import pytest
 import celery
 from cic_eth_registry.erc20 import ERC20Token
 from chainlib.chain import ChainSpec
-from eth_accounts_index import AccountsIndex
-from chainlib.eth.tx import (
-        transaction,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        )
 
 # local imports
 from cic_eth.api import Api
-from cic_eth.queue.query import get_tx
 
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 def test_account_api(
@@ -38,47 +29,6 @@ def test_account_api(
     assert t.successful()
 
 
-def test_account_api_register(
-        default_chain_spec,
-        init_database,
-        account_registry,
-        faucet,
-        custodial_roles,
-        cic_registry,
-        register_lookups,
-        eth_rpc,
-        celery_session_worker,
-        ):
-    api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.create_account('')
-    register_tx_hash = t.get_leaf()
-    assert t.successful()
-
-    set_reserved(default_chain_spec, register_tx_hash, session=init_database)
-
-    tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
-    s = celery.signature(
-            'cic_eth.eth.tx.send',
-            [
-                [tx['signed_tx']],
-                default_chain_spec.asdict(),
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    o = transaction(register_tx_hash)
-    tx_src = eth_rpc.do(o)
-
-    c = AccountsIndex(default_chain_spec)
-    address = c.parse_add_request(tx_src['data'])
-    o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    assert c.parse_have(r)
-
-
 def test_transfer_api(
         default_chain_spec,
         eth_rpc,
@@ -87,15 +37,16 @@ def test_transfer_api(
         custodial_roles,
         agent_roles,
         cic_registry,
-        token_registry,
+        register_tokens,
         register_lookups,
         celery_session_worker,
-        register_tokens,
-        foo_token_symbol,
         ):
 
+    #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
+    foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
+
     api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
-    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) 
+    t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol)
     t.get_leaf()
     assert t.successful()
 

apps/cic-eth/tests/task/api/test_app_noncritical.py

@@ -1,19 +0,0 @@
-# local imports
-from cic_eth.api.api_task import Api
-from cic_eth.task import BaseTask
-
-def test_default_token(
-        default_chain_spec,
-        foo_token,
-        default_token,
-        token_registry,
-        register_tokens,
-        register_lookups,
-        cic_registry,
-        celery_session_worker,
-        ):
-
-    api = Api(str(default_chain_spec), queue=None)     
-    t = api.default_token()
-    r = t.get_leaf()
-    assert r['address'] == foo_token

apps/cic-eth/tests/task/api/test_list.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 from chainlib.eth.nonce import RPCNonceOracle
-from eth_erc20 import ERC20
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import receipt
 
 # local imports

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from cic_eth.pytest.fixtures_celery import *
+from tests.fixtures_celery import *

apps/cic-eth/tests/task/test_task_account.py

@@ -9,7 +9,7 @@ import celery
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import receipt
-from eth_accounts_index.registry import AccountRegistry
+from eth_accounts_index import AccountRegistry
 from hexathon import strip_0x
 from chainqueue.db.enum import StatusEnum
 from chainqueue.db.models.otx import Otx
@@ -156,7 +156,6 @@ def test_gift(
     eth_signer,
     init_celery_tasks,
     cic_registry,
-    register_lookups,
     celery_session_worker,
     ):
 

apps/cic-eth/tests/task/test_task_admin.py

@@ -1,88 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import OverrideNonceOracle
-from chainqueue.sql.tx import (
-        create as queue_create,
-        )
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainlib.eth.tx import TxFormat
-from chainqueue.sql.query import get_nonce_tx_cache
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.enum import StatusBits
-from hexathon import add_0x
-
-# local imports
-from cic_eth.admin.nonce import shift_nonce
-from cic_eth.eth.gas import cache_gas_data
-
-logg = logging.getLogger()
-
-
-def test_shift_nonce(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        celery_session_worker,
-        caplog,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
-
-    tx_hashes = []
-    txs = []
-
-    for i in range(10):
-        c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-        queue_create(
-                default_chain_spec,
-                42+i,
-                agent_roles['ALICE'],
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                session=init_database,
-                )
-        cache_gas_data(
-                tx_hash_hex,
-                tx_signed_raw_hex,
-                default_chain_spec.asdict(),
-                )
-        tx_hashes.append(tx_hash_hex)
-        txs.append(tx_signed_raw_hex)
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.admin.nonce.shift_nonce',
-            [
-                default_chain_spec.asdict(),
-                tx_hashes[3],
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    init_database.commit()
-
-
-    for i in range(42+3, 42+10):
-        txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
-        for k in txs.keys():
-            hsh = add_0x(k)
-            otx = Otx.load(hsh, session=init_database)
-            logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
-            if add_0x(k) == tx_hashes[i-42]:
-                assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
-            else:
-                assert otx.status == 1

apps/cic-eth/tests/task/test_task_erc20.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import pytest
 import celery
-from eth_erc20 import ERC20
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
@@ -13,7 +13,6 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
-from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -168,101 +167,3 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
-
-
-def test_erc20_transfer_from_task(
-        default_chain_spec,
-        foo_token,
-        agent_roles,
-        custodial_roles,
-        eth_signer,
-        eth_rpc,
-        init_database,
-        celery_session_worker,
-        token_roles,
-        ):
-
-    token_object = {
-        'address': foo_token,
-            }
-    transfer_value = 100 * (10 ** 6)
-
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
-    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
-    r = eth_rpc.do(o)
-    o = receipt(tx_hash)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    s_nonce = celery.signature(
-        'cic_eth.eth.nonce.reserve_nonce',
-        [
-            [token_object],
-            default_chain_spec.asdict(),
-            custodial_roles['FOO_TOKEN_GIFTER'],
-            ],
-        queue=None,
-            )
-    s_transfer = celery.signature(
-            'cic_eth.eth.erc20.transfer_from',
-            [
-                custodial_roles['FOO_TOKEN_GIFTER'],
-                agent_roles['BOB'],
-                transfer_value,
-                default_chain_spec.asdict(),
-                agent_roles['ALICE'],
-                ],
-            queue=None,
-            )
-    s_nonce.link(s_transfer)
-    t = s_nonce.apply_async()
-    r = t.get_leaf()
-
-    logg.debug('result {}'.format(r))
-
-
-def test_erc20_allowance_check_task(
-        default_chain_spec,
-        foo_token,
-        agent_roles,
-        custodial_roles,
-        eth_signer,
-        eth_rpc,
-        init_database,
-        celery_session_worker,
-        token_roles,
-        ):
-
-    token_object = {
-        'address': foo_token,
-        'symbol': 'FOO',
-            }
-    transfer_value = 100 * (10 ** 6)
-
-    s_check = celery.signature(
-        'cic_eth.eth.erc20.check_allowance',
-        [
-            [token_object],
-            custodial_roles['FOO_TOKEN_GIFTER'],
-            transfer_value,
-            default_chain_spec.asdict(),
-            agent_roles['ALICE']
-            ],
-        queue=None,
-            )
-    t = s_check.apply_async()
-    with pytest.raises(YouAreBrokeError):
-        t.get() 
-
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
-    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
-    r = eth_rpc.do(o)
-    o = receipt(tx_hash)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    t = s_check.apply_async()
-    t.get() 
-    assert t.successful()

apps/cic-eth/tests/task/test_task_gas.py

@@ -1,286 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-import pytest
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import (
-        OverrideNonceOracle,
-        RPCNonceOracle,
-        )
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
-from chainlib.eth.tx import (
-        unpack,
-        TxFormat,
-        )
-from chainlib.eth.constant import (
-        MINIMUM_FEE_UNITS,
-        MINIMUM_FEE_PRICE,
-        )
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.query import get_tx
-from chainqueue.db.enum import StatusBits
-from chainqueue.sql.state import (
-        set_ready,
-        set_reserved,
-        set_sent,
-        )
-from chainqueue.db.models.otx import Otx
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.eth.gas import cache_gas_data
-from cic_eth.error import OutOfGasError
-
-logg = logging.getLogger()
-
-
-def test_task_check_gas_ok(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                [],
-                None,
-                8000000,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
-
-
-def test_task_check_gas_insufficient(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        whoever,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = OverrideNonceOracle(whoever, 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            whoever,
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                [],
-                None,
-                None,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    try:
-        r = t.get_leaf()
-    except OutOfGasError:
-        pass
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
-
-
-def test_task_check_gas_low(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        init_database,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        whoever,
-        ):
-
-    gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
-    nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
-    r = eth_rpc.do(o)
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) 
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            whoever,
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-
-    init_database.commit()
-
-    s = celery.signature(
-            'cic_eth.eth.gas.check_gas',
-            [
-                [
-                    tx_hash_hex,
-                    ],
-                default_chain_spec.asdict(),
-                ],
-                [],
-                None,
-                None,
-            queue=None
-            )
-    t = s.apply_async()
-    t.get_leaf()
-    assert t.successful()
-
-    init_database.commit()
-
-    tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
-    assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
-
-
-@pytest.mark.parametrize(
-        '_gas_price,_gas_factor',
-        [
-        (None, 1.1),
-        (MINIMUM_FEE_PRICE * 1.1, 0.9),
-        (None, 1.3),
-        ]
-        )
-def test_task_resend_explicit(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        agent_roles,
-        custodial_roles,
-        celery_session_worker,
-        _gas_price,
-        _gas_factor,
-        ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) 
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            0,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
-
-    init_database.commit()
-
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
-            [
-                tx_hash_hex,
-                default_chain_spec.asdict(),
-                _gas_price,
-                _gas_factor,
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    q = init_database.query(Otx)
-    q = q.filter(Otx.tx_hash==strip_0x(r))
-    otx = q.first()
-    if otx == None:
-        raise NotLocalTxError(r)
-
-    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
-    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
-    assert tx_after['gasPrice'] > tx_before['gasPrice']
-

apps/cic-eth/tests/task/test_task_tx.py

@@ -4,27 +4,16 @@ import logging
 # external imports
 import pytest
 import celery
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
+from chainlib.eth.gas import Gas
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
         transaction,
         receipt,
-        raw,
         )
 from hexathon import strip_0x
 from chainqueue.db.models.otx import Otx
-from chainqueue.sql.tx import create as queue_create
-from chainqueue.sql.state import (
-        set_reserved,
-        set_ready,
-        set_sent,
-        )
-from chainqueue.db.enum import StatusBits
 
 # local imports
 from cic_eth.queue.tx import register_tx
@@ -71,6 +60,15 @@ def test_tx_send(
 
 
 def test_sync_tx(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        celery_session_worker,
+        ):
+    pass
+
+
+def test_resend_with_higher_gas(
         init_database,
         default_chain_spec,
         eth_rpc,
@@ -79,48 +77,31 @@ def test_sync_tx(
         celery_session_worker,
         ):
 
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            session=init_database,
-            )
-    cache_gas_data(
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            default_chain_spec.asdict(),
-            )
-    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
-    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
-
-    o = raw(tx_signed_raw_hex)
-    r = eth_rpc.do(o)
-
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED)
+    register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database)
+    cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
+    tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
 
     s = celery.signature(
-            'cic_eth.eth.tx.sync_tx',
+            'cic_eth.eth.gas.resend_with_higher_gas',
             [
                 tx_hash_hex,
                 default_chain_spec.asdict(),
                 ],
-            queue=None
+            queue=None,
             )
     t = s.apply_async()
     r = t.get_leaf()
-    assert t.successful() 
 
-    init_database.commit()
+    q = init_database.query(Otx)
+    q = q.filter(Otx.tx_hash==strip_0x(r))
+    otx = q.first()
+    if otx == None:
+        raise NotLocalTxError(r)
+
+    tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
+    logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
+    assert tx_after['gasPrice'] > tx_before['gasPrice']
 
-    o = Otx.load(tx_hash_hex, session=init_database)
-    assert o.status & StatusBits.FINAL == StatusBits.FINAL

apps/cic-eth/tests/task/test_task_tx_misc.py

@@ -1,170 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-import pytest
-import celery
-from chainqueue.sql.tx import create as queue_create
-from chainlib.eth.nonce import (
-        RPCNonceOracle,
-        OverrideNonceOracle,
-        )
-from chainlib.eth.gas import (
-        OverrideGasOracle,
-        Gas,
-        )
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        receipt,
-        )
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
-from chainqueue.sql.state import (
-        set_reserved,
-        set_ready,
-        )
-
-logg = logging.getLogger()
-
-
-def test_hashes_to_txs(
-        init_database,
-        default_chain_spec,
-        agent_roles,            
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-
-    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex_one,
-            tx_signed_raw_hex_one,
-            session=init_database,
-            )
-
-    #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['ALICE'],
-            tx_hash_hex_two,
-            tx_signed_raw_hex_two,
-            session=init_database,
-            )
-
-    init_database.commit()
-
-    bogus_one = add_0x(os.urandom(32).hex())
-    bogus_two = add_0x(os.urandom(32).hex())
-
-    yarrgs = [
-                bogus_one,
-                tx_hash_hex_two,
-                bogus_two,
-                tx_hash_hex_one,
-            ]   
-    s = celery.signature(
-        'cic_eth.eth.tx.hashes_to_txs',
-        [
-            yarrgs,
-                ],
-        queue=None,
-        )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    assert len(r) == 2
-
-    logg.debug('r {}'.format(r))
-    txs = [
-        tx_signed_raw_hex_two,
-        tx_signed_raw_hex_one,
-            ]
-    for tx in r:
-        txs.remove(add_0x(tx))
-    assert len(txs) == 0
-
-
-
-def test_double_send(
-        init_database,
-        default_chain_spec,
-        agent_roles,            
-        eth_rpc,
-        eth_signer,
-        celery_session_worker,
-        ):
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            42,
-            agent_roles['ALICE'],
-            tx_hash_hex_one,
-            tx_signed_raw_hex_one,
-            session=init_database,
-            )
-    set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
-
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
-
-    queue_create(
-            default_chain_spec,
-            43,
-            agent_roles['ALICE'],
-            tx_hash_hex_two,
-            tx_signed_raw_hex_two,
-            session=init_database,
-            )
-
-    set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
-    set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
-    init_database.commit()
-
-    yarrgs = [
-        tx_signed_raw_hex_one,
-        tx_signed_raw_hex_two,
-            ]
-    s = celery.signature(
-            'cic_eth.eth.tx.send',
-            [
-                yarrgs,
-                default_chain_spec.asdict(),
-                ],
-            queue=None
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-
-    o = receipt(tx_hash_hex_one)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = receipt(tx_hash_hex_two)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-
-

apps/cic-eth/tests/unit/ext/test_address.py

@@ -3,7 +3,7 @@ from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import (
         receipt,
         )
-from eth_address_declarator import Declarator
+from eth_address_declarator import AddressDeclarator
 from hexathon import add_0x
 
 # local imports
@@ -19,12 +19,11 @@ def test_translate(
         agent_roles,
         cic_registry,
         init_celery_tasks,
-        register_lookups,
     ):
 
     nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
 
-    c = Declarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
 
     description = 'alice'.encode('utf-8').ljust(32, b'\x00').hex()
     (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], add_0x(description))

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -8,7 +8,7 @@ from chainlib.eth.tx import (
         count,
         receipt,
         )
-from eth_erc20 import ERC20
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.nonce import RPCNonceOracle
 
 # local imports

apps/cic-eth/tests/unit/test_registry_connect.py

@@ -1,22 +0,0 @@
-# local imports
-from cic_eth.registry import *
-
-def test_registry_connect(
-        eth_rpc,
-        default_chain_spec,
-        address_declarator,
-        token_registry,
-        contract_roles,
-        purge_lookups,
-        registry,
-        agent_roles,
-        ):
-
-    r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])
-

apps/cic-eth/tests/unit/test_stat.py

@@ -1,18 +0,0 @@
-# standard imports
-import datetime
-
-# local imports
-from cic_eth.stat import init_chain_stat
-
-
-def test_chain_stat(
-        eth_rpc,
-        init_eth_tester,
-        ):
-
-    now = int(datetime.datetime.now().timestamp()) + 1
-    for i in range(11):
-        init_eth_tester.time_travel(now + (i * 2))
-    
-    s = init_chain_stat(eth_rpc, block_start=0)
-    assert s.block_average() == 2

apps/cic-eth/tools_requirements.txt

@@ -1,8 +0,0 @@
-crypto-dev-signer~=0.4.14b6
-chainqueue~=0.0.2b5
-confini~=0.3.6rc4
-cic-eth-registry~=0.5.6a1
-redis==3.5.3
-hexathon~=0.0.1a7
-pycryptodome==3.10.1
-pyxdg==0.27

apps/cic-meta/.gitignore

@@ -3,5 +3,3 @@ dist
 dist-web
 dist-server
 scratch
-coverage
-.nyc_output

apps/cic-meta/.gitlab-ci.yml

@@ -3,38 +3,17 @@
     variables:
         APP_NAME: cic-meta
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
 .cic_meta_changes_target:
     rules:
-        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#        - changes:
-#            - $CONTEXT/$APP_NAME/*
-        - when: always
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
-cic-meta-build-mr:
-  stage: build
-  extends:
-    - .cic_meta_variables
-    - .cic_meta_changes_target
-  script:
-    - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
-    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
-    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
-
-test-mr-cic-meta:
+build-mr-cic-meta:
     extends:
-        - .cic_meta_variables
         - .cic_meta_changes_target
-    stage: test
-    image: $IMAGE_TAG 
-    script:
-        - cd /tmp/src/cic-meta
-        - npm install --dev 
-        - npm run test
-        - npm run test:coverage
-    needs: ["cic-meta-build-mr"]
+        - .py_build_merge_request
+        - .cic_meta_variables
 
 build-push-cic-meta:
     extends:

apps/cic-meta/bin/get.js

@@ -1,51 +0,0 @@
-#!/usr/bin/env node
-const colors = require('colors');
-const {Meta} = require("../dist");
-
-let { argv } = require('yargs')
-    .usage('Usage: $0 -m http://localhost:63380 -n publickeys')
-    .example(
-        '$0 -m http://localhost:63380 -n publickeys',
-        'Fetches the public keys blob from the meta server'
-    )
-    .option('m', {
-        alias: 'metaurl',
-        describe: 'The URL for the meta service',
-        demandOption: 'The meta url is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('n', {
-        alias: 'name',
-        describe: 'The name of the resource to be fetched from the meta service',
-        demandOption: 'The name of the resource is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('t', {
-        alias: 'type',
-        describe: 'The type of resource to be fetched from the meta service\n' +
-            'Options: `user`, `phone` and `custom`\n' +
-            'Defaults to `custom`',
-        type: 'string',
-        nargs: 1,
-    })
-    .epilog('Grassroots Economics (c) 2021')
-    .wrap(null);
-
-const metaUrl = argv.m;
-const resourceName = argv.n;
-let type = argv.t;
-if (type === undefined) {
-    type = 'custom'
-}
-
-(async () => {
-    const identifier = await Meta.getIdentifier(resourceName, type);
-    console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
-    const metaResponse = await Meta.get(identifier, metaUrl);
-    if (typeof metaResponse !== "object") {
-        console.error(colors.red('Metadata get failed!'));
-    }
-    console.log(colors.green(metaResponse));
-})();

apps/cic-meta/bin/set.js

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-const fs = require("fs");
-const colors = require('colors');
-const {Meta} = require("../dist");
-
-let { argv } = require('yargs')
-    .usage('Usage: $0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc')
-    .example(
-        '$0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc',
-        'Updates the public keys blob to the meta server'
-    )
-    .option('m', {
-        alias: 'metaurl',
-        describe: 'The URL for the meta service',
-        demandOption: 'The meta url is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('k', {
-        alias: 'privatekey',
-        describe: 'The PGP private key blob file used to sign the changes to the meta service',
-        demandOption: 'The private key file is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('n', {
-        alias: 'name',
-        describe: 'The name of the resource to be set or updated to the meta service',
-        demandOption: 'The name of the resource is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('r', {
-        alias: 'resource',
-        describe: 'The resource file to be set or updated to the meta service',
-        demandOption: 'The resource file is required',
-        type: 'string',
-        nargs: 1,
-    })
-    .option('t', {
-        alias: 'type',
-        describe: 'The type of resource to be set or updated to the meta service\n' +
-            'Options: `user`, `phone` and `custom`\n' +
-            'Defaults to `custom`',
-        type: 'string',
-        nargs: 1,
-    })
-    .epilog('Grassroots Economics (c) 2021')
-    .wrap(null);
-
-const metaUrl = argv.m;
-const privateKeyFile = argv.k;
-const resourceName = argv.n;
-const resourceFile = argv.r;
-let type = argv.t;
-if (type === undefined) {
-    type = 'custom'
-}
-
-const privateKey = readFile(privateKeyFile);
-const resource = readFile(resourceFile);
-
-(async () => {
-    if (privateKey && resource) {
-        const identifier = await Meta.getIdentifier(resourceName, type);
-        console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
-        const meta = new Meta(metaUrl, privateKey);
-        meta.onload = async (status) => {
-            const response = await meta.set(identifier, resource)
-            console.log(colors.green(response));
-        }
-    }
-})();
-
-function readFile(filename) {
-    if(!fs.existsSync(filename)) {
-        console.log(colors.red(`File ${filename} not found`));
-        return;
-    }
-    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
-}

apps/cic-meta/docker/Dockerfile

@@ -4,28 +4,29 @@ WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
-# required to build the cic-client-meta module
+COPY cic-meta/package.json \
+     ./
+
 COPY cic-meta/src/ src/
+COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
 
-# copy the dependencies
-COPY cic-meta/package.json .
-COPY cic-meta/tsconfig.json . 
-COPY cic-meta/webpack.config.js .
-     
 RUN npm install
 
-COPY cic-meta/tests/ tests/
+# see exports_dir gpg.ini
 COPY cic-meta/tests/*.asc /root/pgp/
+RUN alias tsc=node_modules/typescript/bin/tsc 
+
 
-# copy runtime configs
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
+# COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
-# db migrations
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-RUN alias tsc=node_modules/typescript/bin/tsc 
+#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
+
 COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/package.json

@@ -1,30 +1,22 @@
 {
-	"name": "@cicnet/cic-client-meta",
-	"version": "0.0.11",
+	"name": "cic-client-meta",
+	"version": "0.0.7-alpha.8",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
-	"bin": {
-		"meta-set": "bin/set.js",
-		"meta-get": "bin/get.js"
-	},
-	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
-		"test:coverage": "nyc mocha tests/*.ts --timeout 3000 --check-coverage=true",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
 		"clean": "rm -rf dist",
 		"prepare": "npm run build && npm run build-server",
-		"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts",
-		"publish": "npm publish --access public"
+                "start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts" 
 	},
 	"dependencies": {
-		"@cicnet/crdt-meta": "^0.0.10",
 		"@ethereumjs/tx": "^3.0.0-beta.1",
 		"automerge": "^0.14.1",
-		"colors": "^1.4.0",
+		"crdt-meta": "0.0.8",
 		"ethereumjs-wallet": "^1.0.1",
 		"ini": "^1.3.8",
 		"openpgp": "^4.10.8",
@@ -35,9 +27,7 @@
 	"devDependencies": {
 		"@types/mocha": "^8.0.3",
 		"mocha": "^8.2.0",
-		"nock": "^13.1.0",
 		"node-localstorage": "^2.1.6",
-		"nyc": "^15.1.0",
 		"ts-node": "^9.0.0",
 		"typescript": "^4.0.5",
 		"webpack": "^5.4.0",
@@ -53,26 +43,5 @@
 	"license": "GPL-3.0-or-later",
 	"engines": {
 		"node": ">=14.16.1"
-	},
-	"nyc": {
-		"include": [
-			"src/**/*.ts"
-		],
-		"extension": [
-			".ts"
-		],
-		"require": [
-			"ts-node/register"
-		],
-		"reporter": [
-			"text",
-			"html"
-		],
-		"sourceMap": true,
-		"instrument": true,
-		"branches": ">80",
-		"lines": ">80",
-		"functions": ">80",
-		"statements": ">80"
 	}
 }

apps/cic-meta/scripts/dumpconfig.js

@@ -1,4 +1,4 @@
-import { Config } from '@cicnet/crdt-meta';
+import { Config } from 'crdt-meta';
 const fs = require('fs');
 
 if (process.argv[2] === undefined) {

apps/cic-meta/scripts/server/handlers.ts

@@ -1,7 +1,7 @@
 import * as Automerge from 'automerge';
 import * as pgp from 'openpgp';
 
-import { Envelope, Syncable } from '@cicnet/crdt-meta';
+import { Envelope, Syncable } from 'crdt-meta';
 
 
 function handleNoMergeGet(db, digest, keystore) {
@@ -31,7 +31,7 @@ function handleNoMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('message', e);
+				console.error('mesage', e);
 				doh(e);
 			});
 		})
@@ -46,7 +46,7 @@ function handleServerMergePost(data, db, digest, keystore, signer) {
 			let e = undefined;
 			let s = undefined;
 			if (v === undefined) {
-				s = new Syncable(digest, o);
+				s = new Syncable(digest, data);
 				s.onwrap = (e) => {
 					whohoo(e.toJSON());
 				};

apps/cic-meta/scripts/server/server.ts

@@ -3,8 +3,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import * as handlers from './handlers';
-import { PGPKeyStore, PGPSigner, Config } from '@cicnet/crdt-meta';
-import { SqliteAdapter, PostgresAdapter } from '../../src/db';
+import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
 
 import { standardArgs } from './args';
 
@@ -204,7 +203,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty content', data);
+			console.error('empty onctent', data);
 			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;