Initial commit

.gitignore

@@ -0,0 +1,5 @@
+build/
+dist/
+*.egg-info
+*.pyc
+__pycache__

cic_sync_filter/__init__.py

@@ -0,0 +1 @@
+

cic_sync_filter/base.py

@@ -0,0 +1,17 @@
+class SyncFilter:
+    
+    def __init__(self):
+        self.exec_count = 0
+        self.match_count = 0
+
+
+    def filter(self, conn, block, tx, db_session):
+        self.exec_count += 1
+
+
+    def register_match(self):
+        self.match_count += 1
+
+
+    def to_logline(self, block, tx, v):
+        return '{} exec {} match {} block {} tx {}: {}'.format(self, self.exec_count, self.match_count, block.number, tx.index, v)

cic_sync_filter/callback.py

@@ -0,0 +1,134 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from cic_eth_registry.error import (
+    UnknownContractError,
+    NotAContractError,
+    )
+from chainlib.status import Status as TxStatus
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.error import RequestMismatchException
+from chainlib.eth.constant import ZERO_ADDRESS
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from eth_erc20 import ERC20
+from erc20_faucet import Faucet
+from chainsyncer.filter import SyncFilter
+
+# local imports
+#from cic_eth.eth.meta import ExtendedTx
+#from cic_eth.encode import tx_normalize
+from cic_sync_filter.parse import (
+        parse_transfer,
+        parse_transferfrom,
+        parse_giftto,
+        )
+
+logg = logging.getLogger(__name__)
+
+
+class CallbackFilter(SyncFilter):
+
+    trusted_addresses = []
+
+    def __init__(self, chain_spec, method, queue, caller_address=ZERO_ADDRESS):
+        super(CallbackFilter, self).__init__()
+        self.queue = queue
+        self.method = method
+        self.chain_spec = chain_spec
+        self.caller_address = caller_address
+
+
+    def call_back(self, transfer_type, result):
+        result['chain_spec'] = result['chain_spec'].asdict()
+        s = celery.signature(
+            self.method,
+            [
+                result,
+                transfer_type,
+                int(result['status_code'] != 0),
+            ],
+            queue=self.queue,
+            )
+        t = s.apply_async()
+        return t
+
+
+    def parse_data(self, tx, conn):
+        transfer_type = None
+        transfer_data = None
+        # TODO: what's with the mix of attributes and dict keys
+        logg.debug('have payload {}'.format(tx.payload))
+
+        logg.debug('tx status {}'.format(tx.status))
+
+        for parser in [
+                parse_transfer,
+                parse_transferfrom,
+                parse_giftto,
+                ]:
+            try:
+                if tx:
+                    (transfer_type, transfer_data) = parser(tx, conn, self.chain_spec, self.caller_address)
+                    if transfer_type == None:
+                        continue
+                break
+            except RequestMismatchException:
+                continue
+
+
+        logg.debug('resolved method {}'.format(transfer_type))
+
+        if transfer_data != None:
+            transfer_data['status'] = tx.status
+
+        return (transfer_type, transfer_data)
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        super(CallbackFilter, self).filter(conn, block, tx, db_session)
+        transfer_data = None
+        transfer_type = None
+        try:
+            (transfer_type, transfer_data) = self.parse_data(tx, conn)
+        except TypeError:
+            logg.debug('invalid method data length for tx {}'.format(tx.hash))
+            return
+
+        if len(tx.payload) < 8:
+            logg.debug('callbacks filter data length not sufficient for method signature in tx {}, skipping'.format(tx.hash))
+            return
+
+        logg.debug('checking callbacks filter input {}'.format(tx.payload[:8]))
+    
+        t = None
+        if transfer_data != None:
+            token_symbol = None
+            result = None
+            try:
+                tokentx = ExtendedTx(conn, tx.hash, self.chain_spec)
+                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses, caller_address=self.caller_address)
+                tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
+                if transfer_data['status'] == 0:
+                    tokentx.set_status(1)
+                else:
+                    tokentx.set_status(0)
+                result = tokentx.asdict()
+                t = self.call_back(transfer_type, result)
+                self.register_match()
+                logline = 'callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue)
+                logline = self.to_logline(block, tx, logline)
+                logg.info(logline)
+            except UnknownContractError:
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
+            except NotAContractError:
+                logg.debug('callback filter {}:{} skipping "transfer" on non-contract address {} tx {}'.format(self.queue, self.method, transfer_data['to'], tx.hash))
+    
+        return t
+
+    def __str__(self):
+        return 'callbackfilter'

cic_sync_filter/convert.py

@@ -0,0 +1,61 @@
+
+#__convert_log_hash = '0x7154b38b5dd31bb3122436a96d4e09aba5b323ae1fd580025fab55074334c095' # keccak256(Conversion(address,address,address,uint256,uint256,address)
+
+#def parse_convert_log(w3, entry):
+#    data = entry.data[2:]
+#    from_amount = int(data[:64], 16)
+#    to_amount = int(data[64:128], 16)
+#    holder_address_hex_raw = '0x' + data[-40:]
+#    holder_address_hex = w3.toChecksumAddress(holder_address_hex_raw)
+#    o = {
+#            'from_amount': from_amount,
+#            'to_amount': to_amount,
+#            'holder_address': holder_address_hex
+#            }
+#    logg.debug('parsed convert log {}'.format(o))
+#    return o
+
+
+#def convert_filter(w3, tx, rcpt, chain_spec):
+#    destination_token_address = None
+#    recipient_address = None
+#    amount = 0
+#    for l in rcpt['logs']:
+#        event_topic_hex = l['topics'][0].hex()
+#        if event_topic_hex == __convert_log_hash:
+#            tx_hash_hex = tx['hash'].hex()
+#            try:
+#                convert_transfer = TxConvertTransfer.get(tx_hash_hex)
+#            except UnknownConvertError:
+#                logg.warning('skipping unknown convert tx {}'.format(tx_hash_hex))
+#                continue
+#            if convert_transfer.transfer_tx_hash != None:
+#                logg.warning('convert tx {} cache record already has transfer hash {}, skipping'.format(tx_hash_hex, convert_transfer.transfer_hash))
+#                continue
+#            recipient_address = convert_transfer.recipient_address
+#            logg.debug('found convert event {} recipient'.format(tx_hash_hex, recipient_address))
+#            r = parse_convert_log(l)
+#            destination_token_address = l['topics'][3][-20:]
+#
+#    if destination_token_address == zero_address or destination_token_address == None:
+#        return None
+#
+#    destination_token_address_hex = destination_token_address.hex()
+#    s = celery.signature(
+#            'cic_eth.eth.bancor.transfer_converted',
+#            [
+#                [{
+#                    'address': w3.toChecksumAddress(destination_token_address_hex),
+#                    }],
+#                r['holder_address'],
+#                recipient_address,
+#                r['to_amount'],
+#                tx_hash_hex,
+#                str(chain_spec),
+#                ],
+#                queue=queue,
+#            )
+#    logg.info('sending tx signature {}'.format(s))
+#    t = s.apply_async()
+#    logg.debug('submitted transfer after convert task uuid {} {}'.format(t, t.successful()))
+#    return t

cic_sync_filter/gas.py

@@ -0,0 +1,80 @@
+# standard imports
+import logging
+
+# external imports
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.eth.tx import unpack
+from chainqueue.db.enum import StatusBits
+from chainqueue.db.models.tx import TxCache
+from chainqueue.db.models.otx import Otx
+from chainlib.eth.address import to_checksum_address
+
+# local imports
+from cic_eth.db.models.base import SessionBase
+from cic_eth.eth.gas import create_check_gas_task
+from cic_eth.queue.query import get_paused_tx
+from cic_eth.encode import tx_normalize
+from .base import SyncFilter
+
+logg = logging.getLogger()
+
+
+class GasFilter(SyncFilter):
+
+    def __init__(self, chain_spec, queue=None):
+        super(GasFilter, self).__init__()
+        self.queue = queue
+        self.chain_spec = chain_spec
+
+
+    def filter(self, conn, block, tx, db_session):
+        super(GasFilter, self).filter(conn, block, tx, db_session)
+        if tx.value > 0 or len(tx.payload) == 0:
+            tx_hash_hex = add_0x(tx.hash)
+
+            sender_target = tx_normalize.wallet_address(tx.inputs[0])
+            session = SessionBase.bind_session(db_session)
+            q = session.query(TxCache.recipient)
+            q = q.filter(TxCache.sender==sender_target)
+            r = q.all()
+
+            logline = None
+            if len(r) == 0:
+                logline = 'unsolicited gas refill tx {}; cannot find {} among senders'.format(tx_hash_hex, tx.outputs[0])
+                logline = self.to_logline(block, tx, logline)
+                logg.info(logline)
+                SessionBase.release_session(session)
+                return None
+
+            self.register_match()
+
+            txs = get_paused_tx(self.chain_spec, status=StatusBits.GAS_ISSUES, sender=sender_target, session=session, decoder=unpack)
+
+            SessionBase.release_session(session)
+
+            t = None
+            address = to_checksum_address(sender_target)
+            if len(txs) > 0:
+                s = create_check_gas_task(
+                        list(txs.values()),
+                        self.chain_spec,
+                        address,
+                        0,
+                        tx_hashes_hex=list(txs.keys()),
+                        queue=self.queue,
+                )
+                t = s.apply_async()
+                logline = 'resuming {} gas-in-waiting txs for {}'.format(len(txs), sender_target)
+            else:
+                logline = 'gas refill tx {}'.format(tx)
+
+            logline = self.to_logline(block, tx, logline)
+            logg.info(logline)
+            return t
+
+
+    def __str__(self):
+        return 'gasfilter'

cic_sync_filter/log.py

@@ -0,0 +1,9 @@
+from .base import SyncFilter
+
+
+class LogFilter(SyncFilter):
+    
+    def filter(self, conn, block, tx, db_session=None):
+        logg.debug('block {} tx {}'.format(block, tx))
+
+

cic_sync_filter/parse.py

@@ -0,0 +1,77 @@
+from chainlib.eth.constant import ZERO_ADDRESS
+
+def parse_transfer(tx, conn, chain_spec, caller_address=ZERO_ADDRESS):
+    if not tx.payload:
+        return (None, None)
+    r = ERC20.parse_transfer_request(tx.payload)
+    transfer_data = {}
+    transfer_data['to'] = tx_normalize.wallet_address(r[0])
+    transfer_data['value'] = r[1]
+    transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+    transfer_data['token_address'] = tx.inputs[0]
+    return ('transfer', transfer_data)
+
+
+def parse_transferfrom(tx, conn, chain_spec, caller_address=ZERO_ADDRESS):
+    if not tx.payload:
+        return (None, None)
+    r = ERC20.parse_transfer_from_request(tx.payload)
+    transfer_data = {}
+    transfer_data['from'] = tx_normalize.wallet_address(r[0])
+    transfer_data['to'] = tx_normalize.wallet_address(r[1])
+    transfer_data['value'] = r[2]
+    transfer_data['token_address'] = tx.inputs[0]
+    return ('transferfrom', transfer_data)
+
+
+def parse_gas(tx, conn, chain_spec, caller_address=ZERO_ADDRESS):
+    r = (None, None,)
+    if tx.value > 0 or len(tx.payload) == 0:
+        transfer_data = {}
+        transfer_data['to'] = tx_normalize.wallet_address(tx.inputs[0])
+        transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+        transfer_data['value'] = tx.value
+        transfer_data['token_address'] = None
+        r = ('gas', transfer_data,)
+    else:
+        logg.info('value {} payload {}'.format(tx.value, tx.payload))
+    return r
+
+
+def parse_giftto(tx, conn, chain_spec, caller_address=ZERO_ADDRESS):
+    if not tx.payload:
+        return (None, None)
+    r = Faucet.parse_give_to_request(tx.payload)
+    transfer_data = {}
+    transfer_data['to'] = tx_normalize.wallet_address(r[0])
+    transfer_data['value'] = tx.value
+    transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+    #transfer_data['token_address'] = tx.inputs[0]
+    faucet_contract = tx.inputs[0]
+
+    c = Faucet(chain_spec)
+
+    o = c.token(faucet_contract, sender_address=caller_address)
+    r = conn.do(o)
+    transfer_data['token_address'] = add_0x(c.parse_token(r))
+
+    o = c.token_amount(faucet_contract, sender_address=caller_address)
+    r = conn.do(o)
+    transfer_data['value'] = c.parse_token_amount(r)
+
+    return ('tokengift', transfer_data)
+
+
+
+def parse_register(tx, conn, chain_spec, caller_address=ZERO_ADDRESS):
+    if not tx.payload:
+        return (None, None)
+    r = AccountRegistry.parse_add_request(tx.payload)
+    transfer_data = {
+        'value': None,
+        'token_address': None,
+            }
+    transfer_data['to'] = tx_normalize.wallet_address(r)
+    transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0])
+    return ('account_register', transfer_data,)
+

cic_sync_filter/register.py

@@ -0,0 +1,70 @@
+# standard imports
+import logging
+
+# third-party imports
+import celery
+from chainlib.eth.address import to_checksum_address
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+
+# local imports
+from .base import SyncFilter
+
+logg = logging.getLogger(__name__)
+
+account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
+
+
+class RegistrationFilter(SyncFilter):
+
+    def __init__(self, chain_spec, contract_address, queue=None):
+        super(RegistrationFilter, self).__init__()
+        self.chain_spec = chain_spec
+        self.queue = queue
+        self.contract_address = contract_address
+
+
+    def filter(self, conn, block, tx, db_session=None): 
+        super(RegistrationFilter, self).filter(conn, block, tx, db_session)
+        if self.contract_address != tx.inputs[0]:
+            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
+            return None
+
+        for l in tx.logs:
+            event_topic_hex = l['topics'][0]
+            if event_topic_hex == account_registry_add_log_hash:
+                self.register_match()
+                # TODO: use abi conversion method instead
+
+                address_hex = strip_0x(l['topics'][1])[64-40:]
+                address = to_checksum_address(add_0x(address_hex))
+                s_nonce = celery.signature(
+                    'cic_eth.eth.nonce.reserve_nonce',
+                    [
+                        address,
+                        self.chain_spec.asdict(),
+                        ],
+                    queue=self.queue,
+                    )
+                s_gift = celery.signature(
+                    'cic_eth.eth.account.gift',
+                    [
+                        self.chain_spec.asdict(),
+                        ],
+                    queue=self.queue,
+                    )
+                s_nonce.link(s_gift)
+                t = s_nonce.apply_async()
+
+                logline = 'request token gift to {}'.format(address)
+                logline = self.to_logline(block, tx, logline)
+                logg.info(logline)
+
+                return t
+
+
+    def __str__(self):
+        return 'registrationfilter'
+

cic_sync_filter/straggler.py

@@ -0,0 +1,68 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainqueue.sql.state import (
+    obsolete_by_cache,
+    set_fubar,
+    )
+from chainqueue.error import TxStateChangeError
+from hexathon import to_int as hex_to_int
+from chainlib.eth.gas import balance
+from chainqueue.sql.query import get_tx_cache
+from chainqueue.enum import StatusBits
+
+logg = logging.getLogger()
+
+
+class StragglerFilter:
+
+    def __init__(self, chain_spec, gas_balance_threshold, queue='cic-eth'):
+        self.chain_spec = chain_spec
+        self.queue = queue
+        self.gas_balance_threshold = gas_balance_threshold
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        txc = get_tx_cache(self.chain_spec, tx.hash, session=db_session)
+        if txc['status_code'] & StatusBits.GAS_ISSUES > 0:
+            o = balance(tx.outputs[0])
+            r = conn.do(o)
+            gas_balance = hex_to_int(r)
+
+            t = None
+            if gas_balance < self.gas_balance_threshold:
+                logg.debug('WAITFORGAS tx ignored since gas balance {} is below threshold {}'.format(gas_balance, self.gas_balance_threshold))
+                s_touch = celery.signature(
+                        'cic_eth.queue.state.set_checked',
+                        [
+                            self.chain_spec.asdict(),
+                            tx.hash,
+                            ],
+                        queue=self.queue,
+                )
+                t = s_touch.apply_async()
+                return t
+
+
+        try:
+            obsolete_by_cache(self.chain_spec, tx.hash, False, session=db_session)
+        except TxStateChangeError:
+            set_fubar(self.chain_spec, tx.hash, session=db_session)
+            return False
+
+        s_send = celery.signature(
+                'cic_eth.eth.gas.resend_with_higher_gas',
+                [
+                    tx.hash,
+                    self.chain_spec.asdict(),
+                ],
+                queue=self.queue,
+        )
+        t = s_send.apply_async()
+        return t
+
+
+    def __str__(self):
+        return 'stragglerfilter'

cic_sync_filter/token.py

@@ -0,0 +1,83 @@
+# standard imports
+import logging
+
+# external imports
+from eth_erc20 import ERC20
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.address import is_same_address
+from chainlib.eth.error import RequestMismatchException
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.erc20 import ERC20Token
+from cic_eth_registry.error import UnknownContractError
+from eth_token_index import TokenUniqueSymbolIndex
+import celery
+
+# local imports
+from .base import SyncFilter
+
+logg = logging.getLogger(__name__)
+
+
+class TokenFilter(SyncFilter):
+
+    def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
+        super(TokenFilter, self).__init__()
+        self.queue = queue
+        self.chain_spec = chain_spec
+        self.caller_address = call_address
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        super(TokenFilter, self).filter(conn, block, tx, db_session)
+        if not tx.payload:
+            return None
+
+        try:
+            r = ERC20.parse_transfer_request(tx.payload)
+        except RequestMismatchException:
+            return None
+
+        token_address = tx.inputs[0]
+        token = ERC20Token(self.chain_spec, conn, token_address)
+
+        registry = CICRegistry(self.chain_spec, conn)
+        r = None
+        try:
+            r = registry.by_name(token.symbol, sender_address=self.caller_address)
+        except UnknownContractError:
+            logg.debug('token {} not in registry, skipping'.format(token.symbol))
+            return None
+
+        if is_same_address(r, ZERO_ADDRESS):
+            return None
+
+        self.register_match()
+
+        enc = ABIContractEncoder()
+        enc.method('transfer')
+        method = enc.get()
+
+        s = celery.signature(
+                'cic_eth.eth.gas.apply_gas_value_cache',
+                [
+                    token_address,
+                    method,
+                    tx.gas_used,
+                    tx.hash,
+                    ],
+                queue=self.queue,
+                )
+        t = s.apply_async()
+        
+        logline = 'erc20 transfer {} {}'.format(token.symbol, tx.hash)
+        logline = self.to_logline(block, tx, logline)
+        logg.info(logline)
+        return t
+
+
+    def __str__(self):
+        return 'erc20 tx filter'

cic_sync_filter/transferauth.py

@@ -0,0 +1,90 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.contract import (
+        ABIContractType,
+        abi_decode_single,
+        )
+from cic_eth_registry import CICRegistry
+from erc20_transfer_authorization import TransferAuthorization
+
+# local imports
+from cic_eth.encode import tx_normalize
+from .base import SyncFilter
+
+
+logg = logging.getLogger(__name__)
+
+
+class TransferAuthFilter(SyncFilter):
+
+    def __init__(self, registry, chain_spec, conn, queue=None, call_address=ZERO_ADDRESS):
+        self.queue = queue
+        self.chain_spec = chain_spec
+        registry = CICRegistry(chain_spec, conn)
+        self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
+
+
+    def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
+
+        if tx.payload == None:
+            logg.debug('no payload')
+            return False
+
+        payloadlength = len(tx.payload)
+        if payloadlength != 8+256:
+            logg.debug('{} below minimum length for a transfer auth call'.format(payloadlength))
+            logg.debug('payload {}'.format(tx.payload))
+            return False
+
+        recipient = tx.inputs[0]
+        #if recipient != self.transfer_request_contract.address():
+        if recipient != self.transfer_request_contract:
+            logg.debug('not our transfer auth contract address {}'.format(recipient))
+            return False
+
+        r = TransferAuthorization.parse_create_request_request(tx.payload) 
+          
+        sender = tx_normalize.wallet_address(r[0])
+        recipient = tx_normalize.wallet_address(r[1])
+        token = tx_normalize.executable_address(r[2])
+        value = r[3]
+
+        token_data = {
+            'address': token,
+            }
+
+        s_nonce = celery.signature(
+            'cic_eth.eth.nonce.reserve_nonce',
+            [
+                [token_data],
+                self.chain_spec.asdict(), 
+                sender,
+                ],
+            queue=self.queue,
+            )
+        s_approve = celery.signature(
+            'cic_eth.eth.erc20.approve',
+            [
+                sender,
+                recipient,
+                value,
+                self.chain_spec.asdict(),
+                ],
+            queue=self.queue,
+            )
+        s_nonce.link(s_approve)
+        t = s_nonce.apply_async()
+        return t
+
+
+    def __str__(self):
+        return 'cic-eth transfer auth filter'

cic_sync_filter/tx.py

@@ -0,0 +1,70 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from hexathon import (
+        add_0x,
+        )
+from chainsyncer.db.models.base import SessionBase
+from chainqueue.db.models.otx import Otx
+from chainlib.status import Status
+
+# local imports
+from .base import SyncFilter
+
+logg = logging.getLogger(__name__)
+
+
+class TxFilter(SyncFilter):
+
+    def __init__(self, chain_spec, queue):
+        super(TxFilter, self).__init__()
+        self.queue = queue
+        self.chain_spec = chain_spec
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        super(TxFilter, self).filter(conn, block, tx, db_session)
+        db_session = SessionBase.bind_session(db_session)
+        tx_hash_hex = tx.hash
+        otx = Otx.load(add_0x(tx_hash_hex), session=db_session)
+        if otx == None:
+            logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
+            return None
+
+        self.register_match()
+
+        db_session.flush()
+        SessionBase.release_session(db_session)
+        s_final_state = celery.signature(
+                'cic_eth.queue.state.set_final',
+                [
+                    self.chain_spec.asdict(),
+                    add_0x(tx_hash_hex),
+                    tx.block.number,
+                    tx.index,
+                    tx.status == Status.ERROR,
+                    ],
+                queue=self.queue,
+                )
+        s_obsolete_state = celery.signature(
+                'cic_eth.queue.state.obsolete',
+                [
+                    self.chain_spec.asdict(),
+                    add_0x(tx_hash_hex),
+                    True,
+                    ],
+                queue=self.queue,
+                )
+        t = celery.group(s_obsolete_state, s_final_state)()
+
+        logline = 'otx filter match on {}'.format(otx.tx_hash)
+        logline = self.to_logline(block, tx, logline)
+        logg.info(logline)
+
+        return t
+
+
+    def __str__(self):
+        return 'otx filter'

requirements.txt

@@ -0,0 +1,5 @@
+chainsyncer~=0.3.2
+chainqueue~=0.0.6rc9
+celery==4.4.7
+cic-eth-registry~=0.6.9
+erc20-faucet~=0.4.0

test_requirements.txt

@@ -0,0 +1,6 @@
+pytest==6.2.5
+pytest-cov==2.10.1
+eth-tester==0.5.0b3
+py-evm==0.3.0a20
+eth-erc20~=0.2.0
+pytest-celery==0.0.0a1

tests/conftest.py

@@ -0,0 +1,9 @@
+#from chainlib.eth.pytest import * 
+#from cic_eth_registry.pytest import *
+#from pytest_cic.fixtures_database import *
+from cic_eth.pytest.fixtures_database import *
+from cic_eth.pytest.fixtures_contract import *
+from cic_eth.pytest.fixtures_config import *
+from cic_eth.pytest.fixtures_token import *
+from chainlib.eth.pytest import *
+

tests/test_callback_filter.py

@@ -0,0 +1,235 @@
+# standard import
+import logging
+import datetime
+import os
+
+# external imports
+import pytest
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        transaction,
+        Tx,
+        )
+from chainlib.eth.block import Block
+from eth_erc20 import ERC20
+#from sarafu_faucet import MinterFaucet
+from erc20_faucet import Faucet
+from eth_accounts_index.registry import AccountRegistry
+from potaahto.symbols import snake_and_camel
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+
+# local imports
+from cic_sync_filter.callback import CallbackFilter
+from cic_sync_filter.parse import (
+        parse_transfer,
+        parse_transferfrom,
+        parse_giftto,
+        )
+
+logg = logging.getLogger()
+
+
+def test_transfer_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        token_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    logg.debug(r)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = parse_transfer(tx, eth_rpc, fltr.chain_spec, fltr.caller_address)
+
+    assert transfer_type == 'transfer'
+
+
+def test_transfer_from_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        token_roles,
+        contract_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+
+    (tx_hash_hex, o) = txf.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer_from(foo_token, agent_roles['ALICE'], token_roles['FOO_TOKEN_OWNER'], agent_roles['BOB'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = parse_transferfrom(tx, eth_rpc, fltr.chain_spec, fltr.caller_address)
+
+    assert transfer_type == 'transferfrom'
+
+
+def test_faucet_gift_to_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        agent_roles,
+        contract_roles,
+        faucet,
+        account_registry,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=800000)
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], rpc)
+    txf = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+   
+    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], rpc)
+    #txf = MinterFaucet(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    txf = Faucet(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.give_to(faucet, agent_roles['ALICE'], agent_roles['ALICE'])
+    r = rpc.do(o)
+
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+    (transfer_type, transfer_data) = parse_giftto(tx, eth_rpc, fltr.chain_spec, fltr.caller_address)
+
+    assert transfer_type == 'tokengift'
+    assert transfer_data['token_address'] == foo_token
+
+
+def test_callback_filter_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        foo_token,
+        token_roles,
+        agent_roles,
+        contract_roles,
+        register_lookups,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], rpc)
+    gas_oracle = OverrideGasOracle(conn=rpc, limit=200000)
+   
+    txf = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = txf.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 1024)
+    r = rpc.do(o)
+    
+    o = transaction(tx_hash_hex)
+    r = rpc.do(o)
+    logg.debug(r)
+
+    mockblock_src = {
+        'hash': add_0x(os.urandom(32).hex()),
+        'number': '0x2a',
+        'transactions': [tx_hash_hex],
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+            }
+    mockblock = Block(mockblock_src)
+
+    tx_src = snake_and_camel(r)
+    tx = Tx(tx_src, block=mockblock)
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    rcpt = snake_and_camel(r)
+    tx.block.hash = rcpt['block_hash']
+    tx.apply_receipt(rcpt)
+
+    fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])
+
+    class CallbackMock:
+
+        def __init__(self):
+            self.results = {}
+            self.queue = 'test'
+
+        def call_back(self, transfer_type, result):
+            self.results[transfer_type] = result
+            logg.debug('result {}'.format(result))
+            return self
+
+    mock = CallbackMock()
+    fltr.call_back = mock.call_back
+
+    fltr.filter(eth_rpc, mockblock, tx, init_database)
+
+    assert mock.results.get('transfer') != None
+    assert mock.results['transfer']['destination_token'] == strip_0x(foo_token)

tests/test_filter_bogus.py

@@ -0,0 +1,39 @@
+# local imports
+from cic_sync_filter.gas import GasFilter
+from cic_sync_filter.transferauth import TransferAuthFilter
+from cic_sync_filter.callback import CallbackFilter
+from cic_sync_filter.straggler import StragglerFilter
+from cic_sync_filter.tx import TxFilter
+from cic_sync_filter.register import RegistrationFilter
+
+
+# Hit tx mismatch paths on all filters
+def test_filter_bogus(
+        init_database,
+        bogus_tx_block,
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        transfer_auth,
+        cic_registry,
+        contract_roles,
+        register_lookups,
+        account_registry,
+        ):
+
+    fltrs = [
+        TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
+        GasFilter(default_chain_spec, queue=None),
+        TxFilter(default_chain_spec, None),
+        CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
+        StragglerFilter(default_chain_spec, None),
+        RegistrationFilter(default_chain_spec, account_registry, queue=None),
+        ]
+      
+    for fltr in fltrs:
+        r = None
+        try:
+            r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
+        except:
+            pass
+        assert not r

tests/test_gas_filter.py

@@ -0,0 +1,114 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.sql.state import (
+        set_waitforgas,
+        )
+from hexathon import strip_0x
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+
+# local imports
+from cic_eth.runnable.daemons.filters.gas import GasFilter
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.encode import tx_normalize
+from cic_eth.queue.tx import queue_create
+
+logg = logging.getLogger()
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
+    init_database.commit()
+
+    tx_hash_hex_wait = tx_hash_hex
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
+
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['CAROL'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+
+    queue_create(
+            default_chain_spec,
+            43,
+            agent_roles['CAROL'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+    init_database.commit()
+
+    fltr = GasFilter(default_chain_spec, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    r = init_database.execute('select * from otx inner join tx_cache on otx.id = tx_cache.otx_id')
+    for v in r:
+        logg.info('have row {}'.format(v))
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+    init_database.commit()
+
+    otx = Otx.load(tx_hash_hex_wait, session=init_database)
+    assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

tests/test_register_filter.py

@@ -0,0 +1,95 @@
+# standard imports
+import logging
+import os
+
+# external imports
+from eth_accounts_index.registry import AccountRegistry
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import(
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.address import (
+        to_checksum_address,
+        )
+from erc20_faucet import Faucet
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+
+# local imports
+from cic_eth.runnable.daemons.filters.register import RegistrationFilter
+from cic_eth.queue.query import get_account_tx_local
+
+logg = logging.getLogger()
+
+
+def test_register_filter(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        account_registry,
+        faucet,
+        register_lookups,
+        contract_roles,
+        agent_roles,
+        cic_registry,
+        init_celery_tasks,
+        celery_session_worker,
+        caplog,
+        ):
+
+    nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
+
+    c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
+    r = eth_rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block, rcpt=rcpt)
+    tx.apply_receipt(rcpt)
+
+    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(os.urandom(20).hex()), queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert t == None
+
+    fltr = RegistrationFilter(default_chain_spec, to_checksum_address(account_registry), queue=None)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    logg.debug('t {}'.format(t))
+
+    t.get_leaf()
+    assert t.successful()
+
+    gift_txs = get_account_tx_local(default_chain_spec, agent_roles['ALICE'], as_sender=True, session=init_database)
+    ks = list(gift_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    gift = Faucet.parse_give_to_request(gift_tx['data'])
+    assert add_0x(gift[0]) == agent_roles['ALICE']

tests/test_straggler_filter.py

@@ -0,0 +1,209 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import (
+        OverrideNonceOracle,
+        RPCNonceOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        receipt,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        set_sent,
+        set_waitforgas,
+        )
+
+from hexathon import (
+        strip_0x,
+        uniform as hex_uniform,
+        )
+
+# local imports
+from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
+from cic_eth.eth.gas import cache_gas_data
+from cic_eth.queue.query import (
+        get_tx_local,
+        get_account_tx_local,
+        )
+
+logg = logging.getLogger()
+
+
+def test_straggler_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    fltr = StragglerFilter(default_chain_spec, 0, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    t = fltr.filter(None, block, tx, db_session=init_database)
+    logg.debug('foo')
+    tx_hash_hex_successor = t.get_leaf()
+    logg.debug('bar')
+
+    assert t.successful()
+    assert tx_hash_hex_successor != tx_hash_hex
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE > 0
+    assert otx.status & (StatusBits.FINAL | StatusBits.QUEUED | StatusBits.RESERVED) == 0
+
+    otx = Otx.load(tx_hash_hex_successor, session=init_database)
+    assert otx.status == StatusBits.QUEUED
+
+
+
+def test_waitforgas_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        whoever,
+        ):
+
+    safe_gas = 1000000000000000000
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(whoever, 0)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            0,
+            whoever,
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
+
+    fltr = StragglerFilter(default_chain_spec, safe_gas, queue=None)
+     
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    t.get_leaf()
+    assert t.successful()
+
+    otx = get_tx_local(default_chain_spec, tx.hash, session=init_database)
+    assert otx['status'] == StatusBits.GAS_ISSUES
+ 
+    nonce_oracle = RPCNonceOracle(agent_roles['CAROL'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create(agent_roles['CAROL'], whoever, safe_gas - 1)
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    t.get_leaf()
+    assert t.successful()
+
+    otx = get_tx_local(default_chain_spec, tx.hash, session=init_database)
+    assert otx['status'] == StatusBits.GAS_ISSUES
+ 
+
+    nonce_oracle = RPCNonceOracle(agent_roles['CAROL'], conn=eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create(agent_roles['CAROL'], whoever, 1)
+    r = eth_rpc.do(o)
+
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    init_database.commit()
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    t.get_leaf()
+    
+    otx = get_tx_local(default_chain_spec, tx.hash, session=init_database)
+    assert otx['status'] & StatusBits.OBSOLETE > 0
+
+    txs = get_account_tx_local(default_chain_spec, whoever, session=init_database)
+    assert len(txs.keys()) == 2
+    for k in txs.keys():
+        if hex_uniform(strip_0x(tx.hash)) != hex_uniform(strip_0x(k)):
+            otx = get_tx_local(default_chain_spec, k, session=init_database)
+            assert otx['status'] == StatusBits.QUEUED
+ 

tests/test_token_filter.py

@@ -0,0 +1,127 @@
+# external imports
+import pytest
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        receipt,
+        raw,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        Block,
+        block_latest,
+        block_by_number,
+        )
+from chainlib.eth.address import is_same_address
+from chainlib.eth.contract import ABIContractEncoder
+from hexathon import strip_0x
+from eth_token_index import TokenUniqueSymbolIndex
+from cic_eth_registry.error import UnknownContractError
+
+# local imports
+from cic_eth.runnable.daemons.filters.token import TokenFilter
+from cic_eth.db.models.gas_cache import GasCache
+from cic_eth.db.models.base import SessionBase
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        contract_roles,
+        agent_roles,
+        token_roles,
+        foo_token,
+        token_registry,
+        register_lookups,
+        register_tokens,
+        celery_session_worker,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE'])
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    tx.apply_receipt(rcpt)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    assert t.get() == None
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
+    (tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(GasCache)
+    q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex))
+    o = q.first()
+
+    assert is_same_address(o.address, strip_0x(foo_token))
+    assert o.value > 0
+
+    enc = ABIContractEncoder()
+    enc.method('transfer')
+    method = enc.get()
+
+    assert o.method == method
+
+@pytest.mark.xfail(raises=UnknownContractError)
+def test_filter_unknown_contract_error(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        contract_roles,
+        agent_roles,
+        token_roles,
+        foo_token,
+        register_lookups,
+        celery_session_worker,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
+    
+    fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE'])
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src)
+    t = fltr.filter(eth_rpc, None, tx, db_session=init_database)

tests/test_transferauth_filter.py

@@ -0,0 +1,81 @@
+# external imports
+from erc20_transfer_authorization import TransferAuthorization
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.tx import (
+        receipt,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from hexathon import strip_0x
+from chainqueue.sql.query import get_account_tx
+
+# local imports
+from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
+from cic_eth.encode import tx_normalize
+
+
+def test_filter_transferauth(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        contract_roles,
+        transfer_auth,
+        foo_token,
+        celery_session_worker,
+        register_lookups,
+        init_custodial,
+        cic_registry,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
+    c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
+    
+    r = rpc.do(o)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
+
+    o = receipt(tx_hash_hex)
+    r = rpc.do(o)
+    assert r['status'] == 1
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+
+    fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get_leaf()
+    assert t.successful()
+
+    #approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
+    approve_txs = get_account_tx(default_chain_spec.asdict(), tx_normalize.wallet_address(agent_roles['ALICE']), as_sender=True, session=init_database)
+    ks = list(approve_txs.keys())
+    assert len(ks) == 1
+
+    tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
+    tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
+    approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
+
+    c = ERC20(default_chain_spec)
+    approve = c.parse_approve_request(approve_tx['data']) 
+    assert approve[0] == strip_0x(agent_roles['BOB'])

tests/test_tx_filter.py

@@ -0,0 +1,110 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import OverrideNonceOracle
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainqueue.db.models.otx import Otx
+from chainqueue.db.enum import StatusBits
+from chainqueue.sql.tx import create as queue_create
+from chainqueue.sql.state import (
+        set_reserved,
+        set_ready,
+        set_sent,
+        )
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.runnable.daemons.filters.tx import TxFilter
+from cic_eth.eth.gas import cache_gas_data
+
+
+def test_filter_tx(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        celery_session_worker,
+        ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+    tx_hash_hex_orig = tx_hash_hex
+
+    gas_oracle = OverrideGasOracle(price=1100000000, limit=21000)
+    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
+    queue_create(
+            default_chain_spec,
+            42,
+            agent_roles['ALICE'],
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            session=init_database,
+            )
+    cache_gas_data(
+            tx_hash_hex,
+            tx_signed_raw_hex,
+            default_chain_spec.asdict(),
+            )
+
+    set_ready(default_chain_spec, tx_hash_hex, session=init_database)
+    set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
+    set_sent(default_chain_spec, tx_hash_hex, session=init_database)
+
+    fltr = TxFilter(default_chain_spec, None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+    t.get()
+    assert t.successful()
+
+    otx = Otx.load(tx_hash_hex_orig, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
+    assert otx.status & StatusBits.FINAL == StatusBits.FINAL
+
+    otx = Otx.load(tx_hash_hex, session=init_database)
+    assert otx.status & StatusBits.OBSOLETE == 0
+    assert otx.status & StatusBits.FINAL == StatusBits.FINAL