From 69d31d202bece14325774a5947665db2fe9b30d5 Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 30 Sep 2021 15:26:55 +0200 Subject: [PATCH] Fix tx listing, rehabilitate api and filter tests (hex case problems) --- apps/cic-eth/cic_eth/api/api_task.py | 3 ++ apps/cic-eth/cic_eth/eth/account.py | 13 +++++--- apps/cic-eth/cic_eth/eth/erc20.py | 20 +++++++---- apps/cic-eth/cic_eth/eth/meta.py | 7 ++-- apps/cic-eth/cic_eth/ext/tx.py | 7 ++-- apps/cic-eth/cic_eth/queue/query.py | 1 + .../runnable/daemons/filters/register.py | 3 +- apps/cic-eth/requirements.txt | 1 + .../tests/filters/test_callback_filter.py | 10 ++++-- .../tests/filters/test_register_filter.py | 11 ++++--- .../tests/filters/test_transferauth_filter.py | 6 ++-- apps/cic-eth/tests/task/api/test_app.py | 9 +++++ apps/cic-eth/tests/task/api/test_balance.py | 9 +++-- apps/cic-eth/tests/task/api/test_list.py | 33 ++++++++++++------- 14 files changed, 93 insertions(+), 40 deletions(-) diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 79501d30..1ae14123 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -9,6 +9,7 @@ import logging # external imports import celery from chainlib.chain import ChainSpec +from hexathon import strip_0x # local imports from cic_eth.api.base import ApiBase @@ -254,6 +255,8 @@ class Api(ApiBase): :returns: uuid of root task :rtype: celery.Task """ + #from_address = strip_0x(from_address) + #to_address = strip_0x(to_address) s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', [ diff --git a/apps/cic-eth/cic_eth/eth/account.py b/apps/cic-eth/cic_eth/eth/account.py index 42bccec5..ffcb822d 100644 --- a/apps/cic-eth/cic_eth/eth/account.py +++ b/apps/cic-eth/cic_eth/eth/account.py @@ -49,6 +49,7 @@ from cic_eth.queue.tx import ( from cic_eth.encode import ( unpack_normal, ZERO_ADDRESS_NORMAL, + tx_normalize, ) logg = logging.getLogger() @@ -298,13 +299,15 @@ def cache_gift_data( tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx = unpack_normal(tx_signed_raw_bytes, chain_spec) tx_data = Faucet.parse_give_to_request(tx['data']) + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx['to']) session = self.create_session() tx_dict = { 'hash': tx['hash'], - 'from': tx['from'], - 'to': tx['to'], + 'from': sender_address, + 'to': recipient_address, 'source_token': ZERO_ADDRESS_NORMAL, 'destination_token': ZERO_ADDRESS_NORMAL, 'from_value': 0, @@ -338,12 +341,14 @@ def cache_account_data( tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx = unpack_normal(tx_signed_raw_bytes, chain_spec) tx_data = AccountsIndex.parse_add_request(tx['data']) + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx['to']) session = SessionBase.create_session() tx_dict = { 'hash': tx['hash'], - 'from': tx['from'], - 'to': tx['to'], + 'from': sender_address, + 'to': recipient_address, 'source_token': ZERO_ADDRESS_NORMAL, 'destination_token': ZERO_ADDRESS_NORMAL, 'from_value': 0, diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 1a1448a7..cf433c9e 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -12,7 +12,10 @@ from chainlib.eth.tx import ( ) from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token -from hexathon import strip_0x +from hexathon import ( + strip_0x, + add_0x, + ) from chainqueue.error import NotLocalTxError from eth_erc20 import ERC20 from chainqueue.sql.tx import cache_tx_dict @@ -38,6 +41,7 @@ from cic_eth.task import ( CriticalSQLAlchemyAndSignerTask, ) from cic_eth.eth.nonce import CustodialTaskNonceOracle +from cic_eth.encode import tx_normalize celery_app = celery.current_app logg = logging.getLogger() @@ -62,7 +66,8 @@ def balance(tokens, holder_address, chain_spec_dict): for t in tokens: address = t['address'] - token = ERC20Token(chain_spec, rpc, address) + logg.debug('address {} {}'.format(address, holder_address)) + token = ERC20Token(chain_spec, rpc, add_0x(address)) c = ERC20(chain_spec) o = c.balance_of(address, holder_address, sender_address=caller_address) r = rpc.do(o) @@ -371,13 +376,15 @@ def cache_transfer_data( tx = unpack(tx_signed_raw_bytes, chain_spec) tx_data = ERC20.parse_transfer_request(tx['data']) - recipient_address = tx_data[0] + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] + session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, - 'from': tx['from'], + 'from': sender_address, 'to': recipient_address, 'source_token': tx['to'], 'destination_token': tx['to'], @@ -448,13 +455,14 @@ def cache_approve_data( tx = unpack(tx_signed_raw_bytes, chain_spec) tx_data = ERC20.parse_approve_request(tx['data']) - recipient_address = tx_data[0] + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, - 'from': tx['from'], + 'from': sender_address, 'to': recipient_address, 'source_token': tx['to'], 'destination_token': tx['to'], diff --git a/apps/cic-eth/cic_eth/eth/meta.py b/apps/cic-eth/cic_eth/eth/meta.py index 21ab8295..881fad66 100644 --- a/apps/cic-eth/cic_eth/eth/meta.py +++ b/apps/cic-eth/cic_eth/eth/meta.py @@ -2,8 +2,9 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.status import Status as TxStatus from cic_eth_registry.erc20 import ERC20Token +from hexathon import add_0x -# local imports +# local impor:ts from cic_eth.ext.address import translate_address @@ -44,8 +45,8 @@ class ExtendedTx: destination = source if destination_value == None: destination_value = source_value - st = ERC20Token(self.chain_spec, self.rpc, source) - dt = ERC20Token(self.chain_spec, self.rpc, destination) + st = ERC20Token(self.chain_spec, self.rpc, add_0x(source)) + dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination)) self.source_token = source self.source_token_symbol = st.symbol self.source_token_name = st.name diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py index 3adefa2e..f9685ce4 100644 --- a/apps/cic-eth/cic_eth/ext/tx.py +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -32,6 +32,7 @@ from potaahto.symbols import snake_and_camel from cic_eth.queue.time import tx_times from cic_eth.task import BaseTask from cic_eth.db.models.base import SessionBase +from cic_eth.encode import tx_normalize celery_app = celery.current_app logg = logging.getLogger() @@ -134,7 +135,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_address = transfer_data[0] tx_token_value = transfer_data[1] - if address == tx_address: + if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address): status = StatusEnum.SENT try: o = receipt(tx['hash']) @@ -164,10 +165,10 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_r['date_created'] = times['queue'] else: tx_r['date_created'] = times['network'] - txs[tx['hash']] = tx_r + txs[strip_0x(tx['hash'])] = tx_r break - return txs + return txs # TODO: Surely it must be possible to optimize this diff --git a/apps/cic-eth/cic_eth/queue/query.py b/apps/cic-eth/cic_eth/queue/query.py index aa0b2504..3d941599 100644 --- a/apps/cic-eth/cic_eth/queue/query.py +++ b/apps/cic-eth/cic_eth/queue/query.py @@ -58,6 +58,7 @@ def get_tx_local(chain_spec, tx_hash, session=None): @celery_app.task(base=CriticalSQLAlchemyTask) def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None): + address = tx_normalize.wallet_address(address) chain_spec = ChainSpec.from_dict(chain_spec_dict) return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py index 54b8ad58..03cd774e 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py @@ -12,7 +12,8 @@ from hexathon import ( # local imports from .base import SyncFilter -logg = logging.getLogger(__name__) +#logg = logging.getLogger(__name__) +logg = logging.getLogger() account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430' diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 61fb0011..a2dbbc65 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,4 @@ celery==4.4.7 chainlib-eth>=0.0.9a14,<0.1.0 semver==2.13.0 +crypto-dev-signer>=0.4.15rc2,<0.5.0 diff --git a/apps/cic-eth/tests/filters/test_callback_filter.py b/apps/cic-eth/tests/filters/test_callback_filter.py index 695155cb..4b99de12 100644 --- a/apps/cic-eth/tests/filters/test_callback_filter.py +++ b/apps/cic-eth/tests/filters/test_callback_filter.py @@ -18,7 +18,10 @@ from eth_erc20 import ERC20 from sarafu_faucet import MinterFaucet from eth_accounts_index.registry import AccountRegistry from potaahto.symbols import snake_and_camel -from hexathon import add_0x +from hexathon import ( + add_0x, + strip_0x, + ) # local imports from cic_eth.runnable.daemons.filters.callback import CallbackFilter @@ -160,7 +163,7 @@ def test_faucet_gift_to_tx( assert transfer_data['token_address'] == foo_token -def test_callback_filter( +def test_callback_filter_filter( default_chain_spec, init_database, eth_rpc, @@ -213,6 +216,7 @@ def test_callback_filter( def call_back(self, transfer_type, result): self.results[transfer_type] = result + logg.debug('result {}'.format(result)) return self mock = CallbackMock() @@ -221,4 +225,4 @@ def test_callback_filter( fltr.filter(eth_rpc, mockblock, tx, init_database) assert mock.results.get('transfer') != None - assert mock.results['transfer']['destination_token'] == foo_token + assert mock.results['transfer']['destination_token'] == strip_0x(foo_token) diff --git a/apps/cic-eth/tests/filters/test_register_filter.py b/apps/cic-eth/tests/filters/test_register_filter.py index 34ec09c1..ce81b43a 100644 --- a/apps/cic-eth/tests/filters/test_register_filter.py +++ b/apps/cic-eth/tests/filters/test_register_filter.py @@ -17,6 +17,9 @@ from chainlib.eth.block import ( block_by_number, Block, ) +from chainlib.eth.address import ( + to_checksum_address, + ) from erc20_faucet import Faucet from hexathon import ( strip_0x, @@ -25,7 +28,6 @@ from hexathon import ( # local imports from cic_eth.runnable.daemons.filters.register import RegistrationFilter -from cic_eth.encode import tx_normalize from cic_eth.queue.query import get_account_tx_local logg = logging.getLogger() @@ -70,12 +72,13 @@ def test_register_filter( tx = Tx(tx_src, block=block, rcpt=rcpt) tx.apply_receipt(rcpt) - fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None) + fltr = RegistrationFilter(default_chain_spec, to_checksum_address(os.urandom(20).hex()), queue=None) t = fltr.filter(eth_rpc, block, tx, db_session=init_database) assert t == None - fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None) + fltr = RegistrationFilter(default_chain_spec, to_checksum_address(account_registry), queue=None) t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + logg.debug('t {}'.format(t)) t.get_leaf() assert t.successful() @@ -89,4 +92,4 @@ def test_register_filter( gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec) gift = Faucet.parse_give_to_request(gift_tx['data']) - assert gift[0] == agent_roles['ALICE'] + assert add_0x(gift[0]) == agent_roles['ALICE'] diff --git a/apps/cic-eth/tests/filters/test_transferauth_filter.py b/apps/cic-eth/tests/filters/test_transferauth_filter.py index 4d2266c1..ac84139a 100644 --- a/apps/cic-eth/tests/filters/test_transferauth_filter.py +++ b/apps/cic-eth/tests/filters/test_transferauth_filter.py @@ -19,6 +19,7 @@ from chainqueue.sql.query import get_account_tx # local imports from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter +from cic_eth.encode import tx_normalize def test_filter_transferauth( @@ -66,7 +67,8 @@ def test_filter_transferauth( t.get_leaf() assert t.successful() - approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + #approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + approve_txs = get_account_tx(default_chain_spec.asdict(), tx_normalize.wallet_address(agent_roles['ALICE']), as_sender=True, session=init_database) ks = list(approve_txs.keys()) assert len(ks) == 1 @@ -76,4 +78,4 @@ def test_filter_transferauth( c = ERC20(default_chain_spec) approve = c.parse_approve_request(approve_tx['data']) - assert approve[0] == agent_roles['BOB'] + assert approve[0] == strip_0x(agent_roles['BOB']) diff --git a/apps/cic-eth/tests/task/api/test_app.py b/apps/cic-eth/tests/task/api/test_app.py index 1e081584..acc6161d 100644 --- a/apps/cic-eth/tests/task/api/test_app.py +++ b/apps/cic-eth/tests/task/api/test_app.py @@ -10,6 +10,7 @@ from cic_eth_registry.erc20 import ERC20Token from chainlib.chain import ChainSpec from eth_accounts_index import AccountsIndex from chainlib.eth.tx import ( + receipt, transaction, ) from chainqueue.sql.state import ( @@ -29,6 +30,7 @@ def test_account_api( init_database, init_eth_rpc, account_registry, + cic_registry, custodial_roles, celery_session_worker, ): @@ -49,6 +51,7 @@ def test_account_api_register( eth_rpc, celery_session_worker, ): + api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None) t = api.create_account('') register_tx_hash = t.get_leaf() @@ -69,12 +72,18 @@ def test_account_api_register( r = t.get_leaf() assert t.successful() + o = receipt(register_tx_hash) + r = eth_rpc.do(o) + assert r['status'] == 1 + o = transaction(register_tx_hash) tx_src = eth_rpc.do(o) c = AccountsIndex(default_chain_spec) address = c.parse_add_request(tx_src['data']) + logg.debug('address {} '.format(address)) o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER']) + logg.debug('o {}'.format(o)) r = eth_rpc.do(o) assert c.parse_have(r) diff --git a/apps/cic-eth/tests/task/api/test_balance.py b/apps/cic-eth/tests/task/api/test_balance.py index 650c2b10..0a72a62c 100644 --- a/apps/cic-eth/tests/task/api/test_balance.py +++ b/apps/cic-eth/tests/task/api/test_balance.py @@ -3,18 +3,22 @@ import os import logging # external imports +import pytest from chainlib.eth.address import to_checksum_address +from hexathon import add_0x # local imports from cic_eth.api.api_task import Api logg = logging.getLogger() + def test_balance_simple_api( default_chain_spec, init_database, cic_registry, foo_token, + register_lookups, register_tokens, api, celery_session_worker, @@ -22,7 +26,7 @@ def test_balance_simple_api( chain_str = str(default_chain_spec) - a = to_checksum_address('0x' + os.urandom(20).hex()) + a = add_0x(to_checksum_address(os.urandom(20).hex())) t = api.balance(a, 'FOO', include_pending=False) r = t.get_leaf() assert t.successful() @@ -36,6 +40,7 @@ def test_balance_complex_api( init_database, cic_registry, foo_token, + register_lookups, register_tokens, api, celery_session_worker, @@ -43,7 +48,7 @@ def test_balance_complex_api( chain_str = str(default_chain_spec) - a = to_checksum_address('0x' + os.urandom(20).hex()) + a = add_0x(to_checksum_address(os.urandom(20).hex())) t = api.balance(a, 'FOO', include_pending=True) r = t.get_leaf() assert t.successful() diff --git a/apps/cic-eth/tests/task/api/test_list.py b/apps/cic-eth/tests/task/api/test_list.py index 22c07ae8..7f0bdf0f 100644 --- a/apps/cic-eth/tests/task/api/test_list.py +++ b/apps/cic-eth/tests/task/api/test_list.py @@ -6,6 +6,7 @@ import pytest from chainlib.eth.nonce import RPCNonceOracle from eth_erc20 import ERC20 from chainlib.eth.tx import receipt +from hexathon import strip_0x # local imports from cic_eth.api.api_task import Api @@ -23,7 +24,6 @@ from cic_eth.pytest.mock.filter import ( logg = logging.getLogger() -@pytest.mark.xfail() def test_list_tx( default_chain_spec, init_database, @@ -34,8 +34,10 @@ def test_list_tx( agent_roles, foo_token, register_tokens, + register_lookups, init_eth_tester, celery_session_worker, + init_celery_tasks, ): tx_hashes = [] @@ -63,13 +65,16 @@ def test_list_tx( o = receipt(tx_hash_hex) r = eth_rpc.do(o) assert r['status'] == 1 + a = r['block_number'] - block_filter.add(a.to_bytes(4, 'big')) + ab = a.to_bytes(4, 'big') + block_filter.add(ab) - a = r['block_number'] + r['transaction_index'] - tx_filter.add(a.to_bytes(4, 'big')) + bb = r['transaction_index'].to_bytes(4, 'big') + cb = ab + bb + tx_filter.add(cb) - tx_hashes.append(tx_hash_hex) + tx_hashes.append(strip_0x(tx_hash_hex)) # external tx two Nonce.next(agent_roles['ALICE'], 'foo', session=init_database) @@ -83,26 +88,29 @@ def test_list_tx( o = receipt(tx_hash_hex) r = eth_rpc.do(o) assert r['status'] == 1 + a = r['block_number'] - block_filter.add(a.to_bytes(4, 'big')) + ab = a.to_bytes(4, 'big') + block_filter.add(ab) - a = r['block_number'] + r['transaction_index'] - tx_filter.add(a.to_bytes(4, 'big')) + bb = r['transaction_index'].to_bytes(4, 'big') + cb = ab + bb + tx_filter.add(cb) - tx_hashes.append(tx_hash_hex) + tx_hashes.append(strip_0x(tx_hash_hex)) init_eth_tester.mine_blocks(28) # custodial tx 1 api = Api(str(default_chain_spec), queue=None) - t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') #, 'blinky') + t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') r = t.get_leaf() assert t.successful() tx_hashes.append(r) # custodial tx 2 api = Api(str(default_chain_spec), queue=None) - t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') #, 'blinky') + t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') r = t.get_leaf() assert t.successful() tx_hashes.append(r) @@ -117,7 +125,8 @@ def test_list_tx( assert len(r) == 3 logg.debug('rrrr {}'.format(r)) + logg.debug('testing against hashes {}'.format(tx_hashes)) for tx in r: logg.debug('have tx {}'.format(tx)) - tx_hashes.remove(tx['hash']) + tx_hashes.remove(strip_0x(tx['hash'])) assert len(tx_hashes) == 1