diff --git a/apps/cic-eth/.coveragerc b/apps/cic-eth/.coveragerc index 63f983d..47acdac 100644 --- a/apps/cic-eth/.coveragerc +++ b/apps/cic-eth/.coveragerc @@ -5,3 +5,5 @@ omit = cic_eth/db/migrations/* cic_eth/sync/head.py cic_eth/sync/mempool.py + cic_eth/queue/state.py + *redis*.py diff --git a/apps/cic-eth/.gitlab-ci.yml b/apps/cic-eth/.gitlab-ci.yml index ab8e077..80c2ded 100644 --- a/apps/cic-eth/.gitlab-ci.yml +++ b/apps/cic-eth/.gitlab-ci.yml @@ -24,9 +24,7 @@ test-mr-cic-eth: image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest script: - cd apps/$APP_NAME/ - - pytest tests/unit/ - - pytest tests/task/ - - pytest tests/filters/ + - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests needs: ["build-mr-cic-eth"] build-push-cic-eth: diff --git a/apps/cic-eth/cic_eth/admin/nonce.py b/apps/cic-eth/cic_eth/admin/nonce.py index b150cbe..a55c892 100644 --- a/apps/cic-eth/cic_eth/admin/nonce.py +++ b/apps/cic-eth/cic_eth/admin/nonce.py @@ -4,11 +4,18 @@ import logging # external imports import celery from chainlib.chain import ChainSpec -from chainlib.eth.tx import unpack -from chainqueue.query import get_tx -from chainqueue.state import set_cancel +from chainlib.connection import RPCConnection +from chainlib.eth.tx import ( + unpack, + TxFactory, + ) +from chainlib.eth.gas import OverrideGasOracle +from chainqueue.sql.query import get_tx +from chainqueue.sql.state import set_cancel from chainqueue.db.models.otx import Otx from chainqueue.db.models.tx import TxCache +from hexathon import strip_0x +from potaahto.symbols import snake_and_camel # local imports from cic_eth.db.models.base import SessionBase @@ -21,13 +28,14 @@ from cic_eth.admin.ctrl import ( ) from cic_eth.queue.tx import queue_create from cic_eth.eth.gas import create_check_gas_task +from cic_eth.task import BaseTask celery_app = celery.current_app logg = logging.getLogger() -@celery_app.task(bind=True) -def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): +@celery_app.task(bind=True, base=BaseTask) +def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): """Shift all transactions with nonces higher than the offset by the provided position delta. Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN. @@ -38,25 +46,29 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): :type tx_hash_orig_hex: str, 0x-hex :param delta: Amount """ + chain_spec = ChainSpec.from_dict(chainspec_dict) + rpc = RPCConnection.connect(chain_spec, 'default') + rpc_signer = RPCConnection.connect(chain_spec, 'signer') queue = None try: queue = self.request.delivery_info.get('routing_key') except AttributeError: pass - chain_spec = ChainSpec.from_chain_str(chain_str) - tx_brief = get_tx(tx_hash_orig_hex) - tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:])) + session = BaseTask.session_func() + tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session) + tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'])) tx = unpack(tx_raw, chain_spec) nonce = tx_brief['nonce'] address = tx['from'] - logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce)) + logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash'])) - lock_queue(None, chain_str, address) - lock_send(None, chain_str, address) + lock_queue(None, chain_spec.asdict(), address=address) + lock_send(None, chain_spec.asdict(), address=address) + + set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session) - session = SessionBase.create_session() q = session.query(Otx) q = q.join(TxCache) q = q.filter(TxCache.sender==address) @@ -69,49 +81,57 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): for otx in otxs: tx_raw = bytes.fromhex(strip_0x(otx.signed_tx)) tx_new = unpack(tx_raw, chain_spec) + tx_new = snake_and_camel(tx_new) tx_previous_hash_hex = tx_new['hash'] tx_previous_nonce = tx_new['nonce'] - del(tx_new['hash']) - del(tx_new['hash_unsigned']) + tx_new['gas_price'] += 1 + tx_new['gasPrice'] = tx_new['gas_price'] tx_new['nonce'] -= delta - (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str) + logg.debug('tx_new {}'.format(tx_new)) + + del(tx_new['hash']) + del(tx_new['hash_unsigned']) + del(tx_new['hashUnsigned']) + + gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx) + c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new) logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce'])) otx = Otx( - nonce=tx_new['nonce'], - address=tx_new['from'], - tx_hash=tx_hash_hex, - signed_tx=tx_signed_raw_hex, - ) + tx_new['nonce'], + tx_hash_hex, + tx_signed_raw_hex, + ) session.add(otx) - session.commit() # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces. - set_cancel(tx_previous_hash_hex, True) + set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session) - TxCache.clone(tx_previous_hash_hex, tx_hash_hex) + TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session) tx_hashes.append(tx_hash_hex) txs.append(tx_signed_raw_hex) + session.commit() session.close() - s = create_check_gas_and_send_task( + s = create_check_gas_task( txs, - chain_str, + chain_spec, tx_new['from'], - tx_new['gas'], - tx_hashes, - queue, + gas=tx_new['gas'], + tx_hashes_hex=tx_hashes, + queue=queue, ) s_unlock_send = celery.signature( 'cic_eth.admin.ctrl.unlock_send', [ - chain_str, + chain_spec.asdict(), tx_new['from'], ], queue=queue, @@ -119,7 +139,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): s_unlock_direct = celery.signature( 'cic_eth.admin.ctrl.unlock_queue', [ - chain_str, + chain_spec.asdict(), tx_new['from'], ], queue=queue, diff --git a/apps/cic-eth/cic_eth/api/api_admin.py b/apps/cic-eth/cic_eth/api/api_admin.py index 5862dd6..519262c 100644 --- a/apps/cic-eth/cic_eth/api/api_admin.py +++ b/apps/cic-eth/cic_eth/api/api_admin.py @@ -8,6 +8,7 @@ from chainlib.eth.constant import ( ZERO_ADDRESS, ) from cic_eth_registry import CICRegistry +from cic_eth_registry.erc20 import ERC20Token from cic_eth_registry.error import UnknownContractError from chainlib.eth.address import to_checksum_address from chainlib.eth.contract import code @@ -30,13 +31,14 @@ from chainqueue.db.enum import ( status_str, ) from chainqueue.error import TxStateChangeError +from chainqueue.query import get_tx +from eth_erc20 import ERC20 # local imports from cic_eth.db.models.base import SessionBase from cic_eth.db.models.role import AccountRole from cic_eth.db.models.nonce import Nonce from cic_eth.error import InitializationError -from cic_eth.queue.query import get_tx app = celery.current_app @@ -188,6 +190,7 @@ class AdminApi: s_manual = celery.signature( 'cic_eth.queue.state.set_manual', [ + chain_spec.asdict(), tx_hash_hex, ], queue=self.queue, @@ -206,8 +209,9 @@ class AdminApi: s.link(s_gas) return s_manual.apply_async() - - def check_nonce(self, address): + + + def check_nonce(self, chain_spec, address): s = celery.signature( 'cic_eth.queue.query.get_account_tx', [ @@ -228,13 +232,12 @@ class AdminApi: s_get_tx = celery.signature( 'cic_eth.queue.query.get_tx', [ - chain_spec.asdict(), + chain_spec.asdict(), k, ], queue=self.queue, ) tx = s_get_tx.apply_async().get() - #tx = get_tx(k) logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce)) nonce_otx = tx['nonce'] if not is_alive(tx['status']) and tx['status'] & local_fail > 0: @@ -242,7 +245,9 @@ class AdminApi: blocking_tx = k blocking_nonce = nonce_otx elif nonce_otx - last_nonce > 1: - logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from'])) + logg.debug('tx {}'.format(tx)) + tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec) + logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from'])) blocking_tx = k blocking_nonce = nonce_otx break @@ -256,12 +261,13 @@ class AdminApi: 'blocking': blocking_nonce, }, 'tx': { - 'blocking': blocking_tx, - } + 'blocking': add_0x(blocking_tx), } + } - def fix_nonce(self, address, nonce, chain_spec): + # TODO: is risky since it does not validate that there is actually a nonce problem? + def fix_nonce(self, chain_spec, address, nonce): s = celery.signature( 'cic_eth.queue.query.get_account_tx', [ @@ -275,15 +281,17 @@ class AdminApi: txs = s.apply_async().get() tx_hash_hex = None + session = SessionBase.create_session() for k in txs.keys(): - tx_dict = get_tx(k) + tx_dict = get_tx(chain_spec, k, session=session) if tx_dict['nonce'] == nonce: tx_hash_hex = k + session.close() s_nonce = celery.signature( 'cic_eth.admin.nonce.shift_nonce', [ - self.rpc.chain_spec.asdict(), + chain_spec.asdict(), tx_hash_hex, ], queue=self.queue @@ -388,12 +396,13 @@ class AdminApi: t = s.apply_async() tx = t.get() - + source_token = None if tx['source_token'] != ZERO_ADDRESS: + source_token_declaration = None if registry != None: try: - source_token = registry.by_address(tx['source_token']) + source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address) except UnknownContractError: logg.warning('unknown source token contract {} (direct)'.format(tx['source_token'])) else: @@ -406,16 +415,21 @@ class AdminApi: queue=self.queue ) t = s.apply_async() - source_token = t.get() - if source_token == None: - logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token'])) + source_token_declaration = t.get() + + if source_token_declaration != None: + logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token'])) + source_token = ERC20Token(chain_spec, self.rpc, tx['source_token']) + logg.debug('source token set tup {}'.format(source_token)) + destination_token = None if tx['destination_token'] != ZERO_ADDRESS: + destination_token_declaration = None if registry != None: try: - destination_token = registry.by_address(tx['destination_token']) + destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address) except UnknownContractError: logg.warning('unknown destination token contract {}'.format(tx['destination_token'])) else: @@ -428,10 +442,10 @@ class AdminApi: queue=self.queue ) t = s.apply_async() - destination_token = t.get() - if destination_token == None: - logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token'])) - + destination_token_declaration = t.get() + if destination_token_declaration != None: + logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token'])) + destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token']) tx['sender_description'] = 'Custodial account' tx['recipient_description'] = 'Custodial account' @@ -543,13 +557,19 @@ class AdminApi: if role != None: tx['recipient_description'] = role + erc20_c = ERC20(chain_spec) if source_token != None: - tx['source_token_symbol'] = source_token.symbol() - tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call() + tx['source_token_symbol'] = source_token.symbol + o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address) + r = self.rpc.do(o) + tx['sender_token_balance'] = erc20_c.parse_balance_of(r) if destination_token != None: - tx['destination_token_symbol'] = destination_token.symbol() - tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call() + tx['destination_token_symbol'] = destination_token.symbol + o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address) + r = self.rpc.do(o) + tx['recipient_token_balance'] = erc20_c.parse_balance_of(r) + #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call() # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which tx['network_status'] = 'Not in node' diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 93a4d1b..aea0000 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -74,134 +74,134 @@ class Api: return s_token.apply_async() - def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol): - """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed. - - :param from_address: Ethereum address of sender - :type from_address: str, 0x-hex - :param to_address: Ethereum address of receipient - :type to_address: str, 0x-hex - :param target_return: Estimated return from conversion - :type target_return: int - :param minimum_return: The least value of destination token return to allow - :type minimum_return: int - :param from_token_symbol: ERC20 token symbol of token being converted - :type from_token_symbol: str - :param to_token_symbol: ERC20 token symbol of token to receive - :type to_token_symbol: str - :returns: uuid of root task - :rtype: celery.Task - """ - raise NotImplementedError('out of service until new DEX migration is done') - s_check = celery.signature( - 'cic_eth.admin.ctrl.check_lock', - [ - [from_token_symbol, to_token_symbol], - self.chain_spec.asdict(), - LockEnum.QUEUE, - from_address, - ], - queue=self.queue, - ) - s_nonce = celery.signature( - 'cic_eth.eth.nonce.reserve_nonce', - [ - self.chain_spec.asdict(), - ], - queue=self.queue, - ) - s_tokens = celery.signature( - 'cic_eth.eth.erc20.resolve_tokens_by_symbol', - [ - self.chain_str, - ], - queue=self.queue, - ) - s_convert = celery.signature( - 'cic_eth.eth.bancor.convert_with_default_reserve', - [ - from_address, - target_return, - minimum_return, - to_address, - self.chain_spec.asdict(), - ], - queue=self.queue, - ) - s_nonce.link(s_tokens) - s_check.link(s_nonce) - if self.callback_param != None: - s_convert.link(self.callback_success) - s_tokens.link(s_convert).on_error(self.callback_error) - else: - s_tokens.link(s_convert) - - t = s_check.apply_async(queue=self.queue) - return t - - - def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol): - """Executes a chain of celery tasks that performs conversion between two ERC20 tokens. - - :param from_address: Ethereum address of sender - :type from_address: str, 0x-hex - :param target_return: Estimated return from conversion - :type target_return: int - :param minimum_return: The least value of destination token return to allow - :type minimum_return: int - :param from_token_symbol: ERC20 token symbol of token being converted - :type from_token_symbol: str - :param to_token_symbol: ERC20 token symbol of token to receive - :type to_token_symbol: str - :returns: uuid of root task - :rtype: celery.Task - """ - raise NotImplementedError('out of service until new DEX migration is done') - s_check = celery.signature( - 'cic_eth.admin.ctrl.check_lock', - [ - [from_token_symbol, to_token_symbol], - self.chain_spec.asdict(), - LockEnum.QUEUE, - from_address, - ], - queue=self.queue, - ) - s_nonce = celery.signature( - 'cic_eth.eth.nonce.reserve_nonce', - [ - self.chain_spec.asdict(), - ], - queue=self.queue, - ) - s_tokens = celery.signature( - 'cic_eth.eth.erc20.resolve_tokens_by_symbol', - [ - self.chain_spec.asdict(), - ], - queue=self.queue, - ) - s_convert = celery.signature( - 'cic_eth.eth.bancor.convert_with_default_reserve', - [ - from_address, - target_return, - minimum_return, - from_address, - self.chain_spec.asdict(), - ], - queue=self.queue, - ) - s_nonce.link(s_tokens) - s_check.link(s_nonce) - if self.callback_param != None: - s_convert.link(self.callback_success) - s_tokens.link(s_convert).on_error(self.callback_error) - else: - s_tokens.link(s_convert) - - t = s_check.apply_async(queue=self.queue) - return t +# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol): +# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed. +# +# :param from_address: Ethereum address of sender +# :type from_address: str, 0x-hex +# :param to_address: Ethereum address of receipient +# :type to_address: str, 0x-hex +# :param target_return: Estimated return from conversion +# :type target_return: int +# :param minimum_return: The least value of destination token return to allow +# :type minimum_return: int +# :param from_token_symbol: ERC20 token symbol of token being converted +# :type from_token_symbol: str +# :param to_token_symbol: ERC20 token symbol of token to receive +# :type to_token_symbol: str +# :returns: uuid of root task +# :rtype: celery.Task +# """ +# raise NotImplementedError('out of service until new DEX migration is done') +# s_check = celery.signature( +# 'cic_eth.admin.ctrl.check_lock', +# [ +# [from_token_symbol, to_token_symbol], +# self.chain_spec.asdict(), +# LockEnum.QUEUE, +# from_address, +# ], +# queue=self.queue, +# ) +# s_nonce = celery.signature( +# 'cic_eth.eth.nonce.reserve_nonce', +# [ +# self.chain_spec.asdict(), +# ], +# queue=self.queue, +# ) +# s_tokens = celery.signature( +# 'cic_eth.eth.erc20.resolve_tokens_by_symbol', +# [ +# self.chain_str, +# ], +# queue=self.queue, +# ) +# s_convert = celery.signature( +# 'cic_eth.eth.bancor.convert_with_default_reserve', +# [ +# from_address, +# target_return, +# minimum_return, +# to_address, +# self.chain_spec.asdict(), +# ], +# queue=self.queue, +# ) +# s_nonce.link(s_tokens) +# s_check.link(s_nonce) +# if self.callback_param != None: +# s_convert.link(self.callback_success) +# s_tokens.link(s_convert).on_error(self.callback_error) +# else: +# s_tokens.link(s_convert) +# +# t = s_check.apply_async(queue=self.queue) +# return t +# +# +# def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol): +# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens. +# +# :param from_address: Ethereum address of sender +# :type from_address: str, 0x-hex +# :param target_return: Estimated return from conversion +# :type target_return: int +# :param minimum_return: The least value of destination token return to allow +# :type minimum_return: int +# :param from_token_symbol: ERC20 token symbol of token being converted +# :type from_token_symbol: str +# :param to_token_symbol: ERC20 token symbol of token to receive +# :type to_token_symbol: str +# :returns: uuid of root task +# :rtype: celery.Task +# """ +# raise NotImplementedError('out of service until new DEX migration is done') +# s_check = celery.signature( +# 'cic_eth.admin.ctrl.check_lock', +# [ +# [from_token_symbol, to_token_symbol], +# self.chain_spec.asdict(), +# LockEnum.QUEUE, +# from_address, +# ], +# queue=self.queue, +# ) +# s_nonce = celery.signature( +# 'cic_eth.eth.nonce.reserve_nonce', +# [ +# self.chain_spec.asdict(), +# ], +# queue=self.queue, +# ) +# s_tokens = celery.signature( +# 'cic_eth.eth.erc20.resolve_tokens_by_symbol', +# [ +# self.chain_spec.asdict(), +# ], +# queue=self.queue, +# ) +# s_convert = celery.signature( +# 'cic_eth.eth.bancor.convert_with_default_reserve', +# [ +# from_address, +# target_return, +# minimum_return, +# from_address, +# self.chain_spec.asdict(), +# ], +# queue=self.queue, +# ) +# s_nonce.link(s_tokens) +# s_check.link(s_nonce) +# if self.callback_param != None: +# s_convert.link(self.callback_success) +# s_tokens.link(s_convert).on_error(self.callback_error) +# else: +# s_tokens.link(s_convert) +# +# t = s_check.apply_async(queue=self.queue) +# return t def transfer(self, from_address, to_address, value, token_symbol): diff --git a/apps/cic-eth/cic_eth/db/util.py b/apps/cic-eth/cic_eth/db/util.py deleted file mode 100644 index c762062..0000000 --- a/apps/cic-eth/cic_eth/db/util.py +++ /dev/null @@ -1,8 +0,0 @@ -import math - -def num_serialize(n): - if n == 0: - return b'\x00' - binlog = math.log2(n) - bytelength = int(binlog / 8 + 1) - return n.to_bytes(bytelength, 'big') diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 258054c..49e89c7 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -57,10 +57,12 @@ celery_app = celery.current_app logg = logging.getLogger() +MAXIMUM_FEE_UNITS = 8000000 + class MaxGasOracle: def gas(code=None): - return 8000000 + return MAXIMUM_FEE_UNITS def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): @@ -150,7 +152,7 @@ def cache_gas_data( @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task) -def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None): +def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS): """Check the gas level of the sender address of a transaction. If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser. @@ -170,24 +172,30 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir :return: Signed raw transaction data list :rtype: param txs, unchanged """ - if len(txs) == 0: - for i in range(len(tx_hashes)): - o = get_tx(tx_hashes[i]) - txs.append(o['signed_tx']) - if address == None: - address = o['address'] + chain_spec = ChainSpec.from_dict(chain_spec_dict) + logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes)) + + addresspass = None + if len(txs) == 0: + addresspass = [] + for i in range(len(tx_hashes)): + o = get_tx(chain_spec_dict, tx_hashes[i]) + txs.append(o['signed_tx']) + logg.debug('sender {}'.format(o)) + tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec) + if address == None: + address = tx['from'] + elif address != tx['from']: + raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from'])) + addresspass.append(address) - #if not web3.Web3.isChecksumAddress(address): if not is_checksum_address(address): raise ValueError('invalid address {}'.format(address)) - chain_spec = ChainSpec.from_dict(chain_spec_dict) - queue = self.request.delivery_info.get('routing_key') conn = RPCConnection.connect(chain_spec) - # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx gas_balance = 0 try: o = balance(address) @@ -198,6 +206,9 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir conn.disconnect() raise EthError('gas_balance call for {}: {}'.format(address, e)) + if gas_required == None: + gas_required = MAXIMUM_FEE_UNITS + logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required)) session = SessionBase.create_session() gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) @@ -268,7 +279,8 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir queue=queue, ) ready_tasks.append(s) - celery.group(ready_tasks)() + t = celery.group(ready_tasks)() + logg.debug('group {}'.format(t)) return txs diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 4883e87..6602fe6 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -21,6 +21,7 @@ from chainqueue.db.models.tx import Otx from chainqueue.db.models.tx import TxCache from chainqueue.db.enum import StatusBits from chainqueue.error import NotLocalTxError +from potaahto.symbols import snake_and_camel # local imports from cic_eth.db import SessionBase @@ -58,6 +59,9 @@ def hashes_to_txs(self, tx_hashes): if len(tx_hashes) == 0: raise ValueError('no transaction to send') + for i in range(len(tx_hashes)): + tx_hashes[i] = strip_0x(tx_hashes[i]) + queue = self.request.delivery_info['routing_key'] session = SessionBase.create_session() @@ -148,7 +152,7 @@ def send(self, txs, chain_spec_dict): @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task) def sync_tx(self, tx_hash_hex, chain_spec_dict): - """Force update of network status of a simgle transaction + """Force update of network status of a single transaction :param tx_hash_hex: Transaction hash :type tx_hash_hex: str, 0x-hex @@ -173,12 +177,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): # TODO: apply receipt in tx object to validate and normalize input if rcpt != None: + rcpt = snake_and_camel(rcpt) success = rcpt['status'] == 1 - logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success)) + logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success)) s = celery.signature( 'cic_eth.queue.state.set_final', [ + chain_spec_dict, tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], @@ -186,12 +192,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict): ], queue=queue, ) + # TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking else: logg.debug('sync tx {} mempool'.format(tx_hash_hex)) s = celery.signature( 'cic_eth.queue.state.set_sent', [ + chain_spec_dict, tx_hash_hex, ], queue=queue, diff --git a/apps/cic-eth/cic_eth/queue/query.py b/apps/cic-eth/cic_eth/queue/query.py index 1572289..44ed754 100644 --- a/apps/cic-eth/cic_eth/queue/query.py +++ b/apps/cic-eth/cic_eth/queue/query.py @@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash): def get_tx(chain_spec_dict, tx_hash): chain_spec = ChainSpec.from_dict(chain_spec_dict) session = SessionBase.create_session() - r = chainqueue.query.get_tx(chain_spec, tx_hash) + r = chainqueue.query.get_tx(chain_spec, tx_hash, session=session) session.close() return r diff --git a/apps/cic-eth/cic_eth/registry.py b/apps/cic-eth/cic_eth/registry.py index 2b5e2b9..b46b200 100644 --- a/apps/cic-eth/cic_eth/registry.py +++ b/apps/cic-eth/cic_eth/registry.py @@ -5,29 +5,30 @@ import logging from cic_eth_registry import CICRegistry from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup from cic_eth_registry.lookup.tokenindex import TokenIndexLookup +from chainlib.eth.constant import ZERO_ADDRESS logg = logging.getLogger() -def connect_token_registry(rpc, chain_spec): +def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS): registry = CICRegistry(chain_spec, rpc) - token_registry_address = registry.by_name('TokenRegistry') + token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address) logg.debug('using token registry address {}'.format(token_registry_address)) lookup = TokenIndexLookup(chain_spec, token_registry_address) CICRegistry.add_lookup(lookup) -def connect_declarator(rpc, chain_spec, trusted_addresses): +def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS): registry = CICRegistry(chain_spec, rpc) - declarator_address = registry.by_name('AddressDeclarator') + declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address) logg.debug('using declarator address {}'.format(declarator_address)) lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) CICRegistry.add_lookup(lookup) -def connect(rpc, chain_spec, registry_address): +def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS): CICRegistry.address = registry_address registry = CICRegistry(chain_spec, rpc) - registry_address = registry.by_name('ContractRegistry') + registry_address = registry.by_name('ContractRegistry', sender_address=sender_address) return registry diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py index b021909..4b17f86 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py @@ -17,7 +17,8 @@ from cic_eth.db.models.base import SessionBase from cic_eth.eth.gas import create_check_gas_task from .base import SyncFilter -logg = logging.getLogger().getChild(__name__) +#logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger() class GasFilter(SyncFilter): @@ -27,11 +28,11 @@ class GasFilter(SyncFilter): self.chain_spec = chain_spec - def filter(self, conn, block, tx, session): + def filter(self, conn, block, tx, db_session): if tx.value > 0: tx_hash_hex = add_0x(tx.hash) logg.debug('gas refill tx {}'.format(tx_hash_hex)) - session = SessionBase.bind_session(session) + session = SessionBase.bind_session(db_session) q = session.query(TxCache.recipient) q = q.join(Otx) q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex)) @@ -56,7 +57,7 @@ class GasFilter(SyncFilter): tx_hashes_hex=list(txs.keys()), queue=self.queue, ) - s.apply_async() + return s.apply_async() def __str__(self): diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py index dc1bd77..2395322 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py @@ -50,7 +50,8 @@ class RegistrationFilter(SyncFilter): queue=self.queue, ) s_nonce.link(s_gift) - s_nonce.apply_async() + t = s_nonce.apply_async() + return t def __str__(self): diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py index d6646b6..3ba7779 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py @@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter): self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address) - def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None): + def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None): if tx.payload == None: logg.debug('no payload') @@ -45,16 +45,17 @@ class TransferAuthFilter(SyncFilter): return False recipient = tx.inputs[0] - if recipient != self.transfer_request_contract.address(): + #if recipient != self.transfer_request_contract.address(): + if recipient != self.transfer_request_contract: logg.debug('not our transfer auth contract address {}'.format(recipient)) return False r = TransferAuthorization.parse_create_request_request(tx.payload) - - sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) - recipient = abi_decode_single(ABIContractType.ADDRESS, r[1]) - token = abi_decode_single(ABIContractType.ADDRESS, r[2]) - value = abi_decode_single(ABIContractType.UINT256, r[3]) + + sender = r[0] + recipient = r[1] + token = r[2] + value = r[3] token_data = { 'address': token, @@ -64,6 +65,7 @@ class TransferAuthFilter(SyncFilter): 'cic_eth.eth.nonce.reserve_nonce', [ [token_data], + self.chain_spec.asdict(), sender, ], queue=self.queue, @@ -80,7 +82,7 @@ class TransferAuthFilter(SyncFilter): ) s_nonce.link(s_approve) t = s_nonce.apply_async() - return True + return t def __str__(self): diff --git a/apps/cic-eth/cic_eth/stat.py b/apps/cic-eth/cic_eth/stat.py index d3a07fd..e674e12 100644 --- a/apps/cic-eth/cic_eth/stat.py +++ b/apps/cic-eth/cic_eth/stat.py @@ -20,7 +20,11 @@ def init_chain_stat(rpc, block_start=0): if block_start == 0: o = block_latest() r = rpc.do(o) - block_start = int(r, 16) + try: + block_start = int(r, 16) + except TypeError: + block_start = int(r) + logg.debug('blockstart {}'.format(block_start)) for i in range(BLOCK_SAMPLES): o = block_by_number(block_start-10+i) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 2275971..fa542cd 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -20,7 +20,8 @@ import liveness.linux from cic_eth.error import SeppukuError from cic_eth.db.models.base import SessionBase -logg = logging.getLogger().getChild(__name__) +#logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger() celery_app = celery.current_app @@ -118,12 +119,13 @@ def registry(): return CICRegistry.address -@celery_app.task() -def registry_address_lookup(chain_spec_dict, address, connection_tag='default'): +@celery_app.task(bind=True, base=BaseTask) +def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'): chain_spec = ChainSpec.from_dict(chain_spec_dict) conn = RPCConnection.connect(chain_spec, tag=connection_tag) registry = CICRegistry(chain_spec, conn) - return registry.by_address(address) + r = registry.by_address(address, sender_address=self.call_address) + return r @celery_app.task(throws=(UnknownContractError,)) @@ -131,7 +133,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'): chain_spec = ChainSpec.from_dict(chain_spec_dict) conn = RPCConnection.connect(chain_spec, tag=connection_tag) registry = CICRegistry(chain_spec, conn) - return registry.by_name(name) + return registry.by_name(name, sender_address=self.call_address) @celery_app.task() diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index fddcd15..a7ed3ae 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,23 +1,23 @@ -cic-base~=0.1.2b11 +cic-base~=0.1.2b14 celery==4.4.7 crypto-dev-signer~=0.4.14b3 confini~=0.3.6rc3 -cic-eth-registry~=0.5.5a4 +cic-eth-registry~=0.5.5a7 redis==3.5.3 alembic==1.4.2 websockets==8.1 requests~=2.24.0 eth_accounts_index~=0.0.11a12 -erc20-transfer-authorization~=0.3.1a6 +erc20-transfer-authorization~=0.3.1a7 uWSGI==2.0.19.1 semver==2.13.0 websocket-client==0.57.0 moolb~=0.1.1b2 eth-address-index~=0.1.1a11 -chainlib~=0.0.3a2 +chainlib~=0.0.3rc2 hexathon~=0.0.1a7 chainsyncer[sql]~=0.0.2a4 -chainqueue~=0.0.2a2 +chainqueue~=0.0.2b1 sarafu-faucet==0.0.3a3 erc20-faucet==0.2.1a4 coincurve==15.0.0 diff --git a/apps/cic-eth/tests/check/test_check_db.py b/apps/cic-eth/tests/check/test_check_db.py new file mode 100644 index 0000000..941f889 --- /dev/null +++ b/apps/cic-eth/tests/check/test_check_db.py @@ -0,0 +1,8 @@ +# local imports +from cic_eth.check.db import health + +def test_check_health( + init_database, + ): + + assert health() diff --git a/apps/cic-eth/tests/check/test_check_gas.py b/apps/cic-eth/tests/check/test_check_gas.py new file mode 100644 index 0000000..ea0ffb8 --- /dev/null +++ b/apps/cic-eth/tests/check/test_check_gas.py @@ -0,0 +1,20 @@ +# local imports +from cic_eth.check.gas import health +from cic_eth.db.models.role import AccountRole + +def test_check_gas( + config, + init_database, + default_chain_spec, + eth_rpc, + custodial_roles, + whoever, + ): + + config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True) + assert health(config=config) + + AccountRole.set('GAS_GIFTER', whoever, session=init_database) + init_database.commit() + assert not health(config=config) diff --git a/apps/cic-eth/tests/check/test_check_redis.py b/apps/cic-eth/tests/check/test_check_redis.py new file mode 100644 index 0000000..02c3ba7 --- /dev/null +++ b/apps/cic-eth/tests/check/test_check_redis.py @@ -0,0 +1,16 @@ +# external imports +import pytest + +# local imports +from cic_eth.check.redis import health + + +def test_check_redis( + config, + have_redis, + ): + + if have_redis != None: + pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis)) + + assert health(unit='test', config=config) diff --git a/apps/cic-eth/tests/check/test_check_signer.py b/apps/cic-eth/tests/check/test_check_signer.py new file mode 100644 index 0000000..5572e19 --- /dev/null +++ b/apps/cic-eth/tests/check/test_check_signer.py @@ -0,0 +1,13 @@ +# local imports +from cic_eth.check.signer import health + + +def test_check_signer( + default_chain_spec, + config, + eth_signer, + eth_rpc, + ): + + config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + assert health(config=config) diff --git a/apps/cic-eth/tests/conftest.py b/apps/cic-eth/tests/conftest.py index bc7a82b..169a6c7 100644 --- a/apps/cic-eth/tests/conftest.py +++ b/apps/cic-eth/tests/conftest.py @@ -2,9 +2,11 @@ import os import sys import logging +import uuid # external imports from eth_erc20 import ERC20 +import redis # local imports from cic_eth.api import Api @@ -19,6 +21,7 @@ from tests.fixtures_config import * from tests.fixtures_database import * from tests.fixtures_celery import * from tests.fixtures_role import * +from tests.fixtures_contract import * from chainlib.eth.pytest import * from eth_contract_registry.pytest import * from cic_eth_registry.pytest.fixtures_contracts import * @@ -55,3 +58,28 @@ def default_token( ): BaseTask.default_token_symbol = foo_token_symbol BaseTask.default_token_address = foo_token + + + +@pytest.fixture(scope='session') +def have_redis( + config, + ): + + r = redis.Redis( + host = config.get('REDIS_HOST'), + port = config.get('REDIS_PORT'), + db = config.get('REDIS_DB'), + ) + k = str(uuid.uuid4()) + try: + r.set(k, 'foo') + r.delete(k) + except redis.exceptions.ConnectionError as e: + return e + except TypeError as e: + return e + + return None + + diff --git a/apps/cic-eth/tests/filters/test_filter_bogus.py b/apps/cic-eth/tests/filters/test_filter_bogus.py new file mode 100644 index 0000000..30edbc7 --- /dev/null +++ b/apps/cic-eth/tests/filters/test_filter_bogus.py @@ -0,0 +1,38 @@ +# local imports +from cic_eth.runnable.daemons.filters.gas import GasFilter +from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter +from cic_eth.runnable.daemons.filters.callback import CallbackFilter +from cic_eth.runnable.daemons.filters.straggler import StragglerFilter +from cic_eth.runnable.daemons.filters.tx import TxFilter +from cic_eth.runnable.daemons.filters.register import RegistrationFilter + + +# Hit tx mismatch paths on all filters +def test_filter_bogus( + init_database, + bogus_tx_block, + default_chain_spec, + eth_rpc, + eth_signer, + transfer_auth, + cic_registry, + contract_roles, + register_lookups, + ): + + fltrs = [ + TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']), + GasFilter(default_chain_spec, queue=None), + TxFilter(default_chain_spec, None), + CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']), + StragglerFilter(default_chain_spec, None), + RegistrationFilter(default_chain_spec, queue=None), + ] + + for fltr in fltrs: + r = None + try: + r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database) + except: + pass + assert not r diff --git a/apps/cic-eth/tests/filters/test_gas_filter.py b/apps/cic-eth/tests/filters/test_gas_filter.py new file mode 100644 index 0000000..8f91d2f --- /dev/null +++ b/apps/cic-eth/tests/filters/test_gas_filter.py @@ -0,0 +1,101 @@ +# external imports +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import OverrideNonceOracle +from chainqueue.tx import create as queue_create +from chainlib.eth.tx import ( + TxFormat, + unpack, + Tx, + ) +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainlib.eth.block import ( + block_latest, + block_by_number, + Block, + ) +from chainqueue.state import ( + set_waitforgas, + ) +from hexathon import strip_0x +from chainqueue.db.models.otx import Otx +from chainqueue.db.enum import StatusBits + +# local imports +from cic_eth.runnable.daemons.filters.gas import GasFilter +from cic_eth.eth.gas import cache_gas_data + + +def test_filter_gas( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database) + init_database.commit() + tx_hash_hex_wait = tx_hash_hex + otx = Otx.load(tx_hash_hex_wait, session=init_database) + assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES + + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 43, + agent_roles['BOB'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + fltr = GasFilter(default_chain_spec, queue=None) + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + + t.get_leaf() + assert t.successful() + init_database.commit() + + otx = Otx.load(tx_hash_hex_wait, session=init_database) + assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED diff --git a/apps/cic-eth/tests/filters/test_register_filter.py b/apps/cic-eth/tests/filters/test_register_filter.py new file mode 100644 index 0000000..9842bc0 --- /dev/null +++ b/apps/cic-eth/tests/filters/test_register_filter.py @@ -0,0 +1,78 @@ +# external imports +from eth_accounts_index.registry import AccountRegistry +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import OverrideGasOracle +from chainlib.eth.tx import( + receipt, + unpack, + Tx, + ) +from chainlib.eth.block import ( + block_latest, + block_by_number, + Block, + ) +from erc20_faucet import Faucet +from hexathon import strip_0x +from chainqueue.query import get_account_tx + +# local imports +from cic_eth.runnable.daemons.filters.register import RegistrationFilter + + +def test_register_filter( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + account_registry, + faucet, + register_lookups, + contract_roles, + agent_roles, + cic_registry, + init_celery_tasks, + celery_session_worker, + caplog, + ): + + nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc) + + c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE']) + r = eth_rpc.do(o) + tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0])) + + o = receipt(tx_hash_hex) + rcpt = eth_rpc.do(o) + assert rcpt['status'] == 1 + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block, rcpt=rcpt) + tx.apply_receipt(rcpt) + + fltr = RegistrationFilter(default_chain_spec, queue=None) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + + t.get_leaf() + assert t.successful() + + gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + ks = list(gift_txs.keys()) + assert len(ks) == 1 + + tx_raw_signed_hex = strip_0x(gift_txs[ks[0]]) + tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex) + gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec) + + gift = Faucet.parse_give_to_request(gift_tx['data']) + assert gift[0] == agent_roles['ALICE'] diff --git a/apps/cic-eth/tests/filters/test_transferauth_filter.py b/apps/cic-eth/tests/filters/test_transferauth_filter.py new file mode 100644 index 0000000..25822b8 --- /dev/null +++ b/apps/cic-eth/tests/filters/test_transferauth_filter.py @@ -0,0 +1,79 @@ +# external imports +from erc20_transfer_authorization import TransferAuthorization +from eth_erc20 import ERC20 +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import OverrideGasOracle +from chainlib.eth.tx import ( + receipt, + unpack, + Tx, + ) +from chainlib.eth.block import ( + block_latest, + block_by_number, + Block, + ) +from hexathon import strip_0x +from chainqueue.query import get_account_tx + +# local imports +from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter + + +def test_filter_transferauth( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + transfer_auth, + foo_token, + celery_session_worker, + register_lookups, + init_custodial, + cic_registry, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) + gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc) + c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024) + + r = rpc.do(o) + tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0])) + + o = receipt(tx_hash_hex) + r = rpc.do(o) + assert r['status'] == 1 + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + #tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + + fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + + t.get_leaf() + assert t.successful() + + approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + ks = list(approve_txs.keys()) + assert len(ks) == 1 + + tx_raw_signed_hex = strip_0x(approve_txs[ks[0]]) + tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex) + approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec) + + c = ERC20(default_chain_spec) + approve = c.parse_approve_request(approve_tx['data']) + assert approve[0] == agent_roles['BOB'] diff --git a/apps/cic-eth/tests/filters/test_tx_filter.py b/apps/cic-eth/tests/filters/test_tx_filter.py index bd706e5..06d2c94 100644 --- a/apps/cic-eth/tests/filters/test_tx_filter.py +++ b/apps/cic-eth/tests/filters/test_tx_filter.py @@ -23,7 +23,6 @@ from chainqueue.state import ( set_ready, set_sent, ) - from hexathon import strip_0x # local imports @@ -31,7 +30,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter from cic_eth.eth.gas import cache_gas_data -def test_tx( +def test_filter_tx( default_chain_spec, init_database, eth_rpc, diff --git a/apps/cic-eth/tests/fixtures_celery.py b/apps/cic-eth/tests/fixtures_celery.py index 26f640d..188f973 100644 --- a/apps/cic-eth/tests/fixtures_celery.py +++ b/apps/cic-eth/tests/fixtures_celery.py @@ -22,7 +22,6 @@ def init_celery_tasks( @pytest.fixture(scope='session') def celery_includes(): return [ -# 'cic_eth.eth.bancor', 'cic_eth.eth.erc20', 'cic_eth.eth.tx', 'cic_eth.ext.tx', @@ -47,8 +46,8 @@ def celery_config(): bq = tempfile.mkdtemp() bp = tempfile.mkdtemp() rq = tempfile.mkdtemp() - logg.debug('celery broker queue {} processed {}'.format(bq, bp)) - logg.debug('celery backend store {}'.format(rq)) + logg.debug('celery broker session queue {} processed {}'.format(bq, bp)) + logg.debug('celery backend session store {}'.format(rq)) yield { 'broker_url': 'filesystem://', 'broker_transport_options': { @@ -58,12 +57,11 @@ def celery_config(): }, 'result_backend': 'file://{}'.format(rq), } - logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq)) + logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq)) shutil.rmtree(bq) shutil.rmtree(bp) shutil.rmtree(rq) - @pytest.fixture(scope='session') def celery_worker_parameters(): return { diff --git a/apps/cic-eth/tests/fixtures_contract.py b/apps/cic-eth/tests/fixtures_contract.py new file mode 100644 index 0000000..b09b551 --- /dev/null +++ b/apps/cic-eth/tests/fixtures_contract.py @@ -0,0 +1,77 @@ +# standard imports +import os + +# external imports +import pytest +from chainlib.eth.contract import ( + ABIContractEncoder, + ABIContractType, + ) +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import OverrideGasOracle +from chainlib.eth.block import ( + block_latest, + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + TxFactory, + TxFormat, + unpack, + Tx, + ) +from hexathon import strip_0x + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) + + +@pytest.fixture(scope='function') +def bogus_tx_block( + default_chain_spec, + eth_rpc, + eth_signer, + contract_roles, + ): + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc) + + f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r') + bytecode = f.read() + f.close() + + c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) + tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True) + tx = c.set_code(tx, bytecode) + (tx_hash_hex, o) = c.build(tx) + + r = eth_rpc.do(o) + + o = receipt(tx_hash_hex) + r = eth_rpc.do(o) + + contract_address = r['contract_address'] + + enc = ABIContractEncoder() + enc.method('poke') + data = enc.get() + tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True) + tx = c.set_code(tx, data) + (tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC) + r = eth_rpc.do(o) + tx_signed_raw_hex = strip_0x(o['params'][0]) + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + + return (block, tx) diff --git a/apps/cic-eth/tests/task/api/test_admin.py b/apps/cic-eth/tests/task/api/test_admin.py index a37e94a..95b1a06 100644 --- a/apps/cic-eth/tests/task/api/test_admin.py +++ b/apps/cic-eth/tests/task/api/test_admin.py @@ -9,8 +9,14 @@ from chainlib.eth.tx import ( unpack, TxFormat, ) -from chainlib.eth.nonce import RPCNonceOracle -from chainlib.eth.gas import Gas +from chainlib.eth.nonce import ( + RPCNonceOracle, + OverrideNonceOracle, + ) +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) from chainlib.eth.address import to_checksum_address from hexathon import ( strip_0x, @@ -23,7 +29,15 @@ from chainqueue.db.enum import ( StatusBits, status_str, ) -from chainqueue.query import get_tx +from chainqueue.state import ( + set_fubar, + set_ready, + set_reserved, + ) +from chainqueue.query import ( + get_tx, + get_nonce_tx_cache, + ) # local imports from cic_eth.api import AdminApi @@ -36,150 +50,6 @@ from cic_eth.queue.tx import queue_create logg = logging.getLogger() -#def test_resend_inplace( -# default_chain_spec, -# init_database, -# init_w3, -# celery_session_worker, -# ): -# -# chain_str = str(default_chain_spec) -# c = RpcClient(default_chain_spec) -# -# sigs = [] -# -# gas_provider = c.gas_provider() -# -# s_nonce = celery.signature( -# 'cic_eth.eth.nonce.reserve_nonce', -# [ -# init_w3.eth.accounts[0], -# gas_provider, -# ], -# queue=None, -# ) -# s_refill = celery.signature( -# 'cic_eth.eth.gas.refill_gas', -# [ -# chain_str, -# ], -# queue=None, -# ) -# s_nonce.link(s_refill) -# t = s_nonce.apply_async() -# t.get() -# for r in t.collect(): -# pass -# assert t.successful() -# -# q = init_database.query(Otx) -# q = q.join(TxCache) -# q = q.filter(TxCache.recipient==init_w3.eth.accounts[0]) -# o = q.first() -# tx_raw = o.signed_tx -# -# tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec) -# gas_price_before = tx_dict['gasPrice'] -# -# s = celery.signature( -# 'cic_eth.admin.ctrl.lock_send', -# [ -# chain_str, -# init_w3.eth.accounts[0], -# ], -# queue=None, -# ) -# t = s.apply_async() -# t.get() -# assert t.successful() -# -# api = AdminApi(c, queue=None) -# t = api.resend(tx_dict['hash'], chain_str, unlock=True) -# t.get() -# i = 0 -# tx_hash_new_hex = None -# for r in t.collect(): -# tx_hash_new_hex = r[1] -# assert t.successful() -# -# tx_raw_new = get_tx(tx_hash_new_hex) -# logg.debug('get {}'.format(tx_raw_new)) -# tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec) -# assert tx_hash_new_hex != tx_dict['hash'] -# assert tx_dict_new['gasPrice'] > gas_price_before -# -# tx_dict_after = get_tx(tx_dict['hash']) -# -# logg.debug('logggg {}'.format(status_str(tx_dict_after['status']))) -# assert tx_dict_after['status'] & StatusBits.MANUAL - - -#def test_check_fix_nonce( -# default_chain_spec, -# init_database, -# init_eth_account_roles, -# init_w3, -# eth_empty_accounts, -# celery_session_worker, -# ): -# -# chain_str = str(default_chain_spec) -# -# sigs = [] -# for i in range(5): -# s = celery.signature( -# 'cic_eth.eth.gas.refill_gas', -# [ -# eth_empty_accounts[i], -# chain_str, -# ], -# queue=None, -# ) -# sigs.append(s) -# -# t = celery.group(sigs)() -# txs = t.get() -# assert t.successful() -# -# tx_hash = web3.Web3.keccak(hexstr=txs[2]) -# c = RpcClient(default_chain_spec) -# api = AdminApi(c, queue=None) -# address = init_eth_account_roles['eth_account_gas_provider'] -# nonce_spec = api.check_nonce(address) -# assert nonce_spec['nonce']['network'] == 0 -# assert nonce_spec['nonce']['queue'] == 4 -# assert nonce_spec['nonce']['blocking'] == None -# -# s_set = celery.signature( -# 'cic_eth.queue.tx.set_rejected', -# [ -# tx_hash.hex(), -# ], -# queue=None, -# ) -# t = s_set.apply_async() -# t.get() -# t.collect() -# assert t.successful() -# -# -# nonce_spec = api.check_nonce(address) -# assert nonce_spec['nonce']['blocking'] == 2 -# assert nonce_spec['tx']['blocking'] == tx_hash.hex() -# -# t = api.fix_nonce(address, nonce_spec['nonce']['blocking']) -# t.get() -# t.collect() -# assert t.successful() -# -# for tx in txs[3:]: -# tx_hash = web3.Web3.keccak(hexstr=tx) -# tx_dict = get_tx(tx_hash.hex()) -# assert tx_dict['status'] == StatusEnum.OVERRIDDEN -# -# - - def test_have_account( default_chain_spec, custodial_roles, @@ -243,28 +113,6 @@ def test_tag_account( assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL'] -#def test_ready( -# init_database, -# agent_roles, -# eth_rpc, -# ): -# -# api = AdminApi(eth_rpc) -# -# with pytest.raises(InitializationError): -# api.ready() -# -# bogus_account = os.urandom(20) -# bogus_account_hex = '0x' + bogus_account.hex() -# -# api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex)) -# with pytest.raises(KeyError): -# api.ready() -# -# api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0]) -# api.ready() - - def test_tx( default_chain_spec, cic_registry, @@ -286,3 +134,168 @@ def test_tx( api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex) logg.warning('code missing to verify tx contents {}'.format(tx)) + + +def test_check_nonce_gap( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + caplog, + ): + + # NOTE: this only works as long as agents roles start at nonce 0 + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + tx_hashes = [] + txs = [] + + j = 0 + for i in range(10): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + if i == 3: + j = 1 + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1) + + queue_create( + default_chain_spec, + i+j, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes.append(tx_hash_hex) + txs.append(tx_signed_raw_hex) + + + init_database.commit() + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) + r = api.check_nonce(default_chain_spec, agent_roles['ALICE']) + + assert r['nonce']['blocking'] == 4 + assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap + + +def test_check_nonce_localfail( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + caplog, + ): + + # NOTE: this only works as long as agents roles start at nonce 0 + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + tx_hashes = [] + txs = [] + + j = 0 + for i in range(10): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + i, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes.append(tx_hash_hex) + txs.append(tx_signed_raw_hex) + + set_ready(default_chain_spec, tx_hashes[4], session=init_database) + set_reserved(default_chain_spec, tx_hashes[4], session=init_database) + set_fubar(default_chain_spec, tx_hashes[4], session=init_database) + + init_database.commit() + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) + r = api.check_nonce(default_chain_spec, agent_roles['ALICE']) + + assert r['nonce']['blocking'] == 4 + assert r['tx']['blocking'] == tx_hashes[4] + + +def test_fix_nonce( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + init_celery_tasks, + caplog, + ): + + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + tx_hashes = [] + txs = [] + + for i in range(10): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + i, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes.append(tx_hash_hex) + txs.append(tx_signed_raw_hex) + + init_database.commit() + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) + t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3) + r = t.get_leaf() + assert t.successful() + + init_database.commit() + + txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database) + ks = txs.keys() + assert len(ks) == 2 + for k in ks: + hsh = add_0x(k) + otx = Otx.load(hsh, session=init_database) + init_database.refresh(otx) + logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status)) + if add_0x(k) == tx_hashes[3]: + assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE + else: + assert otx.status == 1 diff --git a/apps/cic-eth/tests/task/api/test_admin_noncritical.py b/apps/cic-eth/tests/task/api/test_admin_noncritical.py new file mode 100644 index 0000000..fc10959 --- /dev/null +++ b/apps/cic-eth/tests/task/api/test_admin_noncritical.py @@ -0,0 +1,373 @@ +# standard imports +import logging +import io +import json + +# external imports +import pytest +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import ( + nonce, + OverrideNonceOracle, + RPCNonceOracle, + ) +from chainqueue.tx import create as queue_create +from chainlib.eth.tx import ( + raw, + receipt, + TxFormat, + Tx, + ) +from chainlib.eth.block import block_latest +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainqueue.state import ( + set_reserved, + set_sent, + set_ready, + ) +from chainqueue.db.models.otx import Otx +from chainqueue.db.enum import StatusBits +from chainqueue.query import get_nonce_tx_cache +from eth_erc20 import ERC20 +from cic_eth_registry import CICRegistry + +# local imports +from cic_eth.api.api_admin import AdminApi +from cic_eth.eth.gas import cache_gas_data +from cic_eth.eth.erc20 import cache_transfer_data + +logg = logging.getLogger() + + +def test_admin_api_tx( + default_chain_spec, + init_database, + init_celery_tasks, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + custodial_roles, + celery_session_worker, + foo_token, + address_declarator, + cic_registry, + register_tokens, + register_lookups, + caplog, + ): + + nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc) + + o = nonce(custodial_roles['FOO_TOKEN_GIFTER']) + r = eth_rpc.do(o) + gifter_nonce = int(r, 16) + + #c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + queue_create( + default_chain_spec, + gifter_nonce, # will only work if agent starts at 0 + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_transfer_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + init_database.commit() + + o = raw(tx_signed_raw_hex) + eth_rpc.do(o) + + o = receipt(tx_hash_hex) + r = eth_rpc.do(o) + assert r['status'] == 1 + + set_ready(default_chain_spec, tx_hash_hex, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex, session=init_database) + set_sent(default_chain_spec, tx_hash_hex, session=init_database) + + # lookup by transaction hash, without registry + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex) + logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER'])) + assert tx['tx_hash'] == tx_hash_hex + + # lookup by RLP transaction, without registry + tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex) + assert tx['tx_hash'] == tx_hash_hex + + # lookup by transaction hash, with registry + registry = CICRegistry(default_chain_spec, eth_rpc) + tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry) + assert tx['tx_hash'] == tx_hash_hex + + # lookup by transaction hash, using writer + buf = io.StringIO() + api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf) + tx = json.loads(buf.getvalue()) + assert tx['tx_hash'] == tx_hash_hex + + +def test_admin_api_account( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + caplog, + ): + + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + tx_hashes_alice = [] + txs_alice = [] + + for i in range(3): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + queue_create( + default_chain_spec, + 42+i, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes_alice.append(tx_hash_hex) + txs_alice.append(tx_signed_raw_hex) + + init_database.commit() + + nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13) + tx_hashes_bob = [] + txs_bob = [] + + for i in range(2): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + queue_create( + default_chain_spec, + 13+i, + agent_roles['BOB'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes_bob.append(tx_hash_hex) + txs_bob.append(tx_signed_raw_hex) + + init_database.commit() + + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + r = api.account(default_chain_spec, agent_roles['ALICE']) + assert len(r) == 5 + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False) + assert len(r) == 2 + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False) + assert len(r) == 3 + + +def test_admin_api_account_writer( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + caplog, + ): + + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + init_database.commit() + + buf = io.StringIO() + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf) + + # TODO: improve eval + tx = json.loads(buf.getvalue()) + assert tx['tx_hash'] == tx_hash_hex + + +def test_registry( + eth_rpc, + cic_registry, + contract_roles, + celery_session_worker, + ): + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + t = api.registry() + r = t.get_leaf() + assert r == cic_registry + + +def test_proxy_do( + default_chain_spec, + eth_rpc, + contract_roles, + celery_session_worker, + ): + + o = block_latest() + r = eth_rpc.do(o) + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + t = api.proxy_do(default_chain_spec, o) + rr = t.get_leaf() + + assert r == rr + + +def test_resend_inplace( + init_database, + default_chain_spec, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + set_ready(default_chain_spec, tx_hash_hex, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex, session=init_database) + set_sent(default_chain_spec, tx_hash_hex, session=init_database) + + init_database.commit() + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + t = api.resend(tx_hash_hex, default_chain_spec, unlock=True) + r = t.get_leaf() + assert t.successful() + + + otx = Otx.load(tx_hash_hex, session=init_database) + assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE + + txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database) + assert len(txs) == 2 + + + +@pytest.mark.xfail() +def test_resend_clone( + init_database, + default_chain_spec, + eth_rpc, + eth_signer, + agent_roles, + contract_roles, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + set_ready(default_chain_spec, tx_hash_hex, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex, session=init_database) + set_sent(default_chain_spec, tx_hash_hex, session=init_database) + + init_database.commit() + + api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) + t = api.resend(tx_hash_hex, default_chain_spec, in_place=False) + r = t.get_leaf() + assert t.successful() + + otx = Otx.load(tx_hash_hex, session=init_database) + assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK + assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE + + txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database) + assert len(txs) == 1 + + txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database) + assert len(txs) == 1 + + otx = Otx.load(txs[0], session=init_database) + assert otx.status == 0 diff --git a/apps/cic-eth/tests/task/api/test_app.py b/apps/cic-eth/tests/task/api/test_app.py index 1d3e64a..321009f 100644 --- a/apps/cic-eth/tests/task/api/test_app.py +++ b/apps/cic-eth/tests/task/api/test_app.py @@ -8,11 +8,20 @@ import pytest import celery from cic_eth_registry.erc20 import ERC20Token from chainlib.chain import ChainSpec +from eth_accounts_index import AccountsIndex +from chainlib.eth.tx import ( + transaction, + ) +from chainqueue.state import ( + set_reserved, + ) # local imports from cic_eth.api import Api +from cic_eth.queue.query import get_tx -logg = logging.getLogger(__name__) +#logg = logging.getLogger(__name__) +logg = logging.getLogger() def test_account_api( @@ -29,6 +38,47 @@ def test_account_api( assert t.successful() +def test_account_api_register( + default_chain_spec, + init_database, + account_registry, + faucet, + custodial_roles, + cic_registry, + register_lookups, + eth_rpc, + celery_session_worker, + ): + api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None) + t = api.create_account('') + register_tx_hash = t.get_leaf() + assert t.successful() + + set_reserved(default_chain_spec, register_tx_hash, session=init_database) + + tx = get_tx(default_chain_spec.asdict(), register_tx_hash) + s = celery.signature( + 'cic_eth.eth.tx.send', + [ + [tx['signed_tx']], + default_chain_spec.asdict(), + ], + queue=None + ) + t = s.apply_async() + r = t.get_leaf() + assert t.successful() + + o = transaction(register_tx_hash) + tx_src = eth_rpc.do(o) + + c = AccountsIndex(default_chain_spec) + address = c.parse_add_request(tx_src['data']) + o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER']) + r = eth_rpc.do(o) + assert c.parse_have(r) + + def test_transfer_api( default_chain_spec, eth_rpc, @@ -37,16 +87,15 @@ def test_transfer_api( custodial_roles, agent_roles, cic_registry, - register_tokens, + token_registry, register_lookups, celery_session_worker, + register_tokens, + foo_token_symbol, ): - #token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0]) - foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token) - api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None) - t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol) + t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol) t.get_leaf() assert t.successful() diff --git a/apps/cic-eth/tests/task/api/test_app_noncritical.py b/apps/cic-eth/tests/task/api/test_app_noncritical.py new file mode 100644 index 0000000..7ebb870 --- /dev/null +++ b/apps/cic-eth/tests/task/api/test_app_noncritical.py @@ -0,0 +1,19 @@ +# local imports +from cic_eth.api.api_task import Api +from cic_eth.task import BaseTask + +def test_default_token( + default_chain_spec, + foo_token, + default_token, + token_registry, + register_tokens, + register_lookups, + cic_registry, + celery_session_worker, + ): + + api = Api(str(default_chain_spec), queue=None) + t = api.default_token() + r = t.get_leaf() + assert r['address'] == foo_token diff --git a/apps/cic-eth/tests/task/test_task_account.py b/apps/cic-eth/tests/task/test_task_account.py index a384bab..d1c1c3d 100644 --- a/apps/cic-eth/tests/task/test_task_account.py +++ b/apps/cic-eth/tests/task/test_task_account.py @@ -156,6 +156,7 @@ def test_gift( eth_signer, init_celery_tasks, cic_registry, + register_lookups, celery_session_worker, ): diff --git a/apps/cic-eth/tests/task/test_task_admin.py b/apps/cic-eth/tests/task/test_task_admin.py new file mode 100644 index 0000000..7efa828 --- /dev/null +++ b/apps/cic-eth/tests/task/test_task_admin.py @@ -0,0 +1,88 @@ +# standard imports +import logging + +# external imports +import celery +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import OverrideNonceOracle +from chainqueue.tx import ( + create as queue_create, + ) +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainlib.eth.tx import TxFormat +from chainqueue.query import get_nonce_tx_cache +from chainqueue.db.models.otx import Otx +from chainqueue.db.enum import StatusBits +from hexathon import add_0x + +# local imports +from cic_eth.admin.nonce import shift_nonce +from cic_eth.eth.gas import cache_gas_data + +logg = logging.getLogger() + + +def test_shift_nonce( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + celery_session_worker, + caplog, + ): + + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc) + + tx_hashes = [] + txs = [] + + for i in range(10): + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + queue_create( + default_chain_spec, + 42+i, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_hashes.append(tx_hash_hex) + txs.append(tx_signed_raw_hex) + + init_database.commit() + + s = celery.signature( + 'cic_eth.admin.nonce.shift_nonce', + [ + default_chain_spec.asdict(), + tx_hashes[3], + ], + queue=None + ) + t = s.apply_async() + r = t.get_leaf() + assert t.successful() + init_database.commit() + + + for i in range(42+3, 42+10): + txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database) + for k in txs.keys(): + hsh = add_0x(k) + otx = Otx.load(hsh, session=init_database) + logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status)) + if add_0x(k) == tx_hashes[i-42]: + assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE + else: + assert otx.status == 1 diff --git a/apps/cic-eth/tests/task/test_task_gas.py b/apps/cic-eth/tests/task/test_task_gas.py new file mode 100644 index 0000000..3bd3dd8 --- /dev/null +++ b/apps/cic-eth/tests/task/test_task_gas.py @@ -0,0 +1,286 @@ +# standard imports +import logging + +# external imports +import celery +import pytest +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import ( + OverrideNonceOracle, + RPCNonceOracle, + ) +from chainlib.eth.gas import ( + OverrideGasOracle, + Gas, + ) +from chainlib.eth.tx import ( + unpack, + TxFormat, + ) +from chainlib.eth.constant import ( + MINIMUM_FEE_UNITS, + MINIMUM_FEE_PRICE, + ) +from chainqueue.tx import create as queue_create +from chainqueue.query import get_tx +from chainqueue.db.enum import StatusBits +from chainqueue.state import ( + set_ready, + set_reserved, + set_sent, + ) +from chainqueue.db.models.otx import Otx +from hexathon import strip_0x + +# local imports +from cic_eth.eth.gas import cache_gas_data +from cic_eth.error import OutOfGasError + +logg = logging.getLogger() + + +def test_task_check_gas_ok( + default_chain_spec, + eth_rpc, + eth_signer, + init_database, + agent_roles, + custodial_roles, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 0, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + init_database.commit() + + s = celery.signature( + 'cic_eth.eth.gas.check_gas', + [ + [ + tx_hash_hex, + ], + default_chain_spec.asdict(), + [], + None, + 8000000, + ], + queue=None + ) + t = s.apply_async() + t.get_leaf() + assert t.successful() + + init_database.commit() + + tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database) + assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED + + +def test_task_check_gas_insufficient( + default_chain_spec, + eth_rpc, + eth_signer, + init_database, + agent_roles, + custodial_roles, + celery_session_worker, + whoever, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = OverrideNonceOracle(whoever, 42) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + whoever, + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + init_database.commit() + + s = celery.signature( + 'cic_eth.eth.gas.check_gas', + [ + [ + tx_hash_hex, + ], + default_chain_spec.asdict(), + [], + None, + None, + ], + queue=None + ) + t = s.apply_async() + try: + r = t.get_leaf() + except OutOfGasError: + pass + + init_database.commit() + + tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database) + assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES + + +def test_task_check_gas_low( + default_chain_spec, + eth_rpc, + eth_signer, + init_database, + agent_roles, + custodial_roles, + celery_session_worker, + whoever, + ): + + gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS) + nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6)) + r = eth_rpc.do(o) + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 0, + whoever, + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + + init_database.commit() + + s = celery.signature( + 'cic_eth.eth.gas.check_gas', + [ + [ + tx_hash_hex, + ], + default_chain_spec.asdict(), + ], + [], + None, + None, + queue=None + ) + t = s.apply_async() + t.get_leaf() + assert t.successful() + + init_database.commit() + + tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database) + assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED + + +@pytest.mark.parametrize( + '_gas_price,_gas_factor', + [ + (None, 1.1), + (MINIMUM_FEE_PRICE * 1.1, 0.9), + (None, 1.3), + ] + ) +def test_task_resend_explicit( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + custodial_roles, + celery_session_worker, + _gas_price, + _gas_factor, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 0, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) + + init_database.commit() + + set_ready(default_chain_spec, tx_hash_hex, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex, session=init_database) + set_sent(default_chain_spec, tx_hash_hex, session=init_database) + + s = celery.signature( + 'cic_eth.eth.gas.resend_with_higher_gas', + [ + tx_hash_hex, + default_chain_spec.asdict(), + _gas_price, + _gas_factor, + ], + queue=None + ) + t = s.apply_async() + r = t.get_leaf() + assert t.successful() + + q = init_database.query(Otx) + q = q.filter(Otx.tx_hash==strip_0x(r)) + otx = q.first() + if otx == None: + raise NotLocalTxError(r) + + tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec) + logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice'])) + assert tx_after['gasPrice'] > tx_before['gasPrice'] + diff --git a/apps/cic-eth/tests/task/test_task_tx.py b/apps/cic-eth/tests/task/test_task_tx.py index 939b800..ecf6899 100644 --- a/apps/cic-eth/tests/task/test_task_tx.py +++ b/apps/cic-eth/tests/task/test_task_tx.py @@ -4,16 +4,27 @@ import logging # external imports import pytest import celery -from chainlib.eth.gas import Gas +from chainlib.eth.gas import ( + OverrideGasOracle, + Gas, + ) from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.tx import ( TxFormat, unpack, transaction, receipt, + raw, ) from hexathon import strip_0x from chainqueue.db.models.otx import Otx +from chainqueue.tx import create as queue_create +from chainqueue.state import ( + set_reserved, + set_ready, + set_sent, + ) +from chainqueue.db.enum import StatusBits # local imports from cic_eth.queue.tx import register_tx @@ -60,15 +71,6 @@ def test_tx_send( def test_sync_tx( - default_chain_spec, - eth_rpc, - eth_signer, - celery_session_worker, - ): - pass - - -def test_resend_with_higher_gas( init_database, default_chain_spec, eth_rpc, @@ -77,31 +79,48 @@ def test_resend_with_higher_gas( celery_session_worker, ): - nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) - c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) - (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) - register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) - cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict()) - tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) + nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex, + tx_signed_raw_hex, + session=init_database, + ) + cache_gas_data( + tx_hash_hex, + tx_signed_raw_hex, + default_chain_spec.asdict(), + ) + set_ready(default_chain_spec, tx_hash_hex, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex, session=init_database) + set_sent(default_chain_spec, tx_hash_hex, session=init_database) + + o = raw(tx_signed_raw_hex) + r = eth_rpc.do(o) + + o = receipt(tx_hash_hex) + r = eth_rpc.do(o) + assert r['status'] == 1 s = celery.signature( - 'cic_eth.eth.gas.resend_with_higher_gas', + 'cic_eth.eth.tx.sync_tx', [ tx_hash_hex, default_chain_spec.asdict(), ], - queue=None, + queue=None ) t = s.apply_async() r = t.get_leaf() + assert t.successful() - q = init_database.query(Otx) - q = q.filter(Otx.tx_hash==strip_0x(r)) - otx = q.first() - if otx == None: - raise NotLocalTxError(r) - - tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec) - logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice'])) - assert tx_after['gasPrice'] > tx_before['gasPrice'] + init_database.commit() + o = Otx.load(tx_hash_hex, session=init_database) + assert o.status & StatusBits.FINAL == StatusBits.FINAL diff --git a/apps/cic-eth/tests/task/test_task_tx_misc.py b/apps/cic-eth/tests/task/test_task_tx_misc.py new file mode 100644 index 0000000..992abb8 --- /dev/null +++ b/apps/cic-eth/tests/task/test_task_tx_misc.py @@ -0,0 +1,170 @@ +# standard imports +import os +import logging + +# external imports +import pytest +import celery +from chainqueue.tx import create as queue_create +from chainlib.eth.nonce import ( + RPCNonceOracle, + OverrideNonceOracle, + ) +from chainlib.eth.gas import ( + OverrideGasOracle, + Gas, + ) +from chainlib.eth.tx import ( + TxFormat, + unpack, + receipt, + ) +from hexathon import ( + add_0x, + strip_0x, + ) +from chainqueue.state import ( + set_reserved, + set_ready, + ) + +logg = logging.getLogger() + + +def test_hashes_to_txs( + init_database, + default_chain_spec, + agent_roles, + eth_rpc, + eth_signer, + celery_session_worker, + ): + + nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex_one, + tx_signed_raw_hex_one, + session=init_database, + ) + + #nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 43, + agent_roles['ALICE'], + tx_hash_hex_two, + tx_signed_raw_hex_two, + session=init_database, + ) + + init_database.commit() + + bogus_one = add_0x(os.urandom(32).hex()) + bogus_two = add_0x(os.urandom(32).hex()) + + yarrgs = [ + bogus_one, + tx_hash_hex_two, + bogus_two, + tx_hash_hex_one, + ] + s = celery.signature( + 'cic_eth.eth.tx.hashes_to_txs', + [ + yarrgs, + ], + queue=None, + ) + t = s.apply_async() + r = t.get_leaf() + assert t.successful() + assert len(r) == 2 + + logg.debug('r {}'.format(r)) + txs = [ + tx_signed_raw_hex_two, + tx_signed_raw_hex_one, + ] + for tx in r: + txs.remove(add_0x(tx)) + assert len(txs) == 0 + + + +def test_double_send( + init_database, + default_chain_spec, + agent_roles, + eth_rpc, + eth_signer, + celery_session_worker, + ): + + nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=21000) + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 42, + agent_roles['ALICE'], + tx_hash_hex_one, + tx_signed_raw_hex_one, + session=init_database, + ) + set_ready(default_chain_spec, tx_hash_hex_one, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database) + + c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED) + + queue_create( + default_chain_spec, + 43, + agent_roles['ALICE'], + tx_hash_hex_two, + tx_signed_raw_hex_two, + session=init_database, + ) + + set_ready(default_chain_spec, tx_hash_hex_two, session=init_database) + set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database) + init_database.commit() + + yarrgs = [ + tx_signed_raw_hex_one, + tx_signed_raw_hex_two, + ] + s = celery.signature( + 'cic_eth.eth.tx.send', + [ + yarrgs, + default_chain_spec.asdict(), + ], + queue=None + ) + t = s.apply_async() + r = t.get_leaf() + assert t.successful() + + o = receipt(tx_hash_hex_one) + r = eth_rpc.do(o) + assert r['status'] == 1 + + o = receipt(tx_hash_hex_two) + r = eth_rpc.do(o) + assert r['status'] == 1 + + + diff --git a/apps/cic-eth/tests/testdata/Bogus.bin b/apps/cic-eth/tests/testdata/Bogus.bin new file mode 100644 index 0000000..eacfe1b --- /dev/null +++ b/apps/cic-eth/tests/testdata/Bogus.bin @@ -0,0 +1 @@ +60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033 \ No newline at end of file diff --git a/apps/cic-eth/tests/testdata/Bogus.sol b/apps/cic-eth/tests/testdata/Bogus.sol new file mode 100644 index 0000000..17e3bb8 --- /dev/null +++ b/apps/cic-eth/tests/testdata/Bogus.sol @@ -0,0 +1,10 @@ +pragma solidity ^0.8.0; + +contract Bogus { + + uint256 public a = 0; + + function poke() public { + a++; + } +} diff --git a/apps/cic-eth/tests/unit/ext/test_address.py b/apps/cic-eth/tests/unit/ext/test_address.py index 2a76892..b7d92b6 100644 --- a/apps/cic-eth/tests/unit/ext/test_address.py +++ b/apps/cic-eth/tests/unit/ext/test_address.py @@ -19,6 +19,7 @@ def test_translate( agent_roles, cic_registry, init_celery_tasks, + register_lookups, ): nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) diff --git a/apps/cic-eth/tests/unit/test_registry_connect.py b/apps/cic-eth/tests/unit/test_registry_connect.py new file mode 100644 index 0000000..62f14c1 --- /dev/null +++ b/apps/cic-eth/tests/unit/test_registry_connect.py @@ -0,0 +1,22 @@ +# local imports +from cic_eth.registry import * + +def test_registry_connect( + eth_rpc, + default_chain_spec, + address_declarator, + token_registry, + contract_roles, + purge_lookups, + registry, + agent_roles, + ): + + r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER']) + + connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER']) + r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER']) + + connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER']) + r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER']) + diff --git a/apps/cic-eth/tests/unit/test_stat.py b/apps/cic-eth/tests/unit/test_stat.py new file mode 100644 index 0000000..5da49fd --- /dev/null +++ b/apps/cic-eth/tests/unit/test_stat.py @@ -0,0 +1,18 @@ +# standard imports +import datetime + +# local imports +from cic_eth.stat import init_chain_stat + + +def test_chain_stat( + eth_rpc, + init_eth_tester, + ): + + now = int(datetime.datetime.now().timestamp()) + 1 + for i in range(11): + init_eth_tester.time_travel(now + (i * 2)) + + s = init_chain_stat(eth_rpc, block_start=0) + assert s.block_average() == 2