kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into lash/stale-import-deps

Browse Source
This commit is contained in:
nolash 2021-06-02 17:31:47 +02:00
commit 19c17f4bbb
45 changed files with 2121 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/cic_cache/runnable/daemons/query.py
View File

@ -89,7 +89,7 @@ def process_transactions_all_data(session, env):
offset = r[1] offset = r[1]
end = r[2] end = r[2]
if r[2] < r[1]: if int(r[2]) < int(r[1]):
raise ValueError('cart before the horse, dude') raise ValueError('cart before the horse, dude')
c = DataCache(session) c = DataCache(session)

2
apps/cic-eth/.coveragerc
View File

@ -5,3 +5,5 @@ omit =
cic_eth/db/migrations/* cic_eth/db/migrations/*
cic_eth/sync/head.py cic_eth/sync/head.py
cic_eth/sync/mempool.py cic_eth/sync/mempool.py
cic_eth/queue/state.py
*redis*.py

4
apps/cic-eth/.gitlab-ci.yml
View File

@ -24,9 +24,7 @@ test-mr-cic-eth:
image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
script: script:
- cd apps/$APP_NAME/ - cd apps/$APP_NAME/
- pytest tests/unit/ - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
- pytest tests/task/
- pytest tests/filters/
needs: ["build-mr-cic-eth"] needs: ["build-mr-cic-eth"]
build-push-cic-eth: build-push-cic-eth:

78
apps/cic-eth/cic_eth/admin/nonce.py
View File

@ -4,11 +4,18 @@ import logging
# external imports # external imports
import celery import celery
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack from chainlib.connection import RPCConnection
from chainqueue.query import get_tx from chainlib.eth.tx import (
from chainqueue.state import set_cancel unpack,
TxFactory,
)
from chainlib.eth.gas import OverrideGasOracle
from chainqueue.sql.query import get_tx
from chainqueue.sql.state import set_cancel
from chainqueue.db.models.otx import Otx from chainqueue.db.models.otx import Otx
from chainqueue.db.models.tx import TxCache from chainqueue.db.models.tx import TxCache
from hexathon import strip_0x
from potaahto.symbols import snake_and_camel
# local imports # local imports
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
@ -21,13 +28,14 @@ from cic_eth.admin.ctrl import (
) )
from cic_eth.queue.tx import queue_create from cic_eth.queue.tx import queue_create
from cic_eth.eth.gas import create_check_gas_task from cic_eth.eth.gas import create_check_gas_task
from cic_eth.task import BaseTask
celery_app = celery.current_app celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
@celery_app.task(bind=True) @celery_app.task(bind=True, base=BaseTask)
def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1): def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
"""Shift all transactions with nonces higher than the offset by the provided position delta. """Shift all transactions with nonces higher than the offset by the provided position delta.
Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN. Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@ -38,25 +46,29 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
:type tx_hash_orig_hex: str, 0x-hex :type tx_hash_orig_hex: str, 0x-hex
:param delta: Amount :param delta: Amount
""" """
chain_spec = ChainSpec.from_dict(chainspec_dict)
rpc = RPCConnection.connect(chain_spec, 'default')
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
queue = None queue = None
try: try:
queue = self.request.delivery_info.get('routing_key') queue = self.request.delivery_info.get('routing_key')
except AttributeError: except AttributeError:
pass pass
chain_spec = ChainSpec.from_chain_str(chain_str) session = BaseTask.session_func()
tx_brief = get_tx(tx_hash_orig_hex) tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx'][2:])) tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
tx = unpack(tx_raw, chain_spec) tx = unpack(tx_raw, chain_spec)
nonce = tx_brief['nonce'] nonce = tx_brief['nonce']
address = tx['from'] address = tx['from']
logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce)) logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
lock_queue(None, chain_str, address) lock_queue(None, chain_spec.asdict(), address=address)
lock_send(None, chain_str, address) lock_send(None, chain_spec.asdict(), address=address)
set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
session = SessionBase.create_session()
q = session.query(Otx) q = session.query(Otx)
q = q.join(TxCache) q = q.join(TxCache)
q = q.filter(TxCache.sender==address) q = q.filter(TxCache.sender==address)
@ -69,49 +81,57 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
for otx in otxs: for otx in otxs:
tx_raw = bytes.fromhex(strip_0x(otx.signed_tx)) tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
tx_new = unpack(tx_raw, chain_spec) tx_new = unpack(tx_raw, chain_spec)
tx_new = snake_and_camel(tx_new)
tx_previous_hash_hex = tx_new['hash'] tx_previous_hash_hex = tx_new['hash']
tx_previous_nonce = tx_new['nonce'] tx_previous_nonce = tx_new['nonce']
del(tx_new['hash']) tx_new['gas_price'] += 1
del(tx_new['hash_unsigned']) tx_new['gasPrice'] = tx_new['gas_price']
tx_new['nonce'] -= delta tx_new['nonce'] -= delta
(tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str) logg.debug('tx_new {}'.format(tx_new))
del(tx_new['hash'])
del(tx_new['hash_unsigned'])
del(tx_new['hashUnsigned'])
gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce'])) logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
otx = Otx( otx = Otx(
nonce=tx_new['nonce'], tx_new['nonce'],
address=tx_new['from'], tx_hash_hex,
tx_hash=tx_hash_hex, tx_signed_raw_hex,
signed_tx=tx_signed_raw_hex,
) )
session.add(otx) session.add(otx)
session.commit()
# TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces. # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
set_cancel(tx_previous_hash_hex, True) set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
TxCache.clone(tx_previous_hash_hex, tx_hash_hex) TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
tx_hashes.append(tx_hash_hex) tx_hashes.append(tx_hash_hex)
txs.append(tx_signed_raw_hex) txs.append(tx_signed_raw_hex)
session.commit()
session.close() session.close()
s = create_check_gas_and_send_task( s = create_check_gas_task(
txs, txs,
chain_str, chain_spec,
tx_new['from'], tx_new['from'],
tx_new['gas'], gas=tx_new['gas'],
tx_hashes, tx_hashes_hex=tx_hashes,
queue, queue=queue,
) )
s_unlock_send = celery.signature( s_unlock_send = celery.signature(
'cic_eth.admin.ctrl.unlock_send', 'cic_eth.admin.ctrl.unlock_send',
[ [
chain_str, chain_spec.asdict(),
tx_new['from'], tx_new['from'],
], ],
queue=queue, queue=queue,
@ -119,7 +139,7 @@ def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
s_unlock_direct = celery.signature( s_unlock_direct = celery.signature(
'cic_eth.admin.ctrl.unlock_queue', 'cic_eth.admin.ctrl.unlock_queue',
[ [
chain_str, chain_spec.asdict(),
tx_new['from'], tx_new['from'],
], ],
queue=queue, queue=queue,

62
apps/cic-eth/cic_eth/api/api_admin.py
View File

@ -8,6 +8,7 @@ from chainlib.eth.constant import (
ZERO_ADDRESS, ZERO_ADDRESS,
) )
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from cic_eth_registry.error import UnknownContractError from cic_eth_registry.error import UnknownContractError
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
from chainlib.eth.contract import code from chainlib.eth.contract import code
@ -30,13 +31,14 @@ from chainqueue.db.enum import (
status_str, status_str,
) )
from chainqueue.error import TxStateChangeError from chainqueue.error import TxStateChangeError
from chainqueue.query import get_tx
from eth_erc20 import ERC20
# local imports # local imports
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.nonce import Nonce
from cic_eth.error import InitializationError from cic_eth.error import InitializationError
from cic_eth.queue.query import get_tx
app = celery.current_app app = celery.current_app
@ -188,6 +190,7 @@ class AdminApi:
s_manual = celery.signature( s_manual = celery.signature(
'cic_eth.queue.state.set_manual', 'cic_eth.queue.state.set_manual',
[ [
chain_spec.asdict(),
tx_hash_hex, tx_hash_hex,
], ],
queue=self.queue, queue=self.queue,
@ -207,7 +210,8 @@ class AdminApi:
return s_manual.apply_async() return s_manual.apply_async()
def check_nonce(self, address):
def check_nonce(self, chain_spec, address):
s = celery.signature( s = celery.signature(
'cic_eth.queue.query.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
@ -234,7 +238,6 @@ class AdminApi:
queue=self.queue, queue=self.queue,
) )
tx = s_get_tx.apply_async().get() tx = s_get_tx.apply_async().get()
#tx = get_tx(k)
logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce)) logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
nonce_otx = tx['nonce'] nonce_otx = tx['nonce']
if not is_alive(tx['status']) and tx['status'] & local_fail > 0: if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
@ -242,7 +245,9 @@ class AdminApi:
blocking_tx = k blocking_tx = k
blocking_nonce = nonce_otx blocking_nonce = nonce_otx
elif nonce_otx - last_nonce > 1: elif nonce_otx - last_nonce > 1:
logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx['from'])) logg.debug('tx {}'.format(tx))
tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
blocking_tx = k blocking_tx = k
blocking_nonce = nonce_otx blocking_nonce = nonce_otx
break break
@ -256,12 +261,13 @@ class AdminApi:
'blocking': blocking_nonce, 'blocking': blocking_nonce,
}, },
'tx': { 'tx': {
'blocking': blocking_tx, 'blocking': add_0x(blocking_tx),
} }
} }
def fix_nonce(self, address, nonce, chain_spec): # TODO: is risky since it does not validate that there is actually a nonce problem?
def fix_nonce(self, chain_spec, address, nonce):
s = celery.signature( s = celery.signature(
'cic_eth.queue.query.get_account_tx', 'cic_eth.queue.query.get_account_tx',
[ [
@ -275,15 +281,17 @@ class AdminApi:
txs = s.apply_async().get() txs = s.apply_async().get()
tx_hash_hex = None tx_hash_hex = None
session = SessionBase.create_session()
for k in txs.keys(): for k in txs.keys():
tx_dict = get_tx(k) tx_dict = get_tx(chain_spec, k, session=session)
if tx_dict['nonce'] == nonce: if tx_dict['nonce'] == nonce:
tx_hash_hex = k tx_hash_hex = k
session.close()
s_nonce = celery.signature( s_nonce = celery.signature(
'cic_eth.admin.nonce.shift_nonce', 'cic_eth.admin.nonce.shift_nonce',
[ [
self.rpc.chain_spec.asdict(), chain_spec.asdict(),
tx_hash_hex, tx_hash_hex,
], ],
queue=self.queue queue=self.queue
@ -391,9 +399,10 @@ class AdminApi:
source_token = None source_token = None
if tx['source_token'] != ZERO_ADDRESS: if tx['source_token'] != ZERO_ADDRESS:
source_token_declaration = None
if registry != None: if registry != None:
try: try:
source_token = registry.by_address(tx['source_token']) source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
except UnknownContractError: except UnknownContractError:
logg.warning('unknown source token contract {} (direct)'.format(tx['source_token'])) logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
else: else:
@ -406,16 +415,21 @@ class AdminApi:
queue=self.queue queue=self.queue
) )
t = s.apply_async() t = s.apply_async()
source_token = t.get() source_token_declaration = t.get()
if source_token == None:
logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token'])) if source_token_declaration != None:
logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
logg.debug('source token set tup {}'.format(source_token))
destination_token = None destination_token = None
if tx['destination_token'] != ZERO_ADDRESS: if tx['destination_token'] != ZERO_ADDRESS:
destination_token_declaration = None
if registry != None: if registry != None:
try: try:
destination_token = registry.by_address(tx['destination_token']) destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
except UnknownContractError: except UnknownContractError:
logg.warning('unknown destination token contract {}'.format(tx['destination_token'])) logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
else: else:
@ -428,10 +442,10 @@ class AdminApi:
queue=self.queue queue=self.queue
) )
t = s.apply_async() t = s.apply_async()
destination_token = t.get() destination_token_declaration = t.get()
if destination_token == None: if destination_token_declaration != None:
logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token'])) logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
tx['sender_description'] = 'Custodial account' tx['sender_description'] = 'Custodial account'
tx['recipient_description'] = 'Custodial account' tx['recipient_description'] = 'Custodial account'
@ -543,13 +557,19 @@ class AdminApi:
if role != None: if role != None:
tx['recipient_description'] = role tx['recipient_description'] = role
erc20_c = ERC20(chain_spec)
if source_token != None: if source_token != None:
tx['source_token_symbol'] = source_token.symbol() tx['source_token_symbol'] = source_token.symbol
tx['sender_token_balance'] = source_token.function('balanceOf')(tx['sender']).call() o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
r = self.rpc.do(o)
tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
if destination_token != None: if destination_token != None:
tx['destination_token_symbol'] = destination_token.symbol() tx['destination_token_symbol'] = destination_token.symbol
tx['recipient_token_balance'] = source_token.function('balanceOf')(tx['recipient']).call() o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
r = self.rpc.do(o)
tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
#tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
# TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
tx['network_status'] = 'Not in node' tx['network_status'] = 'Not in node'

256
apps/cic-eth/cic_eth/api/api_task.py
View File

@ -74,134 +74,134 @@ class Api:
return s_token.apply_async() return s_token.apply_async()
def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol): # def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
"""Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed. # """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
#
:param from_address: Ethereum address of sender # :param from_address: Ethereum address of sender
:type from_address: str, 0x-hex # :type from_address: str, 0x-hex
:param to_address: Ethereum address of receipient # :param to_address: Ethereum address of receipient
:type to_address: str, 0x-hex # :type to_address: str, 0x-hex
:param target_return: Estimated return from conversion # :param target_return: Estimated return from conversion
:type target_return: int # :type target_return: int
:param minimum_return: The least value of destination token return to allow # :param minimum_return: The least value of destination token return to allow
:type minimum_return: int # :type minimum_return: int
:param from_token_symbol: ERC20 token symbol of token being converted # :param from_token_symbol: ERC20 token symbol of token being converted
:type from_token_symbol: str # :type from_token_symbol: str
:param to_token_symbol: ERC20 token symbol of token to receive # :param to_token_symbol: ERC20 token symbol of token to receive
:type to_token_symbol: str # :type to_token_symbol: str
:returns: uuid of root task # :returns: uuid of root task
:rtype: celery.Task # :rtype: celery.Task
""" # """
raise NotImplementedError('out of service until new DEX migration is done') # raise NotImplementedError('out of service until new DEX migration is done')
s_check = celery.signature( # s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock', # 'cic_eth.admin.ctrl.check_lock',
[ # [
[from_token_symbol, to_token_symbol], # [from_token_symbol, to_token_symbol],
self.chain_spec.asdict(), # self.chain_spec.asdict(),
LockEnum.QUEUE, # LockEnum.QUEUE,
from_address, # from_address,
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_nonce = celery.signature( # s_nonce = celery.signature(
'cic_eth.eth.nonce.reserve_nonce', # 'cic_eth.eth.nonce.reserve_nonce',
[ # [
self.chain_spec.asdict(), # self.chain_spec.asdict(),
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_tokens = celery.signature( # s_tokens = celery.signature(
'cic_eth.eth.erc20.resolve_tokens_by_symbol', # 'cic_eth.eth.erc20.resolve_tokens_by_symbol',
[ # [
self.chain_str, # self.chain_str,
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_convert = celery.signature( # s_convert = celery.signature(
'cic_eth.eth.bancor.convert_with_default_reserve', # 'cic_eth.eth.bancor.convert_with_default_reserve',
[ # [
from_address, # from_address,
target_return, # target_return,
minimum_return, # minimum_return,
to_address, # to_address,
self.chain_spec.asdict(), # self.chain_spec.asdict(),
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_nonce.link(s_tokens) # s_nonce.link(s_tokens)
s_check.link(s_nonce) # s_check.link(s_nonce)
if self.callback_param != None: # if self.callback_param != None:
s_convert.link(self.callback_success) # s_convert.link(self.callback_success)
s_tokens.link(s_convert).on_error(self.callback_error) # s_tokens.link(s_convert).on_error(self.callback_error)
else: # else:
s_tokens.link(s_convert) # s_tokens.link(s_convert)
#
t = s_check.apply_async(queue=self.queue) # t = s_check.apply_async(queue=self.queue)
return t # return t
#
#
def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol): # def convert(self, from_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
"""Executes a chain of celery tasks that performs conversion between two ERC20 tokens. # """Executes a chain of celery tasks that performs conversion between two ERC20 tokens.
#
:param from_address: Ethereum address of sender # :param from_address: Ethereum address of sender
:type from_address: str, 0x-hex # :type from_address: str, 0x-hex
:param target_return: Estimated return from conversion # :param target_return: Estimated return from conversion
:type target_return: int # :type target_return: int
:param minimum_return: The least value of destination token return to allow # :param minimum_return: The least value of destination token return to allow
:type minimum_return: int # :type minimum_return: int
:param from_token_symbol: ERC20 token symbol of token being converted # :param from_token_symbol: ERC20 token symbol of token being converted
:type from_token_symbol: str # :type from_token_symbol: str
:param to_token_symbol: ERC20 token symbol of token to receive # :param to_token_symbol: ERC20 token symbol of token to receive
:type to_token_symbol: str # :type to_token_symbol: str
:returns: uuid of root task # :returns: uuid of root task
:rtype: celery.Task # :rtype: celery.Task
""" # """
raise NotImplementedError('out of service until new DEX migration is done') # raise NotImplementedError('out of service until new DEX migration is done')
s_check = celery.signature( # s_check = celery.signature(
'cic_eth.admin.ctrl.check_lock', # 'cic_eth.admin.ctrl.check_lock',
[ # [
[from_token_symbol, to_token_symbol], # [from_token_symbol, to_token_symbol],
self.chain_spec.asdict(), # self.chain_spec.asdict(),
LockEnum.QUEUE, # LockEnum.QUEUE,
from_address, # from_address,
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_nonce = celery.signature( # s_nonce = celery.signature(
'cic_eth.eth.nonce.reserve_nonce', # 'cic_eth.eth.nonce.reserve_nonce',
[ # [
self.chain_spec.asdict(), # self.chain_spec.asdict(),
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_tokens = celery.signature( # s_tokens = celery.signature(
'cic_eth.eth.erc20.resolve_tokens_by_symbol', # 'cic_eth.eth.erc20.resolve_tokens_by_symbol',
[ # [
self.chain_spec.asdict(), # self.chain_spec.asdict(),
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_convert = celery.signature( # s_convert = celery.signature(
'cic_eth.eth.bancor.convert_with_default_reserve', # 'cic_eth.eth.bancor.convert_with_default_reserve',
[ # [
from_address, # from_address,
target_return, # target_return,
minimum_return, # minimum_return,
from_address, # from_address,
self.chain_spec.asdict(), # self.chain_spec.asdict(),
], # ],
queue=self.queue, # queue=self.queue,
) # )
s_nonce.link(s_tokens) # s_nonce.link(s_tokens)
s_check.link(s_nonce) # s_check.link(s_nonce)
if self.callback_param != None: # if self.callback_param != None:
s_convert.link(self.callback_success) # s_convert.link(self.callback_success)
s_tokens.link(s_convert).on_error(self.callback_error) # s_tokens.link(s_convert).on_error(self.callback_error)
else: # else:
s_tokens.link(s_convert) # s_tokens.link(s_convert)
#
t = s_check.apply_async(queue=self.queue) # t = s_check.apply_async(queue=self.queue)
return t # return t
def transfer(self, from_address, to_address, value, token_symbol): def transfer(self, from_address, to_address, value, token_symbol):

8
apps/cic-eth/cic_eth/db/util.py
View File

@ -1,8 +0,0 @@
import math
def num_serialize(n):
if n == 0:
return b'\x00'
binlog = math.log2(n)
bytelength = int(binlog / 8 + 1)
return n.to_bytes(bytelength, 'big')

38
apps/cic-eth/cic_eth/eth/gas.py
View File

@ -57,10 +57,12 @@ celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
MAXIMUM_FEE_UNITS = 8000000
class MaxGasOracle: class MaxGasOracle:
def gas(code=None): def gas(code=None):
return 8000000 return MAXIMUM_FEE_UNITS
def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@ -150,7 +152,7 @@ def cache_gas_data(
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task) @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=None): def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
"""Check the gas level of the sender address of a transaction. """Check the gas level of the sender address of a transaction.
If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser. If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@ -170,24 +172,30 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
:return: Signed raw transaction data list :return: Signed raw transaction data list
:rtype: param txs, unchanged :rtype: param txs, unchanged
""" """
if len(txs) == 0: chain_spec = ChainSpec.from_dict(chain_spec_dict)
for i in range(len(tx_hashes)): logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
o = get_tx(tx_hashes[i])
txs.append(o['signed_tx']) addresspass = None
if address == None: if len(txs) == 0:
address = o['address'] addresspass = []
for i in range(len(tx_hashes)):
o = get_tx(chain_spec_dict, tx_hashes[i])
txs.append(o['signed_tx'])
logg.debug('sender {}'.format(o))
tx = unpack(bytes.fromhex(strip_0x(o['signed_tx'])), chain_spec)
if address == None:
address = tx['from']
elif address != tx['from']:
raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
addresspass.append(address)
#if not web3.Web3.isChecksumAddress(address):
if not is_checksum_address(address): if not is_checksum_address(address):
raise ValueError('invalid address {}'.format(address)) raise ValueError('invalid address {}'.format(address))
chain_spec = ChainSpec.from_dict(chain_spec_dict)
queue = self.request.delivery_info.get('routing_key') queue = self.request.delivery_info.get('routing_key')
conn = RPCConnection.connect(chain_spec) conn = RPCConnection.connect(chain_spec)
# TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
gas_balance = 0 gas_balance = 0
try: try:
o = balance(address) o = balance(address)
@ -198,6 +206,9 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
conn.disconnect() conn.disconnect()
raise EthError('gas_balance call for {}: {}'.format(address, e)) raise EthError('gas_balance call for {}: {}'.format(address, e))
if gas_required == None:
gas_required = MAXIMUM_FEE_UNITS
logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required)) logg.debug('address {} has gas {} needs {}'.format(address, gas_balance, gas_required))
session = SessionBase.create_session() session = SessionBase.create_session()
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
@ -268,7 +279,8 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
queue=queue, queue=queue,
) )
ready_tasks.append(s) ready_tasks.append(s)
celery.group(ready_tasks)() t = celery.group(ready_tasks)()
logg.debug('group {}'.format(t))
return txs return txs

12
apps/cic-eth/cic_eth/eth/tx.py
View File

@ -21,6 +21,7 @@ from chainqueue.db.models.tx import Otx
from chainqueue.db.models.tx import TxCache from chainqueue.db.models.tx import TxCache
from chainqueue.db.enum import StatusBits from chainqueue.db.enum import StatusBits
from chainqueue.error import NotLocalTxError from chainqueue.error import NotLocalTxError
from potaahto.symbols import snake_and_camel
# local imports # local imports
from cic_eth.db import SessionBase from cic_eth.db import SessionBase
@ -58,6 +59,9 @@ def hashes_to_txs(self, tx_hashes):
if len(tx_hashes) == 0: if len(tx_hashes) == 0:
raise ValueError('no transaction to send') raise ValueError('no transaction to send')
for i in range(len(tx_hashes)):
tx_hashes[i] = strip_0x(tx_hashes[i])
queue = self.request.delivery_info['routing_key'] queue = self.request.delivery_info['routing_key']
session = SessionBase.create_session() session = SessionBase.create_session()
@ -148,7 +152,7 @@ def send(self, txs, chain_spec_dict):
@celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task) @celery_app.task(bind=True, throws=(NotFoundEthException,), base=CriticalWeb3Task)
def sync_tx(self, tx_hash_hex, chain_spec_dict): def sync_tx(self, tx_hash_hex, chain_spec_dict):
"""Force update of network status of a simgle transaction """Force update of network status of a single transaction
:param tx_hash_hex: Transaction hash :param tx_hash_hex: Transaction hash
:type tx_hash_hex: str, 0x-hex :type tx_hash_hex: str, 0x-hex
@ -173,12 +177,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
# TODO: apply receipt in tx object to validate and normalize input # TODO: apply receipt in tx object to validate and normalize input
if rcpt != None: if rcpt != None:
rcpt = snake_and_camel(rcpt)
success = rcpt['status'] == 1 success = rcpt['status'] == 1
logg.debug('sync tx {} mined block {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], success)) logg.debug('sync tx {} mined block {} tx index {} success {}'.format(tx_hash_hex, rcpt['blockNumber'], rcpt['transactionIndex'], success))
s = celery.signature( s = celery.signature(
'cic_eth.queue.state.set_final', 'cic_eth.queue.state.set_final',
[ [
chain_spec_dict,
tx_hash_hex, tx_hash_hex,
rcpt['blockNumber'], rcpt['blockNumber'],
rcpt['transactionIndex'], rcpt['transactionIndex'],
@ -186,12 +192,14 @@ def sync_tx(self, tx_hash_hex, chain_spec_dict):
], ],
queue=queue, queue=queue,
) )
# TODO: it's not entirely clear how we can reliable determine that its in mempool without explicitly checking
else: else:
logg.debug('sync tx {} mempool'.format(tx_hash_hex)) logg.debug('sync tx {} mempool'.format(tx_hash_hex))
s = celery.signature( s = celery.signature(
'cic_eth.queue.state.set_sent', 'cic_eth.queue.state.set_sent',
[ [
chain_spec_dict,
tx_hash_hex, tx_hash_hex,
], ],
queue=queue, queue=queue,

2
apps/cic-eth/cic_eth/queue/query.py
View File

@ -37,7 +37,7 @@ def get_tx_cache(chain_spec_dict, tx_hash):
def get_tx(chain_spec_dict, tx_hash): def get_tx(chain_spec_dict, tx_hash):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
session = SessionBase.create_session() session = SessionBase.create_session()
r = chainqueue.query.get_tx(chain_spec, tx_hash) r = chainqueue.query.get_tx(chain_spec, tx_hash, session=session)
session.close() session.close()
return r return r

13
apps/cic-eth/cic_eth/registry.py
View File

@ -5,29 +5,30 @@ import logging
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
from cic_eth_registry.lookup.tokenindex import TokenIndexLookup from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
from chainlib.eth.constant import ZERO_ADDRESS
logg = logging.getLogger() logg = logging.getLogger()
def connect_token_registry(rpc, chain_spec): def connect_token_registry(rpc, chain_spec, sender_address=ZERO_ADDRESS):
registry = CICRegistry(chain_spec, rpc) registry = CICRegistry(chain_spec, rpc)
token_registry_address = registry.by_name('TokenRegistry') token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
logg.debug('using token registry address {}'.format(token_registry_address)) logg.debug('using token registry address {}'.format(token_registry_address))
lookup = TokenIndexLookup(chain_spec, token_registry_address) lookup = TokenIndexLookup(chain_spec, token_registry_address)
CICRegistry.add_lookup(lookup) CICRegistry.add_lookup(lookup)
def connect_declarator(rpc, chain_spec, trusted_addresses): def connect_declarator(rpc, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
registry = CICRegistry(chain_spec, rpc) registry = CICRegistry(chain_spec, rpc)
declarator_address = registry.by_name('AddressDeclarator') declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
logg.debug('using declarator address {}'.format(declarator_address)) logg.debug('using declarator address {}'.format(declarator_address))
lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
CICRegistry.add_lookup(lookup) CICRegistry.add_lookup(lookup)
def connect(rpc, chain_spec, registry_address): def connect(rpc, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
CICRegistry.address = registry_address CICRegistry.address = registry_address
registry = CICRegistry(chain_spec, rpc) registry = CICRegistry(chain_spec, rpc)
registry_address = registry.by_name('ContractRegistry') registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
return registry return registry

9
apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py
View File

@ -17,7 +17,8 @@ from cic_eth.db.models.base import SessionBase
from cic_eth.eth.gas import create_check_gas_task from cic_eth.eth.gas import create_check_gas_task
from .base import SyncFilter from .base import SyncFilter
logg = logging.getLogger().getChild(__name__) #logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger()
class GasFilter(SyncFilter): class GasFilter(SyncFilter):
@ -27,11 +28,11 @@ class GasFilter(SyncFilter):
self.chain_spec = chain_spec self.chain_spec = chain_spec
def filter(self, conn, block, tx, session): def filter(self, conn, block, tx, db_session):
if tx.value > 0: if tx.value > 0:
tx_hash_hex = add_0x(tx.hash) tx_hash_hex = add_0x(tx.hash)
logg.debug('gas refill tx {}'.format(tx_hash_hex)) logg.debug('gas refill tx {}'.format(tx_hash_hex))
session = SessionBase.bind_session(session) session = SessionBase.bind_session(db_session)
q = session.query(TxCache.recipient) q = session.query(TxCache.recipient)
q = q.join(Otx) q = q.join(Otx)
q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex)) q = q.filter(Otx.tx_hash==strip_0x(tx_hash_hex))
@ -56,7 +57,7 @@ class GasFilter(SyncFilter):
tx_hashes_hex=list(txs.keys()), tx_hashes_hex=list(txs.keys()),
queue=self.queue, queue=self.queue,
) )
s.apply_async() return s.apply_async()
def __str__(self): def __str__(self):

3
apps/cic-eth/cic_eth/runnable/daemons/filters/register.py
View File

@ -50,7 +50,8 @@ class RegistrationFilter(SyncFilter):
queue=self.queue, queue=self.queue,
) )
s_nonce.link(s_gift) s_nonce.link(s_gift)
s_nonce.apply_async() t = s_nonce.apply_async()
return t
def __str__(self): def __str__(self):

16
apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py
View File

@ -32,7 +32,7 @@ class TransferAuthFilter(SyncFilter):
self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address) self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None): def filter(self, conn, block, tx, db_session): #rcpt, chain_str, session=None):
if tx.payload == None: if tx.payload == None:
logg.debug('no payload') logg.debug('no payload')
@ -45,16 +45,17 @@ class TransferAuthFilter(SyncFilter):
return False return False
recipient = tx.inputs[0] recipient = tx.inputs[0]
if recipient != self.transfer_request_contract.address(): #if recipient != self.transfer_request_contract.address():
if recipient != self.transfer_request_contract:
logg.debug('not our transfer auth contract address {}'.format(recipient)) logg.debug('not our transfer auth contract address {}'.format(recipient))
return False return False
r = TransferAuthorization.parse_create_request_request(tx.payload) r = TransferAuthorization.parse_create_request_request(tx.payload)
sender = abi_decode_single(ABIContractType.ADDRESS, r[0]) sender = r[0]
recipient = abi_decode_single(ABIContractType.ADDRESS, r[1]) recipient = r[1]
token = abi_decode_single(ABIContractType.ADDRESS, r[2]) token = r[2]
value = abi_decode_single(ABIContractType.UINT256, r[3]) value = r[3]
token_data = { token_data = {
'address': token, 'address': token,
@ -64,6 +65,7 @@ class TransferAuthFilter(SyncFilter):
'cic_eth.eth.nonce.reserve_nonce', 'cic_eth.eth.nonce.reserve_nonce',
[ [
[token_data], [token_data],
self.chain_spec.asdict(),
sender, sender,
], ],
queue=self.queue, queue=self.queue,
@ -80,7 +82,7 @@ class TransferAuthFilter(SyncFilter):
) )
s_nonce.link(s_approve) s_nonce.link(s_approve)
t = s_nonce.apply_async() t = s_nonce.apply_async()
return True return t
def __str__(self): def __str__(self):

4
apps/cic-eth/cic_eth/stat.py
View File

@ -20,7 +20,11 @@ def init_chain_stat(rpc, block_start=0):
if block_start == 0: if block_start == 0:
o = block_latest() o = block_latest()
r = rpc.do(o) r = rpc.do(o)
try:
block_start = int(r, 16) block_start = int(r, 16)
except TypeError:
block_start = int(r)
logg.debug('blockstart {}'.format(block_start))
for i in range(BLOCK_SAMPLES): for i in range(BLOCK_SAMPLES):
o = block_by_number(block_start-10+i) o = block_by_number(block_start-10+i)

12
apps/cic-eth/cic_eth/task.py
View File

@ -20,7 +20,8 @@ import liveness.linux
from cic_eth.error import SeppukuError from cic_eth.error import SeppukuError
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
logg = logging.getLogger().getChild(__name__) #logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger()
celery_app = celery.current_app celery_app = celery.current_app
@ -118,12 +119,13 @@ def registry():
return CICRegistry.address return CICRegistry.address
@celery_app.task() @celery_app.task(bind=True, base=BaseTask)
def registry_address_lookup(chain_spec_dict, address, connection_tag='default'): def registry_address_lookup(self, chain_spec_dict, address, connection_tag='default'):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
conn = RPCConnection.connect(chain_spec, tag=connection_tag) conn = RPCConnection.connect(chain_spec, tag=connection_tag)
registry = CICRegistry(chain_spec, conn) registry = CICRegistry(chain_spec, conn)
return registry.by_address(address) r = registry.by_address(address, sender_address=self.call_address)
return r
@celery_app.task(throws=(UnknownContractError,)) @celery_app.task(throws=(UnknownContractError,))
@ -131,7 +133,7 @@ def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
chain_spec = ChainSpec.from_dict(chain_spec_dict) chain_spec = ChainSpec.from_dict(chain_spec_dict)
conn = RPCConnection.connect(chain_spec, tag=connection_tag) conn = RPCConnection.connect(chain_spec, tag=connection_tag)
registry = CICRegistry(chain_spec, conn) registry = CICRegistry(chain_spec, conn)
return registry.by_name(name) return registry.by_name(name, sender_address=self.call_address)
@celery_app.task() @celery_app.task()

10
apps/cic-eth/requirements.txt
View File

@ -1,23 +1,23 @@
cic-base~=0.1.2b11 cic-base~=0.1.2b14
celery==4.4.7 celery==4.4.7
crypto-dev-signer~=0.4.14b3 crypto-dev-signer~=0.4.14b3
confini~=0.3.6rc3 confini~=0.3.6rc3
cic-eth-registry~=0.5.5a4 cic-eth-registry~=0.5.5a7
redis==3.5.3 redis==3.5.3
alembic==1.4.2 alembic==1.4.2
websockets==8.1 websockets==8.1
requests~=2.24.0 requests~=2.24.0
eth_accounts_index~=0.0.11a12 eth_accounts_index~=0.0.11a12
erc20-transfer-authorization~=0.3.1a6 erc20-transfer-authorization~=0.3.1a7
uWSGI==2.0.19.1 uWSGI==2.0.19.1
semver==2.13.0 semver==2.13.0
websocket-client==0.57.0 websocket-client==0.57.0
moolb~=0.1.1b2 moolb~=0.1.1b2
eth-address-index~=0.1.1a11 eth-address-index~=0.1.1a11
chainlib~=0.0.3a2 chainlib~=0.0.3rc2
hexathon~=0.0.1a7 hexathon~=0.0.1a7
chainsyncer[sql]~=0.0.2a4 chainsyncer[sql]~=0.0.2a4
chainqueue~=0.0.2a2 chainqueue~=0.0.2b1
sarafu-faucet==0.0.3a3 sarafu-faucet==0.0.3a3
erc20-faucet==0.2.1a4 erc20-faucet==0.2.1a4
coincurve==15.0.0 coincurve==15.0.0

8
apps/cic-eth/tests/check/test_check_db.py Normal file
View File

@ -0,0 +1,8 @@
# local imports
from cic_eth.check.db import health
def test_check_health(
init_database,
):
assert health()

20
apps/cic-eth/tests/check/test_check_gas.py Normal file
View File

@ -0,0 +1,20 @@
# local imports
from cic_eth.check.gas import health
from cic_eth.db.models.role import AccountRole
def test_check_gas(
config,
init_database,
default_chain_spec,
eth_rpc,
custodial_roles,
whoever,
):
config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
assert health(config=config)
AccountRole.set('GAS_GIFTER', whoever, session=init_database)
init_database.commit()
assert not health(config=config)

16
apps/cic-eth/tests/check/test_check_redis.py Normal file
View File

@ -0,0 +1,16 @@
# external imports
import pytest
# local imports
from cic_eth.check.redis import health
def test_check_redis(
config,
have_redis,
):
if have_redis != None:
pytest.skip('cannot connect to redis, skipping test: {}'.format(have_redis))
assert health(unit='test', config=config)

13
apps/cic-eth/tests/check/test_check_signer.py Normal file
View File

@ -0,0 +1,13 @@
# local imports
from cic_eth.check.signer import health
def test_check_signer(
default_chain_spec,
config,
eth_signer,
eth_rpc,
):
config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
assert health(config=config)

28
apps/cic-eth/tests/conftest.py
View File

@ -2,9 +2,11 @@
import os import os
import sys import sys
import logging import logging
import uuid
# external imports # external imports
from eth_erc20 import ERC20 from eth_erc20 import ERC20
import redis
# local imports # local imports
from cic_eth.api import Api from cic_eth.api import Api
@ -19,6 +21,7 @@ from tests.fixtures_config import *
from tests.fixtures_database import * from tests.fixtures_database import *
from tests.fixtures_celery import * from tests.fixtures_celery import *
from tests.fixtures_role import * from tests.fixtures_role import *
from tests.fixtures_contract import *
from chainlib.eth.pytest import * from chainlib.eth.pytest import *
from eth_contract_registry.pytest import * from eth_contract_registry.pytest import *
from cic_eth_registry.pytest.fixtures_contracts import * from cic_eth_registry.pytest.fixtures_contracts import *
@ -55,3 +58,28 @@ def default_token(
): ):
BaseTask.default_token_symbol = foo_token_symbol BaseTask.default_token_symbol = foo_token_symbol
BaseTask.default_token_address = foo_token BaseTask.default_token_address = foo_token
@pytest.fixture(scope='session')
def have_redis(
config,
):
r = redis.Redis(
host = config.get('REDIS_HOST'),
port = config.get('REDIS_PORT'),
db = config.get('REDIS_DB'),
)
k = str(uuid.uuid4())
try:
r.set(k, 'foo')
r.delete(k)
except redis.exceptions.ConnectionError as e:
return e
except TypeError as e:
return e
return None

38
apps/cic-eth/tests/filters/test_filter_bogus.py Normal file
View File

@ -0,0 +1,38 @@
# local imports
from cic_eth.runnable.daemons.filters.gas import GasFilter
from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
from cic_eth.runnable.daemons.filters.callback import CallbackFilter
from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
from cic_eth.runnable.daemons.filters.tx import TxFilter
from cic_eth.runnable.daemons.filters.register import RegistrationFilter
# Hit tx mismatch paths on all filters
def test_filter_bogus(
init_database,
bogus_tx_block,
default_chain_spec,
eth_rpc,
eth_signer,
transfer_auth,
cic_registry,
contract_roles,
register_lookups,
):
fltrs = [
TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER']),
GasFilter(default_chain_spec, queue=None),
TxFilter(default_chain_spec, None),
CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER']),
StragglerFilter(default_chain_spec, None),
RegistrationFilter(default_chain_spec, queue=None),
]
for fltr in fltrs:
r = None
try:
r = fltr.filter(eth_rpc, bogus_tx_block[0], bogus_tx_block[1], db_session=init_database)
except:
pass
assert not r

101
apps/cic-eth/tests/filters/test_gas_filter.py Normal file
View File

@ -0,0 +1,101 @@
# external imports
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import OverrideNonceOracle
from chainqueue.tx import create as queue_create
from chainlib.eth.tx import (
TxFormat,
unpack,
Tx,
)
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainqueue.state import (
set_waitforgas,
)
from hexathon import strip_0x
from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusBits
# local imports
from cic_eth.runnable.daemons.filters.gas import GasFilter
from cic_eth.eth.gas import cache_gas_data
def test_filter_gas(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
celery_session_worker,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
set_waitforgas(default_chain_spec, tx_hash_hex, session=init_database)
init_database.commit()
tx_hash_hex_wait = tx_hash_hex
otx = Otx.load(tx_hash_hex_wait, session=init_database)
assert otx.status & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
43,
agent_roles['BOB'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
fltr = GasFilter(default_chain_spec, queue=None)
o = block_latest()
r = eth_rpc.do(o)
o = block_by_number(r, include_tx=False)
r = eth_rpc.do(o)
block = Block(r)
block.txs = [tx_hash_hex]
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
tx = Tx(tx_src, block=block)
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
t.get_leaf()
assert t.successful()
init_database.commit()
otx = Otx.load(tx_hash_hex_wait, session=init_database)
assert otx.status & StatusBits.QUEUED == StatusBits.QUEUED

78
apps/cic-eth/tests/filters/test_register_filter.py Normal file
View File

@ -0,0 +1,78 @@
# external imports
from eth_accounts_index.registry import AccountRegistry
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.tx import(
receipt,
unpack,
Tx,
)
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from erc20_faucet import Faucet
from hexathon import strip_0x
from chainqueue.query import get_account_tx
# local imports
from cic_eth.runnable.daemons.filters.register import RegistrationFilter
def test_register_filter(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
account_registry,
faucet,
register_lookups,
contract_roles,
agent_roles,
cic_registry,
init_celery_tasks,
celery_session_worker,
caplog,
):
nonce_oracle = RPCNonceOracle(contract_roles['ACCOUNT_REGISTRY_WRITER'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(limit=AccountRegistry.gas(), conn=eth_rpc)
c = AccountRegistry(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, o) = c.add(account_registry, contract_roles['ACCOUNT_REGISTRY_WRITER'], agent_roles['ALICE'])
r = eth_rpc.do(o)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
o = receipt(tx_hash_hex)
rcpt = eth_rpc.do(o)
assert rcpt['status'] == 1
o = block_latest()
r = eth_rpc.do(o)
o = block_by_number(r, include_tx=False)
r = eth_rpc.do(o)
block = Block(r)
block.txs = [tx_hash_hex]
tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
tx = Tx(tx_src, block=block, rcpt=rcpt)
tx.apply_receipt(rcpt)
fltr = RegistrationFilter(default_chain_spec, queue=None)
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
t.get_leaf()
assert t.successful()
gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
ks = list(gift_txs.keys())
assert len(ks) == 1
tx_raw_signed_hex = strip_0x(gift_txs[ks[0]])
tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
gift = Faucet.parse_give_to_request(gift_tx['data'])
assert gift[0] == agent_roles['ALICE']

79
apps/cic-eth/tests/filters/test_transferauth_filter.py Normal file
View File

@ -0,0 +1,79 @@
# external imports
from erc20_transfer_authorization import TransferAuthorization
from eth_erc20 import ERC20
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.tx import (
receipt,
unpack,
Tx,
)
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from hexathon import strip_0x
from chainqueue.query import get_account_tx
# local imports
from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter
def test_filter_transferauth(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
transfer_auth,
foo_token,
celery_session_worker,
register_lookups,
init_custodial,
cic_registry,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
gas_oracle = OverrideGasOracle(limit=200000, conn=eth_rpc)
c = TransferAuthorization(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, o) = c.create_request(transfer_auth, contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], agent_roles['BOB'], foo_token, 1024)
r = rpc.do(o)
tx_signed_raw_bytes = bytes.fromhex(strip_0x(o['params'][0]))
o = receipt(tx_hash_hex)
r = rpc.do(o)
assert r['status'] == 1
o = block_latest()
r = eth_rpc.do(o)
o = block_by_number(r, include_tx=False)
r = eth_rpc.do(o)
block = Block(r)
block.txs = [tx_hash_hex]
#tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
tx = Tx(tx_src, block=block)
fltr = TransferAuthFilter(cic_registry, default_chain_spec, eth_rpc, call_address=contract_roles['CONTRACT_DEPLOYER'])
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
t.get_leaf()
assert t.successful()
approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
ks = list(approve_txs.keys())
assert len(ks) == 1
tx_raw_signed_hex = strip_0x(approve_txs[ks[0]])
tx_raw_signed_bytes = bytes.fromhex(tx_raw_signed_hex)
approve_tx = unpack(tx_raw_signed_bytes, default_chain_spec)
c = ERC20(default_chain_spec)
approve = c.parse_approve_request(approve_tx['data'])
assert approve[0] == agent_roles['BOB']

3
apps/cic-eth/tests/filters/test_tx_filter.py
View File

@ -23,7 +23,6 @@ from chainqueue.state import (
set_ready, set_ready,
set_sent, set_sent,
) )
from hexathon import strip_0x from hexathon import strip_0x
# local imports # local imports
@ -31,7 +30,7 @@ from cic_eth.runnable.daemons.filters.tx import TxFilter
from cic_eth.eth.gas import cache_gas_data from cic_eth.eth.gas import cache_gas_data
def test_tx( def test_filter_tx(
default_chain_spec, default_chain_spec,
init_database, init_database,
eth_rpc, eth_rpc,

8
apps/cic-eth/tests/fixtures_celery.py
View File

@ -22,7 +22,6 @@ def init_celery_tasks(
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def celery_includes(): def celery_includes():
return [ return [
# 'cic_eth.eth.bancor',
'cic_eth.eth.erc20', 'cic_eth.eth.erc20',
'cic_eth.eth.tx', 'cic_eth.eth.tx',
'cic_eth.ext.tx', 'cic_eth.ext.tx',
@ -47,8 +46,8 @@ def celery_config():
bq = tempfile.mkdtemp() bq = tempfile.mkdtemp()
bp = tempfile.mkdtemp() bp = tempfile.mkdtemp()
rq = tempfile.mkdtemp() rq = tempfile.mkdtemp()
logg.debug('celery broker queue {} processed {}'.format(bq, bp)) logg.debug('celery broker session queue {} processed {}'.format(bq, bp))
logg.debug('celery backend store {}'.format(rq)) logg.debug('celery backend session store {}'.format(rq))
yield { yield {
'broker_url': 'filesystem://', 'broker_url': 'filesystem://',
'broker_transport_options': { 'broker_transport_options': {
@ -58,12 +57,11 @@ def celery_config():
}, },
'result_backend': 'file://{}'.format(rq), 'result_backend': 'file://{}'.format(rq),
} }
logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq)) logg.debug('cleaning up celery session filesystem backend files {} {} {}'.format(bq, bp, rq))
shutil.rmtree(bq) shutil.rmtree(bq)
shutil.rmtree(bp) shutil.rmtree(bp)
shutil.rmtree(rq) shutil.rmtree(rq)
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def celery_worker_parameters(): def celery_worker_parameters():
return { return {

77
apps/cic-eth/tests/fixtures_contract.py Normal file
View File

@ -0,0 +1,77 @@
# standard imports
import os
# external imports
import pytest
from chainlib.eth.contract import (
ABIContractEncoder,
ABIContractType,
)
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
TxFactory,
TxFormat,
unpack,
Tx,
)
from hexathon import strip_0x
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(script_dir)
@pytest.fixture(scope='function')
def bogus_tx_block(
default_chain_spec,
eth_rpc,
eth_signer,
contract_roles,
):
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(limit=2000000, conn=eth_rpc)
f = open(os.path.join(script_dir, 'testdata', 'Bogus.bin'), 'r')
bytecode = f.read()
f.close()
c = TxFactory(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
tx = c.template(contract_roles['CONTRACT_DEPLOYER'], None, use_nonce=True)
tx = c.set_code(tx, bytecode)
(tx_hash_hex, o) = c.build(tx)
r = eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
contract_address = r['contract_address']
enc = ABIContractEncoder()
enc.method('poke')
data = enc.get()
tx = c.template(contract_roles['CONTRACT_DEPLOYER'], contract_address, use_nonce=True)
tx = c.set_code(tx, data)
(tx_hash_hex, o) = c.finalize(tx, TxFormat.JSONRPC)
r = eth_rpc.do(o)
tx_signed_raw_hex = strip_0x(o['params'][0])
o = block_latest()
r = eth_rpc.do(o)
o = block_by_number(r, include_tx=False)
r = eth_rpc.do(o)
block = Block(r)
block.txs = [tx_hash_hex]
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
tx = Tx(tx_src, block=block)
return (block, tx)

351
apps/cic-eth/tests/task/api/test_admin.py
View File

@ -9,8 +9,14 @@ from chainlib.eth.tx import (
unpack, unpack,
TxFormat, TxFormat,
) )
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import (
from chainlib.eth.gas import Gas RPCNonceOracle,
OverrideNonceOracle,
)
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
from hexathon import ( from hexathon import (
strip_0x, strip_0x,
@ -23,7 +29,15 @@ from chainqueue.db.enum import (
StatusBits, StatusBits,
status_str, status_str,
) )
from chainqueue.query import get_tx from chainqueue.state import (
set_fubar,
set_ready,
set_reserved,
)
from chainqueue.query import (
get_tx,
get_nonce_tx_cache,
)
# local imports # local imports
from cic_eth.api import AdminApi from cic_eth.api import AdminApi
@ -36,150 +50,6 @@ from cic_eth.queue.tx import queue_create
logg = logging.getLogger() logg = logging.getLogger()
#def test_resend_inplace(
# default_chain_spec,
# init_database,
# init_w3,
# celery_session_worker,
# ):
#
# chain_str = str(default_chain_spec)
# c = RpcClient(default_chain_spec)
#
# sigs = []
#
# gas_provider = c.gas_provider()
#
# s_nonce = celery.signature(
# 'cic_eth.eth.nonce.reserve_nonce',
# [
# init_w3.eth.accounts[0],
# gas_provider,
# ],
# queue=None,
# )
# s_refill = celery.signature(
# 'cic_eth.eth.gas.refill_gas',
# [
# chain_str,
# ],
# queue=None,
# )
# s_nonce.link(s_refill)
# t = s_nonce.apply_async()
# t.get()
# for r in t.collect():
# pass
# assert t.successful()
#
# q = init_database.query(Otx)
# q = q.join(TxCache)
# q = q.filter(TxCache.recipient==init_w3.eth.accounts[0])
# o = q.first()
# tx_raw = o.signed_tx
#
# tx_dict = unpack(bytes.fromhex(tx_raw), default_chain_spec)
# gas_price_before = tx_dict['gasPrice']
#
# s = celery.signature(
# 'cic_eth.admin.ctrl.lock_send',
# [
# chain_str,
# init_w3.eth.accounts[0],
# ],
# queue=None,
# )
# t = s.apply_async()
# t.get()
# assert t.successful()
#
# api = AdminApi(c, queue=None)
# t = api.resend(tx_dict['hash'], chain_str, unlock=True)
# t.get()
# i = 0
# tx_hash_new_hex = None
# for r in t.collect():
# tx_hash_new_hex = r[1]
# assert t.successful()
#
# tx_raw_new = get_tx(tx_hash_new_hex)
# logg.debug('get {}'.format(tx_raw_new))
# tx_dict_new = unpack(bytes.fromhex(tx_raw_new['signed_tx']), default_chain_spec)
# assert tx_hash_new_hex != tx_dict['hash']
# assert tx_dict_new['gasPrice'] > gas_price_before
#
# tx_dict_after = get_tx(tx_dict['hash'])
#
# logg.debug('logggg {}'.format(status_str(tx_dict_after['status'])))
# assert tx_dict_after['status'] & StatusBits.MANUAL
#def test_check_fix_nonce(
# default_chain_spec,
# init_database,
# init_eth_account_roles,
# init_w3,
# eth_empty_accounts,
# celery_session_worker,
# ):
#
# chain_str = str(default_chain_spec)
#
# sigs = []
# for i in range(5):
# s = celery.signature(
# 'cic_eth.eth.gas.refill_gas',
# [
# eth_empty_accounts[i],
# chain_str,
# ],
# queue=None,
# )
# sigs.append(s)
#
# t = celery.group(sigs)()
# txs = t.get()
# assert t.successful()
#
# tx_hash = web3.Web3.keccak(hexstr=txs[2])
# c = RpcClient(default_chain_spec)
# api = AdminApi(c, queue=None)
# address = init_eth_account_roles['eth_account_gas_provider']
# nonce_spec = api.check_nonce(address)
# assert nonce_spec['nonce']['network'] == 0
# assert nonce_spec['nonce']['queue'] == 4
# assert nonce_spec['nonce']['blocking'] == None
#
# s_set = celery.signature(
# 'cic_eth.queue.tx.set_rejected',
# [
# tx_hash.hex(),
# ],
# queue=None,
# )
# t = s_set.apply_async()
# t.get()
# t.collect()
# assert t.successful()
#
#
# nonce_spec = api.check_nonce(address)
# assert nonce_spec['nonce']['blocking'] == 2
# assert nonce_spec['tx']['blocking'] == tx_hash.hex()
#
# t = api.fix_nonce(address, nonce_spec['nonce']['blocking'])
# t.get()
# t.collect()
# assert t.successful()
#
# for tx in txs[3:]:
# tx_hash = web3.Web3.keccak(hexstr=tx)
# tx_dict = get_tx(tx_hash.hex())
# assert tx_dict['status'] == StatusEnum.OVERRIDDEN
#
#
def test_have_account( def test_have_account(
default_chain_spec, default_chain_spec,
custodial_roles, custodial_roles,
@ -243,28 +113,6 @@ def test_tag_account(
assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL'] assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL']
#def test_ready(
# init_database,
# agent_roles,
# eth_rpc,
# ):
#
# api = AdminApi(eth_rpc)
#
# with pytest.raises(InitializationError):
# api.ready()
#
# bogus_account = os.urandom(20)
# bogus_account_hex = '0x' + bogus_account.hex()
#
# api.tag_account('ETH_GAS_PROVIDER_ADDRESS', web3.Web3.toChecksumAddress(bogus_account_hex))
# with pytest.raises(KeyError):
# api.ready()
#
# api.tag_account('ETH_GAS_PROVIDER_ADDRESS', eth_empty_accounts[0])
# api.ready()
def test_tx( def test_tx(
default_chain_spec, default_chain_spec,
cic_registry, cic_registry,
@ -286,3 +134,168 @@ def test_tx(
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT']) api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex) tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
logg.warning('code missing to verify tx contents {}'.format(tx)) logg.warning('code missing to verify tx contents {}'.format(tx))
def test_check_nonce_gap(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
caplog,
):
# NOTE: this only works as long as agents roles start at nonce 0
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
tx_hashes = []
txs = []
j = 0
for i in range(10):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
if i == 3:
j = 1
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], i+1)
queue_create(
default_chain_spec,
i+j,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes.append(tx_hash_hex)
txs.append(tx_signed_raw_hex)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
assert r['nonce']['blocking'] == 4
assert r['tx']['blocking'] == tx_hashes[3] # one less because there is a gap
def test_check_nonce_localfail(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
caplog,
):
# NOTE: this only works as long as agents roles start at nonce 0
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
tx_hashes = []
txs = []
j = 0
for i in range(10):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
i,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes.append(tx_hash_hex)
txs.append(tx_signed_raw_hex)
set_ready(default_chain_spec, tx_hashes[4], session=init_database)
set_reserved(default_chain_spec, tx_hashes[4], session=init_database)
set_fubar(default_chain_spec, tx_hashes[4], session=init_database)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
r = api.check_nonce(default_chain_spec, agent_roles['ALICE'])
assert r['nonce']['blocking'] == 4
assert r['tx']['blocking'] == tx_hashes[4]
def test_fix_nonce(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
init_celery_tasks,
caplog,
):
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 0)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
tx_hashes = []
txs = []
for i in range(10):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
i,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes.append(tx_hash_hex)
txs.append(tx_signed_raw_hex)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['DEFAULT'])
t = api.fix_nonce(default_chain_spec, agent_roles['ALICE'], 3)
r = t.get_leaf()
assert t.successful()
init_database.commit()
txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
ks = txs.keys()
assert len(ks) == 2
for k in ks:
hsh = add_0x(k)
otx = Otx.load(hsh, session=init_database)
init_database.refresh(otx)
logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))
if add_0x(k) == tx_hashes[3]:
assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
else:
assert otx.status == 1

373
apps/cic-eth/tests/task/api/test_admin_noncritical.py Normal file
View File

@ -0,0 +1,373 @@
# standard imports
import logging
import io
import json
# external imports
import pytest
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import (
nonce,
OverrideNonceOracle,
RPCNonceOracle,
)
from chainqueue.tx import create as queue_create
from chainlib.eth.tx import (
raw,
receipt,
TxFormat,
Tx,
)
from chainlib.eth.block import block_latest
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainqueue.state import (
set_reserved,
set_sent,
set_ready,
)
from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusBits
from chainqueue.query import get_nonce_tx_cache
from eth_erc20 import ERC20
from cic_eth_registry import CICRegistry
# local imports
from cic_eth.api.api_admin import AdminApi
from cic_eth.eth.gas import cache_gas_data
from cic_eth.eth.erc20 import cache_transfer_data
logg = logging.getLogger()
def test_admin_api_tx(
default_chain_spec,
init_database,
init_celery_tasks,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
custodial_roles,
celery_session_worker,
foo_token,
address_declarator,
cic_registry,
register_tokens,
register_lookups,
caplog,
):
nonce_oracle = RPCNonceOracle(custodial_roles['FOO_TOKEN_GIFTER'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(limit=100000, conn=eth_rpc)
o = nonce(custodial_roles['FOO_TOKEN_GIFTER'])
r = eth_rpc.do(o)
gifter_nonce = int(r, 16)
#c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
gifter_nonce, # will only work if agent starts at 0
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_transfer_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
init_database.commit()
o = raw(tx_signed_raw_hex)
eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
assert r['status'] == 1
set_ready(default_chain_spec, tx_hash_hex, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
set_sent(default_chain_spec, tx_hash_hex, session=init_database)
# lookup by transaction hash, without registry
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex)
logg.debug('deployed {}'.format(contract_roles['CONTRACT_DEPLOYER']))
assert tx['tx_hash'] == tx_hash_hex
# lookup by RLP transaction, without registry
tx = api.tx(default_chain_spec, tx_raw=tx_signed_raw_hex)
assert tx['tx_hash'] == tx_hash_hex
# lookup by transaction hash, with registry
registry = CICRegistry(default_chain_spec, eth_rpc)
tx = api.tx(default_chain_spec, tx_hash=tx_hash_hex, registry=registry)
assert tx['tx_hash'] == tx_hash_hex
# lookup by transaction hash, using writer
buf = io.StringIO()
api.tx(default_chain_spec, tx_hash=tx_hash_hex, renderer=json.dumps, w=buf)
tx = json.loads(buf.getvalue())
assert tx['tx_hash'] == tx_hash_hex
def test_admin_api_account(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
caplog,
):
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
tx_hashes_alice = []
txs_alice = []
for i in range(3):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42+i,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes_alice.append(tx_hash_hex)
txs_alice.append(tx_signed_raw_hex)
init_database.commit()
nonce_oracle = OverrideNonceOracle(agent_roles['BOB'], 13)
tx_hashes_bob = []
txs_bob = []
for i in range(2):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
13+i,
agent_roles['BOB'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes_bob.append(tx_hash_hex)
txs_bob.append(tx_signed_raw_hex)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
r = api.account(default_chain_spec, agent_roles['ALICE'])
assert len(r) == 5
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=False)
assert len(r) == 2
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
r = api.account(default_chain_spec, agent_roles['ALICE'], include_recipient=False)
assert len(r) == 3
def test_admin_api_account_writer(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
caplog,
):
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
init_database.commit()
buf = io.StringIO()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
api.account(default_chain_spec, agent_roles['ALICE'], renderer=json.dumps, w=buf)
# TODO: improve eval
tx = json.loads(buf.getvalue())
assert tx['tx_hash'] == tx_hash_hex
def test_registry(
eth_rpc,
cic_registry,
contract_roles,
celery_session_worker,
):
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
t = api.registry()
r = t.get_leaf()
assert r == cic_registry
def test_proxy_do(
default_chain_spec,
eth_rpc,
contract_roles,
celery_session_worker,
):
o = block_latest()
r = eth_rpc.do(o)
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
t = api.proxy_do(default_chain_spec, o)
rr = t.get_leaf()
assert r == rr
def test_resend_inplace(
init_database,
default_chain_spec,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
set_ready(default_chain_spec, tx_hash_hex, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
set_sent(default_chain_spec, tx_hash_hex, session=init_database)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
t = api.resend(tx_hash_hex, default_chain_spec, unlock=True)
r = t.get_leaf()
assert t.successful()
otx = Otx.load(tx_hash_hex, session=init_database)
assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
assert len(txs) == 2
@pytest.mark.xfail()
def test_resend_clone(
init_database,
default_chain_spec,
eth_rpc,
eth_signer,
agent_roles,
contract_roles,
celery_session_worker,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
set_ready(default_chain_spec, tx_hash_hex, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
set_sent(default_chain_spec, tx_hash_hex, session=init_database)
init_database.commit()
api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER'])
t = api.resend(tx_hash_hex, default_chain_spec, in_place=False)
r = t.get_leaf()
assert t.successful()
otx = Otx.load(tx_hash_hex, session=init_database)
assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
assert len(txs) == 1
txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
assert len(txs) == 1
otx = Otx.load(txs[0], session=init_database)
assert otx.status == 0

61
apps/cic-eth/tests/task/api/test_app.py
View File

@ -8,11 +8,20 @@ import pytest
import celery import celery
from cic_eth_registry.erc20 import ERC20Token from cic_eth_registry.erc20 import ERC20Token
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from eth_accounts_index import AccountsIndex
from chainlib.eth.tx import (
transaction,
)
from chainqueue.state import (
set_reserved,
)
# local imports # local imports
from cic_eth.api import Api from cic_eth.api import Api
from cic_eth.queue.query import get_tx
logg = logging.getLogger(__name__) #logg = logging.getLogger(__name__)
logg = logging.getLogger()
def test_account_api( def test_account_api(
@ -29,6 +38,47 @@ def test_account_api(
assert t.successful() assert t.successful()
def test_account_api_register(
default_chain_spec,
init_database,
account_registry,
faucet,
custodial_roles,
cic_registry,
register_lookups,
eth_rpc,
celery_session_worker,
):
api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None)
t = api.create_account('')
register_tx_hash = t.get_leaf()
assert t.successful()
set_reserved(default_chain_spec, register_tx_hash, session=init_database)
tx = get_tx(default_chain_spec.asdict(), register_tx_hash)
s = celery.signature(
'cic_eth.eth.tx.send',
[
[tx['signed_tx']],
default_chain_spec.asdict(),
],
queue=None
)
t = s.apply_async()
r = t.get_leaf()
assert t.successful()
o = transaction(register_tx_hash)
tx_src = eth_rpc.do(o)
c = AccountsIndex(default_chain_spec)
address = c.parse_add_request(tx_src['data'])
o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER'])
r = eth_rpc.do(o)
assert c.parse_have(r)
def test_transfer_api( def test_transfer_api(
default_chain_spec, default_chain_spec,
eth_rpc, eth_rpc,
@ -37,16 +87,15 @@ def test_transfer_api(
custodial_roles, custodial_roles,
agent_roles, agent_roles,
cic_registry, cic_registry,
register_tokens, token_registry,
register_lookups, register_lookups,
celery_session_worker, celery_session_worker,
register_tokens,
foo_token_symbol,
): ):
#token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
foo_token_cache = ERC20Token(default_chain_spec, eth_rpc, foo_token)
api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None) api = Api(str(default_chain_spec), callback_param='transfer', callback_task='cic_eth.callbacks.noop.noop', queue=None)
t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1024, foo_token_cache.symbol) t = api.transfer(custodial_roles['FOO_TOKEN_GIFTER'], agent_roles['ALICE'], 1, foo_token_symbol)
t.get_leaf() t.get_leaf()
assert t.successful() assert t.successful()

19
apps/cic-eth/tests/task/api/test_app_noncritical.py Normal file
View File

@ -0,0 +1,19 @@
# local imports
from cic_eth.api.api_task import Api
from cic_eth.task import BaseTask
def test_default_token(
default_chain_spec,
foo_token,
default_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
celery_session_worker,
):
api = Api(str(default_chain_spec), queue=None)
t = api.default_token()
r = t.get_leaf()
assert r['address'] == foo_token

1
apps/cic-eth/tests/task/test_task_account.py
View File

@ -156,6 +156,7 @@ def test_gift(
eth_signer, eth_signer,
init_celery_tasks, init_celery_tasks,
cic_registry, cic_registry,
register_lookups,
celery_session_worker, celery_session_worker,
): ):

88
apps/cic-eth/tests/task/test_task_admin.py Normal file
View File

@ -0,0 +1,88 @@
# standard imports
import logging
# external imports
import celery
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import OverrideNonceOracle
from chainqueue.tx import (
create as queue_create,
)
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainlib.eth.tx import TxFormat
from chainqueue.query import get_nonce_tx_cache
from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusBits
from hexathon import add_0x
# local imports
from cic_eth.admin.nonce import shift_nonce
from cic_eth.eth.gas import cache_gas_data
logg = logging.getLogger()
def test_shift_nonce(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
celery_session_worker,
caplog,
):
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(limit=21000, conn=eth_rpc)
tx_hashes = []
txs = []
for i in range(10):
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42+i,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_hashes.append(tx_hash_hex)
txs.append(tx_signed_raw_hex)
init_database.commit()
s = celery.signature(
'cic_eth.admin.nonce.shift_nonce',
[
default_chain_spec.asdict(),
tx_hashes[3],
],
queue=None
)
t = s.apply_async()
r = t.get_leaf()
assert t.successful()
init_database.commit()
for i in range(42+3, 42+10):
txs = get_nonce_tx_cache(default_chain_spec, i, agent_roles['ALICE'], session=init_database)
for k in txs.keys():
hsh = add_0x(k)
otx = Otx.load(hsh, session=init_database)
logg.debug('checking nonce {} tx {} status {}'.format(i, otx.tx_hash, otx.status))
if add_0x(k) == tx_hashes[i-42]:
assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
else:
assert otx.status == 1

286
apps/cic-eth/tests/task/test_task_gas.py Normal file
View File

@ -0,0 +1,286 @@
# standard imports
import logging
# external imports
import celery
import pytest
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import (
OverrideNonceOracle,
RPCNonceOracle,
)
from chainlib.eth.gas import (
OverrideGasOracle,
Gas,
)
from chainlib.eth.tx import (
unpack,
TxFormat,
)
from chainlib.eth.constant import (
MINIMUM_FEE_UNITS,
MINIMUM_FEE_PRICE,
)
from chainqueue.tx import create as queue_create
from chainqueue.query import get_tx
from chainqueue.db.enum import StatusBits
from chainqueue.state import (
set_ready,
set_reserved,
set_sent,
)
from chainqueue.db.models.otx import Otx
from hexathon import strip_0x
# local imports
from cic_eth.eth.gas import cache_gas_data
from cic_eth.error import OutOfGasError
logg = logging.getLogger()
def test_task_check_gas_ok(
default_chain_spec,
eth_rpc,
eth_signer,
init_database,
agent_roles,
custodial_roles,
celery_session_worker,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
0,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
init_database.commit()
s = celery.signature(
'cic_eth.eth.gas.check_gas',
[
[
tx_hash_hex,
],
default_chain_spec.asdict(),
[],
None,
8000000,
],
queue=None
)
t = s.apply_async()
t.get_leaf()
assert t.successful()
init_database.commit()
tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
def test_task_check_gas_insufficient(
default_chain_spec,
eth_rpc,
eth_signer,
init_database,
agent_roles,
custodial_roles,
celery_session_worker,
whoever,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = OverrideNonceOracle(whoever, 42)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
whoever,
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
init_database.commit()
s = celery.signature(
'cic_eth.eth.gas.check_gas',
[
[
tx_hash_hex,
],
default_chain_spec.asdict(),
[],
None,
None,
],
queue=None
)
t = s.apply_async()
try:
r = t.get_leaf()
except OutOfGasError:
pass
init_database.commit()
tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
assert tx['status'] & StatusBits.GAS_ISSUES == StatusBits.GAS_ISSUES
def test_task_check_gas_low(
default_chain_spec,
eth_rpc,
eth_signer,
init_database,
agent_roles,
custodial_roles,
celery_session_worker,
whoever,
):
gas_oracle = OverrideGasOracle(price=MINIMUM_FEE_PRICE, limit=MINIMUM_FEE_UNITS)
nonce_oracle = RPCNonceOracle(custodial_roles['GAS_GIFTER'], conn=eth_rpc)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, o) = c.create(custodial_roles['GAS_GIFTER'], whoever, 100 * (10 ** 6))
r = eth_rpc.do(o)
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(whoever, conn=eth_rpc)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(whoever, agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
0,
whoever,
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
init_database.commit()
s = celery.signature(
'cic_eth.eth.gas.check_gas',
[
[
tx_hash_hex,
],
default_chain_spec.asdict(),
],
[],
None,
None,
queue=None
)
t = s.apply_async()
t.get_leaf()
assert t.successful()
init_database.commit()
tx = get_tx(default_chain_spec, tx_hash_hex, session=init_database)
assert tx['status'] & StatusBits.QUEUED == StatusBits.QUEUED
@pytest.mark.parametrize(
'_gas_price,_gas_factor',
[
(None, 1.1),
(MINIMUM_FEE_PRICE * 1.1, 0.9),
(None, 1.3),
]
)
def test_task_resend_explicit(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
agent_roles,
custodial_roles,
celery_session_worker,
_gas_price,
_gas_factor,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
0,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec)
init_database.commit()
set_ready(default_chain_spec, tx_hash_hex, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
set_sent(default_chain_spec, tx_hash_hex, session=init_database)
s = celery.signature(
'cic_eth.eth.gas.resend_with_higher_gas',
[
tx_hash_hex,
default_chain_spec.asdict(),
_gas_price,
_gas_factor,
],
queue=None
)
t = s.apply_async()
r = t.get_leaf()
assert t.successful()
q = init_database.query(Otx)
q = q.filter(Otx.tx_hash==strip_0x(r))
otx = q.first()
if otx == None:
raise NotLocalTxError(r)
tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
assert tx_after['gasPrice'] > tx_before['gasPrice']

73
apps/cic-eth/tests/task/test_task_tx.py
View File

@ -4,16 +4,27 @@ import logging
# external imports # external imports
import pytest import pytest
import celery import celery
from chainlib.eth.gas import Gas from chainlib.eth.gas import (
OverrideGasOracle,
Gas,
)
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import ( from chainlib.eth.tx import (
TxFormat, TxFormat,
unpack, unpack,
transaction, transaction,
receipt, receipt,
raw,
) )
from hexathon import strip_0x from hexathon import strip_0x
from chainqueue.db.models.otx import Otx from chainqueue.db.models.otx import Otx
from chainqueue.tx import create as queue_create
from chainqueue.state import (
set_reserved,
set_ready,
set_sent,
)
from chainqueue.db.enum import StatusBits
# local imports # local imports
from cic_eth.queue.tx import register_tx from cic_eth.queue.tx import register_tx
@ -60,15 +71,6 @@ def test_tx_send(
def test_sync_tx( def test_sync_tx(
default_chain_spec,
eth_rpc,
eth_signer,
celery_session_worker,
):
pass
def test_resend_with_higher_gas(
init_database, init_database,
default_chain_spec, default_chain_spec,
eth_rpc, eth_rpc,
@ -77,31 +79,48 @@ def test_resend_with_higher_gas(
celery_session_worker, celery_session_worker,
): ):
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc) nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
(tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 1024, tx_format=TxFormat.RLP_SIGNED) c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
cache_gas_data(tx_hash_hex, tx_signed_raw_hex, default_chain_spec.asdict())
tx_before = unpack(bytes.fromhex(strip_0x(tx_signed_raw_hex)), default_chain_spec) queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex,
tx_signed_raw_hex,
session=init_database,
)
cache_gas_data(
tx_hash_hex,
tx_signed_raw_hex,
default_chain_spec.asdict(),
)
set_ready(default_chain_spec, tx_hash_hex, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex, session=init_database)
set_sent(default_chain_spec, tx_hash_hex, session=init_database)
o = raw(tx_signed_raw_hex)
r = eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
assert r['status'] == 1
s = celery.signature( s = celery.signature(
'cic_eth.eth.gas.resend_with_higher_gas', 'cic_eth.eth.tx.sync_tx',
[ [
tx_hash_hex, tx_hash_hex,
default_chain_spec.asdict(), default_chain_spec.asdict(),
], ],
queue=None, queue=None
) )
t = s.apply_async() t = s.apply_async()
r = t.get_leaf() r = t.get_leaf()
assert t.successful()
q = init_database.query(Otx) init_database.commit()
q = q.filter(Otx.tx_hash==strip_0x(r))
otx = q.first()
if otx == None:
raise NotLocalTxError(r)
tx_after = unpack(bytes.fromhex(strip_0x(otx.signed_tx)), default_chain_spec)
logg.debug('gasprices before {} after {}'.format(tx_before['gasPrice'], tx_after['gasPrice']))
assert tx_after['gasPrice'] > tx_before['gasPrice']
o = Otx.load(tx_hash_hex, session=init_database)
assert o.status & StatusBits.FINAL == StatusBits.FINAL

170
apps/cic-eth/tests/task/test_task_tx_misc.py Normal file
View File

@ -0,0 +1,170 @@
# standard imports
import os
import logging
# external imports
import pytest
import celery
from chainqueue.tx import create as queue_create
from chainlib.eth.nonce import (
RPCNonceOracle,
OverrideNonceOracle,
)
from chainlib.eth.gas import (
OverrideGasOracle,
Gas,
)
from chainlib.eth.tx import (
TxFormat,
unpack,
receipt,
)
from hexathon import (
add_0x,
strip_0x,
)
from chainqueue.state import (
set_reserved,
set_ready,
)
logg = logging.getLogger()
def test_hashes_to_txs(
init_database,
default_chain_spec,
agent_roles,
eth_rpc,
eth_signer,
celery_session_worker,
):
nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 42)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex_one,
tx_signed_raw_hex_one,
session=init_database,
)
#nonce_oracle = OverrideNonceOracle(agent_roles['ALICE'], 43)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
43,
agent_roles['ALICE'],
tx_hash_hex_two,
tx_signed_raw_hex_two,
session=init_database,
)
init_database.commit()
bogus_one = add_0x(os.urandom(32).hex())
bogus_two = add_0x(os.urandom(32).hex())
yarrgs = [
bogus_one,
tx_hash_hex_two,
bogus_two,
tx_hash_hex_one,
]
s = celery.signature(
'cic_eth.eth.tx.hashes_to_txs',
[
yarrgs,
],
queue=None,
)
t = s.apply_async()
r = t.get_leaf()
assert t.successful()
assert len(r) == 2
logg.debug('r {}'.format(r))
txs = [
tx_signed_raw_hex_two,
tx_signed_raw_hex_one,
]
for tx in r:
txs.remove(add_0x(tx))
assert len(txs) == 0
def test_double_send(
init_database,
default_chain_spec,
agent_roles,
eth_rpc,
eth_signer,
celery_session_worker,
):
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], conn=eth_rpc)
gas_oracle = OverrideGasOracle(price=1000000000, limit=21000)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex_one, tx_signed_raw_hex_one) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
42,
agent_roles['ALICE'],
tx_hash_hex_one,
tx_signed_raw_hex_one,
session=init_database,
)
set_ready(default_chain_spec, tx_hash_hex_one, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex_one, session=init_database)
c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex_two, tx_signed_raw_hex_two) = c.create(agent_roles['ALICE'], agent_roles['CAROL'], 200 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
queue_create(
default_chain_spec,
43,
agent_roles['ALICE'],
tx_hash_hex_two,
tx_signed_raw_hex_two,
session=init_database,
)
set_ready(default_chain_spec, tx_hash_hex_two, session=init_database)
set_reserved(default_chain_spec, tx_hash_hex_two, session=init_database)
init_database.commit()
yarrgs = [
tx_signed_raw_hex_one,
tx_signed_raw_hex_two,
]
s = celery.signature(
'cic_eth.eth.tx.send',
[
yarrgs,
default_chain_spec.asdict(),
],
queue=None
)
t = s.apply_async()
r = t.get_leaf()
assert t.successful()
o = receipt(tx_hash_hex_one)
r = eth_rpc.do(o)
assert r['status'] == 1
o = receipt(tx_hash_hex_two)
r = eth_rpc.do(o)
assert r['status'] == 1

1
apps/cic-eth/tests/testdata/Bogus.bin vendored Normal file
View File

@ -0,0 +1 @@
60806040526000805534801561001457600080fd5b50610181806100246000396000f3fe608060405234801561001057600080fd5b5060043610610053576000357c0100000000000000000000000000000000000000000000000000000000900480630dbe671f146100585780631817835814610076575b600080fd5b610060610080565b60405161006d91906100ae565b60405180910390f35b61007e610086565b005b60005481565b600080815480929190610098906100d3565b9190505550565b6100a8816100c9565b82525050565b60006020820190506100c3600083018461009f565b92915050565b6000819050919050565b60006100de826100c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8214156101115761011061011c565b5b600182019050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fdfea264697066735822122034ad8e91e864f030d47f5b93e281869206c1b203c36dc79a209ac9c9c16e577564736f6c63430008040033

10
apps/cic-eth/tests/testdata/Bogus.sol vendored Normal file
View File

@ -0,0 +1,10 @@
pragma solidity ^0.8.0;
contract Bogus {
uint256 public a = 0;
function poke() public {
a++;
}
}

1
apps/cic-eth/tests/unit/ext/test_address.py
View File

@ -19,6 +19,7 @@ def test_translate(
agent_roles, agent_roles,
cic_registry, cic_registry,
init_celery_tasks, init_celery_tasks,
register_lookups,
): ):
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)

22
apps/cic-eth/tests/unit/test_registry_connect.py Normal file
View File

@ -0,0 +1,22 @@
# local imports
from cic_eth.registry import *
def test_registry_connect(
eth_rpc,
default_chain_spec,
address_declarator,
token_registry,
contract_roles,
purge_lookups,
registry,
agent_roles,
):
r = connect(eth_rpc, default_chain_spec, registry, sender_address=contract_roles['CONTRACT_DEPLOYER'])
connect_declarator(eth_rpc, default_chain_spec, [agent_roles['ALICE']], sender_address=contract_roles['CONTRACT_DEPLOYER'])
r.by_name('AddressDeclarator', sender_address=contract_roles['CONTRACT_DEPLOYER'])
connect_token_registry(eth_rpc, default_chain_spec, sender_address=contract_roles['CONTRACT_DEPLOYER'])
r.by_name('TokenRegistry', sender_address=contract_roles['CONTRACT_DEPLOYER'])

18
apps/cic-eth/tests/unit/test_stat.py Normal file
View File

@ -0,0 +1,18 @@
# standard imports
import datetime
# local imports
from cic_eth.stat import init_chain_stat
def test_chain_stat(
eth_rpc,
init_eth_tester,
):
now = int(datetime.datetime.now().timestamp()) + 1
for i in range(11):
init_eth_tester.time_travel(now + (i * 2))
s = init_chain_stat(eth_rpc, block_start=0)
assert s.block_average() == 2

0
apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py → apps/data-seeding/cic_eth/traffic/cmd/cache.py
View File

76
apps/data-seeding/create_import_users.py
View File

@ -2,27 +2,24 @@
# standard imports # standard imports
import json import json
import time
import datetime import datetime
import random import random
import logging import logging
import os import os
import base64
import hashlib import hashlib
import sys
import argparse import argparse
import random import random
# external imports # external imports
import vobject
import celery import celery
from faker import Faker from faker import Faker
from collections import OrderedDict
import confini import confini
from cic_types.models.person import ( from cic_types.models.person import (
Person, Person,
generate_vcard_from_contact_data, generate_vcard_from_contact_data,
get_contact_data_from_vcard, get_contact_data_from_vcard,
) )
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
import phonenumbers import phonenumbers
@ -32,17 +29,21 @@ logg = logging.getLogger()
fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
script_dir = os.path.realpath(os.path.dirname(__file__)) script_dir = os.path.realpath(os.path.dirname(__file__))
#config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') # config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config')) config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record') argparser.add_argument('--tag', type=str, action='append',
argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)') help='Tags to add to record')
argparser.add_argument('--gift-threshold', type=int,
help='If set, users will be funded with additional random balance (in token integer units)')
argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', action='store_true', help='Be more verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose')
argparser.add_argument('--dir', default='out', type=str, help='path to users export dir tree') argparser.add_argument('--dir', default='out', type=str,
argparser.add_argument('user_count', type=int, help='amount of users to generate') help='path to users export dir tree')
argparser.add_argument('user_count', type=int,
help='amount of users to generate')
args = argparser.parse_args() args = argparser.parse_args()
if args.v: if args.v:
@ -60,7 +61,8 @@ dt_then = dt_now - datetime.timedelta(weeks=150)
ts_now = int(dt_now.timestamp()) ts_now = int(dt_now.timestamp())
ts_then = int(dt_then.timestamp()) ts_then = int(dt_then.timestamp())
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) celery_app = celery.Celery(broker=config.get(
'CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
gift_max = args.gift_threshold or 0 gift_max = args.gift_threshold or 0
gift_factor = (10**6) gift_factor = (10**6)
@ -75,7 +77,7 @@ categories = [
"transport", "transport",
"farming/labor", "farming/labor",
"savingsgroup", "savingsgroup",
] ]
phone_idx = [] phone_idx = []
@ -88,6 +90,7 @@ if tags == None or len(tags) == 0:
random.seed() random.seed()
def genPhoneIndex(phone): def genPhoneIndex(phone):
h = hashlib.new('sha256') h = hashlib.new('sha256')
h.update(phone.encode('utf-8')) h.update(phone.encode('utf-8'))
@ -147,10 +150,10 @@ def genDob():
def gen(): def gen():
old_blockchain_address = '0x' + os.urandom(20).hex() old_blockchain_address = '0x' + os.urandom(20).hex()
old_blockchain_checksum_address = to_checksum_address(old_blockchain_address) old_blockchain_checksum_address = to_checksum_address(
old_blockchain_address)
gender = random.choice(['female', 'male', 'other']) gender = random.choice(['female', 'male', 'other'])
phone = genPhone() phone = genPhone()
city = fake.city_name()
v = genPersonal(phone) v = genPersonal(phone)
contact_data = get_contact_data_from_vcard(v) contact_data = get_contact_data_from_vcard(v)
@ -167,11 +170,43 @@ def gen():
], ],
}, },
} }
p.location['area_name'] = city p.products = [fake.random_element(elements=OrderedDict(
[('fruit', 0.25),
('maji', 0.05),
('milk', 0.1),
('teacher', 0.1),
('doctor', 0.05),
('boutique', 0.15),
('recycling', 0.05),
('farmer', 0.05),
('oil', 0.05),
('pastor', 0.1),
('chama', 0.03),
('pastor', 0.01),
('bzrTsuZieaq', 0.01)
]))]
p.location['area_name'] = fake.random_element(elements=OrderedDict(
[('mnarani', 0.05),
('chilumani', 0.1),
('madewani', 0.1),
('kisauni', 0.05),
('bangla', 0.1),
('kabiro', 0.1),
('manyani', 0.05),
('ruben', 0.15),
('makupa', 0.05),
('kingston', 0.05),
('rangala', 0.05),
('homabay', 0.1),
('nakuru', 0.03),
('kajiado', 0.01),
('zurtWicKtily', 0.01)
]))
if random.randint(0, 1): if random.randint(0, 1):
p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude() # fake.local_latitude()
p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude() p.location['latitude'] = (random.random() + 180) - 90
# fake.local_latitude()
p.location['longitude'] = (random.random() + 360) - 180
return (old_blockchain_checksum_address, phone, p) return (old_blockchain_checksum_address, phone, p)
@ -238,8 +273,9 @@ if __name__ == '__main__':
ft.write('{}:{}\n'.format(eth, ','.join(tags))) ft.write('{}:{}\n'.format(eth, ','.join(tags)))
amount = genAmount() amount = genAmount()
fa.write('{},{}\n'.format(eth,amount)) fa.write('{},{}\n'.format(eth, amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone)) logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(
pidx, uid, eth, amount, phone))
i += 1 i += 1