diff --git a/apps/cic-eth/admin_requirements.txt b/apps/cic-eth/admin_requirements.txt index cc62067..b1dbf3b 100644 --- a/apps/cic-eth/admin_requirements.txt +++ b/apps/cic-eth/admin_requirements.txt @@ -1,5 +1,5 @@ SQLAlchemy==1.3.20 -cic-eth-registry~=0.5.6a2 +cic-eth-registry>=0.5.6a2,<0.6.0 hexathon~=0.0.1a7 -chainqueue~=0.0.2b6 -eth-erc20~=0.0.10a3 +chainqueue>=0.0.3a1,<0.1.0 +eth-erc20>=0.0.10a3,<0.1.0 diff --git a/apps/cic-eth/cic_eth/admin/ctrl.py b/apps/cic-eth/cic_eth/admin/ctrl.py index af6dd64..bd53d96 100644 --- a/apps/cic-eth/cic_eth/admin/ctrl.py +++ b/apps/cic-eth/cic_eth/admin/ctrl.py @@ -6,6 +6,11 @@ import logging import celery from chainlib.eth.constant import ZERO_ADDRESS from chainlib.chain import ChainSpec +from hexathon import ( + add_0x, + strip_0x, + uniform as hex_uniform, + ) # local imports from cic_eth.db.enum import LockEnum @@ -19,6 +24,12 @@ from cic_eth.error import LockedError celery_app = celery.current_app logg = logging.getLogger() + +def normalize_address(a): + if a == None: + return None + return add_0x(hex_uniform(strip_0x(a))) + @celery_app.task(base=CriticalSQLAlchemyTask) def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None): """Task wrapper to set arbitrary locks @@ -32,6 +43,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = '::' if chain_spec_dict != None: chain_str = str(ChainSpec.from_dict(chain_spec_dict)) @@ -53,6 +65,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum. :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = '::' if chain_spec_dict != None: chain_str = str(ChainSpec.from_dict(chain_spec_dict)) @@ -72,6 +85,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash) logg.debug('Send locked for {}, flag now {}'.format(address, r)) @@ -89,6 +103,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS): :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.reset(chain_str, LockEnum.SEND, address=address) logg.debug('Send unlocked for {}, flag now {}'.format(address, r)) @@ -106,6 +121,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash) logg.debug('Queue direct locked for {}, flag now {}'.format(address, r)) @@ -123,6 +139,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS): :returns: New lock state for address :rtype: number """ + address = normalize_address(address) chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.reset(chain_str, LockEnum.QUEUE, address=address) logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r)) @@ -131,6 +148,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS): @celery_app.task(base=CriticalSQLAlchemyTask) def check_lock(chained_input, chain_spec_dict, lock_flags, address=None): + address = normalize_address(address) chain_str = '::' if chain_spec_dict != None: chain_str = str(ChainSpec.from_dict(chain_spec_dict)) diff --git a/apps/cic-eth/cic_eth/admin/nonce.py b/apps/cic-eth/cic_eth/admin/nonce.py index a55c892..e51f95f 100644 --- a/apps/cic-eth/cic_eth/admin/nonce.py +++ b/apps/cic-eth/cic_eth/admin/nonce.py @@ -14,7 +14,11 @@ from chainqueue.sql.query import get_tx from chainqueue.sql.state import set_cancel from chainqueue.db.models.otx import Otx from chainqueue.db.models.tx import TxCache -from hexathon import strip_0x +from hexathon import ( + strip_0x, + add_0x, + uniform as hex_uniform, + ) from potaahto.symbols import snake_and_camel # local imports @@ -69,15 +73,17 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session) + query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh q = session.query(Otx) q = q.join(TxCache) - q = q.filter(TxCache.sender==address) + q = q.filter(TxCache.sender==query_address) q = q.filter(Otx.nonce>=nonce+delta) q = q.order_by(Otx.nonce.asc()) otxs = q.all() tx_hashes = [] txs = [] + gas_total = 0 for otx in otxs: tx_raw = bytes.fromhex(strip_0x(otx.signed_tx)) tx_new = unpack(tx_raw, chain_spec) @@ -89,8 +95,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): tx_new['gas_price'] += 1 tx_new['gasPrice'] = tx_new['gas_price'] tx_new['nonce'] -= delta + gas_total += tx_new['gas_price'] * tx_new['gas'] logg.debug('tx_new {}'.format(tx_new)) + logg.debug('gas running total {}'.format(gas_total)) del(tx_new['hash']) del(tx_new['hash_unsigned']) @@ -122,8 +130,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): s = create_check_gas_task( txs, chain_spec, - tx_new['from'], - gas=tx_new['gas'], + #tx_new['from'], + address, + #gas=tx_new['gas'], + gas=gas_total, tx_hashes_hex=tx_hashes, queue=queue, ) @@ -132,7 +142,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): 'cic_eth.admin.ctrl.unlock_send', [ chain_spec.asdict(), - tx_new['from'], + address, + #tx_new['from'], ], queue=queue, ) @@ -140,7 +151,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1): 'cic_eth.admin.ctrl.unlock_queue', [ chain_spec.asdict(), - tx_new['from'], + address, + #tx_new['from'], ], queue=queue, ) diff --git a/apps/cic-eth/cic_eth/api/admin.py b/apps/cic-eth/cic_eth/api/admin.py index 34fc4d6..035499b 100644 --- a/apps/cic-eth/cic_eth/api/admin.py +++ b/apps/cic-eth/cic_eth/api/admin.py @@ -21,6 +21,7 @@ from chainlib.hash import keccak256_hex_to_hex from hexathon import ( strip_0x, add_0x, + uniform as hex_uniform, ) from chainlib.eth.gas import balance from chainqueue.db.enum import ( @@ -307,6 +308,8 @@ class AdminApi: :param address: Ethereum address to return transactions for :type address: str, 0x-hex """ + + address = add_0x(hex_uniform(strip_0x(address))) last_nonce = -1 s = celery.signature( 'cic_eth.queue.query.get_account_tx', diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py index 0921706..6486bd5 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py @@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:30:55.398388 from alembic import op import sqlalchemy as sa -from chainqueue.db.migrations.sqlalchemy import ( +#from chainqueue.db.migrations.sqlalchemy import ( +from chainqueue.db.migrations.default.export import ( chainqueue_upgrade, chainqueue_downgrade, ) diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py index 8f3be98..4859d70 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py @@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:36:44.459603 from alembic import op import sqlalchemy as sa -from chainsyncer.db.migrations.sqlalchemy import ( +#from chainsyncer.db.migrations.sqlalchemy import ( +from chainsyncer.db.migrations.default.export import ( chainsyncer_upgrade, chainsyncer_downgrade, ) diff --git a/apps/cic-eth/cic_eth/eth/account.py b/apps/cic-eth/cic_eth/eth/account.py index 0cf636b..dd6832e 100644 --- a/apps/cic-eth/cic_eth/eth/account.py +++ b/apps/cic-eth/cic_eth/eth/account.py @@ -23,7 +23,7 @@ from chainlib.error import JSONRPCException from eth_accounts_index.registry import AccountRegistry from eth_accounts_index import AccountsIndex from sarafu_faucet import MinterFaucet -from chainqueue.db.models.tx import TxCache +from chainqueue.sql.tx import cache_tx_dict # local import from cic_eth_registry import CICRegistry @@ -300,20 +300,17 @@ def cache_gift_data( session = self.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx['to'], - ZERO_ADDRESS, - ZERO_ADDRESS, - 0, - 0, - session=session, - ) + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': tx['to'], + 'source_token': ZERO_ADDRESS, + 'destination_token': ZERO_ADDRESS, + 'from_value': 0, + 'to_value': 0, + } - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) @@ -342,18 +339,15 @@ def cache_account_data( tx_data = AccountsIndex.parse_add_request(tx['data']) session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx['to'], - ZERO_ADDRESS, - ZERO_ADDRESS, - 0, - 0, - session=session, - ) - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': tx['to'], + 'source_token': ZERO_ADDRESS, + 'destination_token': ZERO_ADDRESS, + 'from_value': 0, + 'to_value': 0, + } + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) diff --git a/apps/cic-eth/cic_eth/eth/bancor.py.bak b/apps/cic-eth/cic_eth/eth/bancor.py.bak deleted file mode 100644 index 57b187b..0000000 --- a/apps/cic-eth/cic_eth/eth/bancor.py.bak +++ /dev/null @@ -1,385 +0,0 @@ -# standard imports -import os -import logging - -# third-party imports -import celery -import web3 -from cic_registry import CICRegistry -from cic_registry.chain import ChainSpec - -# local imports -from cic_eth.db import SessionBase -from cic_eth.db.models.convert import TxConvertTransfer -from cic_eth.db.models.otx import Otx -from cic_eth.db.models.tx import TxCache -from cic_eth.eth.task import sign_and_register_tx -from cic_eth.eth.task import create_check_gas_and_send_task -from cic_eth.eth.token import TokenTxFactory -from cic_eth.eth.factory import TxFactory -from cic_eth.eth.util import unpack_signed_raw_tx -from cic_eth.eth.rpc import RpcClient - -celery_app = celery.current_app -#logg = celery_app.log.get_default_logger() -logg = logging.getLogger() - -contract_function_signatures = { - 'convert': 'f3898a97', - 'convert2': '569706eb', - } - - -class BancorTxFactory(TxFactory): - - """Factory for creating Bancor network transactions. - """ - def convert( - self, - source_token_address, - destination_token_address, - reserve_address, - source_amount, - minimum_return, - chain_spec, - fee_beneficiary='0x0000000000000000000000000000000000000000', - fee_ppm=0, - ): - """Create a BancorNetwork "convert" transaction. - - :param source_token_address: ERC20 contract address for token to convert from - :type source_token_address: str, 0x-hex - :param destination_token_address: ERC20 contract address for token to convert to - :type destination_token_address: str, 0x-hex - :param reserve_address: ERC20 contract address of Common reserve token - :type reserve_address: str, 0x-hex - :param source_amount: Amount of source tokens to convert - :type source_amount: int - :param minimum_return: Minimum amount of destination tokens to accept as result for conversion - :type source_amount: int - :return: Unsigned "convert" transaction in standard Ethereum format - :rtype: dict - """ - network_contract = CICRegistry.get_contract(chain_spec, 'BancorNetwork') - network_gas = network_contract.gas('convert') - tx_convert_buildable = network_contract.contract.functions.convert2( - [ - source_token_address, - source_token_address, - reserve_address, - destination_token_address, - destination_token_address, - ], - source_amount, - minimum_return, - fee_beneficiary, - fee_ppm, - ) - tx_convert = tx_convert_buildable.buildTransaction({ - 'from': self.address, - 'gas': network_gas, - 'gasPrice': self.gas_price, - 'chainId': chain_spec.chain_id(), - 'nonce': self.next_nonce(), - }) - return tx_convert - - -def unpack_convert(data): - f = data[2:10] - if f != contract_function_signatures['convert2']: - raise ValueError('Invalid convert data ({})'.format(f)) - - d = data[10:] - path = d[384:] - source = path[64-40:64] - destination = path[-40:] - - amount = int(d[64:128], 16) - min_return = int(d[128:192], 16) - fee_recipient = d[192:256] - fee = int(d[256:320], 16) - return { - 'amount': amount, - 'min_return': min_return, - 'source_token': web3.Web3.toChecksumAddress('0x' + source), - 'destination_token': web3.Web3.toChecksumAddress('0x' + destination), - 'fee_recipient': fee_recipient, - 'fee': fee, - } - - - -# Kept for historical reference, it unpacks a convert call without fee parameters -#def _unpack_convert_mint(data): -# f = data[2:10] -# if f != contract_function_signatures['convert2']: -# raise ValueError('Invalid convert data ({})'.format(f)) -# -# d = data[10:] -# path = d[256:] -# source = path[64-40:64] -# destination = path[-40:] -# -# amount = int(d[64:128], 16) -# min_return = int(d[128:192], 16) -# return { -# 'amount': amount, -# 'min_return': min_return, -# 'source_token': web3.Web3.toChecksumAddress('0x' + source), -# 'destination_token': web3.Web3.toChecksumAddress('0x' + destination), -# } - - -@celery_app.task(bind=True) -def convert_with_default_reserve(self, tokens, from_address, source_amount, minimum_return, to_address, chain_str): - """Performs a conversion between two liquid tokens using Bancor network. - - :param tokens: Token pair, source and destination respectively - :type tokens: list of str, 0x-hex - :param from_address: Ethereum address of sender - :type from_address: str, 0x-hex - :param source_amount: Amount of source tokens to convert - :type source_amount: int - :param minimum_return: Minimum about of destination tokens to receive - :type minimum_return: int - """ - - chain_spec = ChainSpec.from_chain_str(chain_str) - queue = self.request.delivery_info['routing_key'] - - c = RpcClient(chain_spec, holder_address=from_address) - - cr = CICRegistry.get_contract(chain_spec, 'BancorNetwork') - source_token = CICRegistry.get_address(chain_spec, tokens[0]['address']) - reserve_address = CICRegistry.get_contract(chain_spec, 'BNTToken', 'ERC20').address() - - tx_factory = TokenTxFactory(from_address, c) - - tx_approve_zero = tx_factory.approve(source_token.address(), cr.address(), 0, chain_spec) - (tx_approve_zero_hash_hex, tx_approve_zero_signed_hex) = sign_and_register_tx(tx_approve_zero, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') - - tx_approve = tx_factory.approve(source_token.address(), cr.address(), source_amount, chain_spec) - (tx_approve_hash_hex, tx_approve_signed_hex) = sign_and_register_tx(tx_approve, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') - - tx_factory = BancorTxFactory(from_address, c) - tx_convert = tx_factory.convert( - tokens[0]['address'], - tokens[1]['address'], - reserve_address, - source_amount, - minimum_return, - chain_spec, - ) - (tx_convert_hash_hex, tx_convert_signed_hex) = sign_and_register_tx(tx_convert, chain_str, queue, 'cic_eth.eth.bancor.otx_cache_convert') - - # TODO: consider moving save recipient to async task / chain it before the tx send - if to_address != None: - save_convert_recipient(tx_convert_hash_hex, to_address, chain_str) - - s = create_check_gas_and_send_task( - [tx_approve_zero_signed_hex, tx_approve_signed_hex, tx_convert_signed_hex], - chain_str, - from_address, - tx_approve_zero['gasPrice'] * tx_approve_zero['gas'], - tx_hashes_hex=[tx_approve_hash_hex], - queue=queue, - ) - s.apply_async() - return tx_convert_hash_hex - - -#@celery_app.task() -#def process_approval(tx_hash_hex): -# t = session.query(TxConvertTransfer).query(TxConvertTransfer.approve_tx_hash==tx_hash_hex).first() -# c = session.query(Otx).query(Otx.tx_hash==t.convert_tx_hash) -# gas_limit = 8000000 -# gas_price = GasOracle.gas_price() -# -# # TODO: use celery group instead -# s_queue = celery.signature( -# 'cic_eth.queue.tx.create', -# [ -# nonce, -# c['address'], # TODO: check that this is in fact sender address -# c['tx_hash'], -# c['signed_tx'], -# ] -# ) -# s_queue.apply_async() -# -# s_check_gas = celery.signature( -# 'cic_eth.eth.gas.check_gas', -# [ -# c['address'], -# [c['signed_tx']], -# gas_limit * gas_price, -# ] -# ) -# s_send = celery.signature( -# 'cic_eth.eth.tx.send', -# [], -# ) -# -# s_set_sent = celery.signature( -# 'cic_eth.queue.state.set_sent', -# [False], -# ) -# s_send.link(s_set_sent) -# s_check_gas.link(s_send) -# s_check_gas.apply_async() -# return tx_hash_hex - - - -@celery_app.task() -def save_convert_recipient(convert_hash, recipient_address, chain_str): - """Registers the recipient target for a convert-and-transfer operation. - - :param convert_hash: Transaction hash of convert operation - :type convert_hash: str, 0x-hex - :param recipient_address: Address of consequtive transfer recipient - :type recipient_address: str, 0x-hex - """ - session = SessionBase.create_session() - t = TxConvertTransfer(convert_hash, recipient_address, chain_str) - session.add(t) - session.commit() - session.close() - - -@celery_app.task() -def save_convert_transfer(convert_hash, transfer_hash): - """Registers that the transfer part of a convert-and-transfer operation has been executed. - - :param convert_hash: Transaction hash of convert operation - :type convert_hash: str, 0x-hex - :param convert_hash: Transaction hash of transfer operation - :type convert_hash: str, 0x-hex - :returns: transfer_hash, - :rtype: list, single str, 0x-hex - """ - session = SessionBase.create_session() - t = TxConvertTransfer.get(convert_hash) - t.transfer(transfer_hash) - session.add(t) - session.commit() - session.close() - return [transfer_hash] - - -# TODO: seems unused, consider removing -@celery_app.task() -def resolve_converters_by_tokens(tokens, chain_str): - """Return converters for a list of tokens. - - :param tokens: Token addresses to look up - :type tokens: list of str, 0x-hex - :return: Addresses of matching converters - :rtype: list of str, 0x-hex - """ - chain_spec = ChainSpec.from_chain_str(chain_str) - for t in tokens: - c = CICRegistry.get_contract(chain_spec, 'ConverterRegistry') - fn = c.function('getConvertersByAnchors') - try: - converters = fn([t['address']]).call() - except Exception as e: - raise e - t['converters'] = converters - - return tokens - - -@celery_app.task(bind=True) -def transfer_converted(self, tokens, holder_address, receiver_address, value, tx_convert_hash_hex, chain_str): - """Execute the ERC20 transfer of a convert-and-transfer operation. - - First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument. - - :param tokens: Token addresses - :type tokens: list of str, 0x-hex - :param holder_address: Token holder address - :type holder_address: str, 0x-hex - :param holder_address: Token receiver address - :type holder_address: str, 0x-hex - :param value: Amount of token, in 'wei' - :type value: int - :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument - :return: Transaction hash - :rtype: str, 0x-hex - """ - # we only allow one token, one transfer - if len(tokens) != 1: - raise TokenCountError - - chain_spec = ChainSpec.from_chain_str(chain_str) - - queue = self.request.delivery_info['routing_key'] - - c = RpcClient(chain_spec, holder_address=holder_address) - - # get transaction parameters - gas_price = c.gas_price() - tx_factory = TokenTxFactory(holder_address, c) - - token_address = tokens[0]['address'] - tx_transfer = tx_factory.transfer( - token_address, - receiver_address, - value, - chain_spec, - ) - (tx_transfer_hash_hex, tx_transfer_signed_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.token.otx_cache_transfer') - - # send transaction - logg.info('transfer converted token {} from {} to {} value {} {}'.format(token_address, holder_address, receiver_address, value, tx_transfer_signed_hex)) - s = create_check_gas_and_send_task( - [tx_transfer_signed_hex], - chain_str, - holder_address, - tx_transfer['gasPrice'] * tx_transfer['gas'], - None, - queue, - ) - s_save = celery.signature( - 'cic_eth.eth.bancor.save_convert_transfer', - [ - tx_convert_hash_hex, - tx_transfer_hash_hex, - ], - queue=queue, - ) - s_save.link(s) - s_save.apply_async() - return tx_transfer_hash_hex - - -@celery_app.task() -def otx_cache_convert( - tx_hash_hex, - tx_signed_raw_hex, - chain_str, - ): - - chain_spec = ChainSpec.from_chain_str(chain_str) - tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:]) - tx = unpack(tx_signed_raw_bytes, chain_spec) - tx_data = unpack_convert(tx['data']) - logg.debug('tx data {}'.format(tx_data)) - - session = TxCache.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx['from'], - tx_data['source_token'], - tx_data['destination_token'], - tx_data['amount'], - tx_data['amount'], - ) - session.add(tx_cache) - session.commit() - session.close() - return tx_hash_hex - diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 1f4751b..1a1448a 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -13,9 +13,9 @@ from chainlib.eth.tx import ( from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token from hexathon import strip_0x -from chainqueue.db.models.tx import TxCache from chainqueue.error import NotLocalTxError from eth_erc20 import ERC20 +from chainqueue.sql.tx import cache_tx_dict # local imports from cic_eth.db.models.base import SessionBase @@ -375,19 +375,16 @@ def cache_transfer_data( token_value = tx_data[1] session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - recipient_address, - tx['to'], - tx['to'], - token_value, - token_value, - session=session, - ) - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': recipient_address, + 'source_token': tx['to'], + 'destination_token': tx['to'], + 'from_value': token_value, + 'to_value': token_value, + } + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) @@ -417,19 +414,16 @@ def cache_transfer_from_data( token_value = tx_data[2] session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - recipient_address, - tx['to'], - tx['to'], - token_value, - token_value, - session=session, - ) - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': recipient_address, + 'source_token': tx['to'], + 'destination_token': tx['to'], + 'from_value': token_value, + 'to_value': token_value, + } + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) @@ -458,19 +452,16 @@ def cache_approve_data( token_value = tx_data[1] session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - recipient_address, - tx['to'], - tx['to'], - token_value, - token_value, - session=session, - ) - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': recipient_address, + 'source_token': tx['to'], + 'destination_token': tx['to'], + 'from_value': token_value, + 'to_value': token_value, + } + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 49e89c7..5e9e86e 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -9,6 +9,7 @@ from chainlib.chain import ChainSpec from chainlib.eth.address import is_checksum_address from chainlib.connection import RPCConnection from chainqueue.db.enum import StatusBits +from chainqueue.sql.tx import cache_tx_dict from chainlib.eth.gas import ( balance, price, @@ -133,20 +134,17 @@ def cache_gas_data( session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx['to'], - ZERO_ADDRESS, - ZERO_ADDRESS, - tx['value'], - tx['value'], - session=session, - ) + tx_dict = { + 'hash': tx_hash_hex, + 'from': tx['from'], + 'to': tx['to'], + 'source_token': ZERO_ADDRESS, + 'destination_token': ZERO_ADDRESS, + 'from_value': tx['value'], + 'to_value': tx['value'], + } - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id + (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session) session.close() return (tx_hash_hex, cache_id) diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 6602fe6..5eaf468 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -18,7 +18,6 @@ from hexathon import ( strip_0x, ) from chainqueue.db.models.tx import Otx -from chainqueue.db.models.tx import TxCache from chainqueue.db.enum import StatusBits from chainqueue.error import NotLocalTxError from potaahto.symbols import snake_and_camel diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py index 702fd8d..fa183b4 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py @@ -11,6 +11,7 @@ from chainqueue.db.enum import StatusBits from chainqueue.db.models.tx import TxCache from chainqueue.db.models.otx import Otx from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx +from chainlib.eth.address import to_checksum_address # local imports from cic_eth.db.models.base import SessionBase @@ -47,12 +48,13 @@ class GasFilter(SyncFilter): SessionBase.release_session(session) + address = to_checksum_address(r[0]) logg.info('resuming gas-in-waiting txs for {}'.format(r[0])) if len(txs) > 0: s = create_check_gas_task( list(txs.values()), self.chain_spec, - r[0], + address, 0, tx_hashes_hex=list(txs.keys()), queue=self.queue, diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 036c79b..039b9ba 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -9,8 +9,8 @@ import semver version = ( 0, 12, - 1, - 'alpha.2', + 2, + 'alpha.3', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 2404c5e..64ce7d1 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -13,7 +13,7 @@ ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages # --force-reinstall \ # --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ # -r requirements.txt -COPY *requirements.txt . +COPY *requirements.txt ./ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url https://pypi.org/simple \ --extra-index-url $GITLAB_PYTHON_REGISTRY \ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 3548672..09811a0 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib~=0.0.5a2 +chainlib-eth>=0.0.6a1,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index 66ea7f8..c70193a 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -1,15 +1,15 @@ -chainsyncer[sql]~=0.0.3a5 -chainqueue~=0.0.2b7 +chainqueue>=0.0.3a2,<0.1.0 +chainsyncer[sql]>=0.0.5a1,<0.1.0 alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 liveness~=0.0.1a7 -eth-address-index~=0.1.2a2 -eth-accounts-index~=0.0.12a2 -cic-eth-registry~=0.5.6a2 -erc20-faucet~=0.2.2a2 -erc20-transfer-authorization~=0.3.2a2 -sarafu-faucet~=0.0.4a3 +eth-address-index>=0.1.3a1,<0.2.0 +eth-accounts-index>=0.0.13a1,<0.1.0 +cic-eth-registry>=0.5.7a1,<0.6.0 +erc20-faucet>=0.2.3a1,<0.3.0 +erc20-transfer-authorization>=0.3.3a1,<0.4.0 +sarafu-faucet>=0.0.4a5,<0.1.0 moolb~=0.1.1b2 diff --git a/apps/cic-eth/test_requirements.txt b/apps/cic-eth/test_requirements.txt index 088324c..cf7c693 100644 --- a/apps/cic-eth/test_requirements.txt +++ b/apps/cic-eth/test_requirements.txt @@ -6,4 +6,4 @@ pytest-redis==2.0.0 redis==3.5.3 eth-tester==0.5.0b3 py-evm==0.3.0a20 -eth-erc20~=0.0.10a3 +eth-erc20~=0.0.11a1 diff --git a/apps/cic-eth/tests/task/api/test_admin.py b/apps/cic-eth/tests/task/api/test_admin.py index fd41c2b..19c9090 100644 --- a/apps/cic-eth/tests/task/api/test_admin.py +++ b/apps/cic-eth/tests/task/api/test_admin.py @@ -290,6 +290,7 @@ def test_fix_nonce( txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database) ks = txs.keys() assert len(ks) == 2 + for k in ks: hsh = add_0x(k) otx = Otx.load(hsh, session=init_database) diff --git a/apps/cic-eth/tests/task/api/test_admin_noncritical.py b/apps/cic-eth/tests/task/api/test_admin_noncritical.py index 7d21737..014480f 100644 --- a/apps/cic-eth/tests/task/api/test_admin_noncritical.py +++ b/apps/cic-eth/tests/task/api/test_admin_noncritical.py @@ -184,7 +184,7 @@ def test_admin_api_account( api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) - r = api.account(default_chain_spec, agent_roles['ALICE']) + r = api.account(default_chain_spec, agent_roles['ALICE'], include_sender=True, include_recipient=True) assert len(r) == 5 api = AdminApi(eth_rpc, queue=None, call_address=contract_roles['CONTRACT_DEPLOYER']) diff --git a/apps/cic-eth/tests/unit/queue/test_query.py b/apps/cic-eth/tests/unit/queue/test_query.py index b82fbd4..cd3f6ad 100644 --- a/apps/cic-eth/tests/unit/queue/test_query.py +++ b/apps/cic-eth/tests/unit/queue/test_query.py @@ -9,6 +9,11 @@ from chainlib.eth.gas import ( Gas, ) from chainlib.chain import ChainSpec +from hexathon import ( + add_0x, + strip_0x, + uniform as hex_uniform, + ) # local imports from cic_eth.db.enum import LockEnum @@ -34,7 +39,10 @@ def test_upcoming_with_lock( gas_oracle = RPCGasOracle(eth_rpc) c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) - (tx_hash_hex, tx_rpc) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6)) + alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE']))) + bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB']))) + + (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6)) tx_signed_raw_hex = tx_rpc['params'][0] register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) @@ -43,12 +51,12 @@ def test_upcoming_with_lock( txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING) assert len(txs.keys()) == 1 - Lock.set(str(default_chain_spec), LockEnum.SEND, address=agent_roles['ALICE']) + Lock.set(str(default_chain_spec), LockEnum.SEND, address=alice_normal) - txs = get_upcoming_tx(default_chain_spec, StatusEnum.PENDING) + txs = get_upcoming_tx(default_chain_spec, status=StatusEnum.PENDING) assert len(txs.keys()) == 0 - (tx_hash_hex, tx_rpc) = c.create(agent_roles['BOB'], agent_roles['ALICE'], 100 * (10 ** 6)) + (tx_hash_hex, tx_rpc) = c.create(bob_normal, alice_normal, 100 * (10 ** 6)) tx_signed_raw_hex = tx_rpc['params'][0] register_tx(tx_hash_hex, tx_signed_raw_hex, default_chain_spec, None, session=init_database) diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 7e9d74c..03a0014 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,7 +1,7 @@ -crypto-dev-signer~=0.4.14b7 +crypto-dev-signer>=0.4.14b7,<=0.4.14 chainqueue~=0.0.2b6 confini>=0.3.6rc4,<0.5.0 -cic-eth-registry~=0.5.6a2 +cic-eth-registry>=0.5.7a1,<0.6.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 diff --git a/apps/cic-notify/cic_notify/version.py b/apps/cic-notify/cic_notify/version.py index 475867b..3a99ed0 100644 --- a/apps/cic-notify/cic_notify/version.py +++ b/apps/cic-notify/cic_notify/version.py @@ -9,7 +9,7 @@ import semver logg = logging.getLogger() -version = (0, 4, 0, 'alpha.8') +version = (0, 4, 0, 'alpha.10') version_object = semver.VersionInfo( major=version[0], diff --git a/apps/cic-notify/requirements.txt b/apps/cic-notify/requirements.txt index 5b67709..953027d 100644 --- a/apps/cic-notify/requirements.txt +++ b/apps/cic-notify/requirements.txt @@ -1 +1,7 @@ -cic_base[full_graph]~=0.2.0a3 +confini~=0.4.1a1 +africastalking==1.2.3 +SQLAlchemy==1.3.20 +alembic==1.4.2 +psycopg2==2.8.6 +celery==4.4.7 +redis==3.5.3 diff --git a/apps/cic-notify/test_requirements.txt b/apps/cic-notify/test_requirements.txt index 41c5f09..5a2ab67 100644 --- a/apps/cic-notify/test_requirements.txt +++ b/apps/cic-notify/test_requirements.txt @@ -2,3 +2,4 @@ pytest~=6.0.1 pytest-celery~=0.0.0a1 pytest-mock~=3.3.1 pysqlite3~=0.4.3 +pytest-cov==2.10.1 diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 2b96a9d..e47f87b 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,4 +1,17 @@ -cic_base[full_graph]==0.2.0a3 -cic-eth~=0.12.0a1 -cic-notify~=0.4.0a8 -cic-types~=0.1.0a11 +cic-eth~=0.12.2a3 +cic-notify~=0.4.0a10 +cic-types~=0.1.0a14 +confini~=0.4.1a1 +semver==2.13.0 +alembic==1.4.2 +SQLAlchemy==1.3.20 +psycopg2==2.8.6 +tinydb==4.2.0 +phonenumbers==8.12.12 +redis==3.5.3 +celery==4.4.7 +python-i18n==0.3.9 +pyxdg==0.27 +bcrypt==3.2.0 +uWSGI==2.0.19.1 +transitions==0.8.4 diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 223e7c4..9d5cb9b 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,11 +1,10 @@ -sarafu-faucet~=0.0.4a4 -cic-eth[tools]==0.12.1a2 -eth-erc20~=0.0.10a3 -erc20-demurrage-token==0.0.2a4 -eth-address-index~=0.1.2a2 -eth-accounts-index~=0.0.12a2 -cic-eth-registry~=0.5.6a2 -erc20-faucet~=0.2.2a2 -erc20-transfer-authorization~=0.3.2a2 -sarafu-faucet~=0.0.4a3 -chainlib-eth~=0.0.5a4 +cic-eth[tools]==0.12.2a3 +eth-erc20>=0.0.11a1,<0.1.0 +erc20-demurrage-token>=0.0.2a5,<0.1.0 +eth-address-index>=0.1.3a1,<0.2.0 +eth-accounts-index>=0.0.13a1,<0.1.0 +cic-eth-registry>=0.5.7a1,<=0.6.0 +erc20-faucet>=0.2.3a1,<0.3.0 +erc20-transfer-authorization>=0.3.3a1,<0.4.0 +sarafu-faucet>=0.0.4a4,<0.1.0 +chainlib-eth>=0.0.6a1,<0.1.0 diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index 3cd76f3..aba7ea6 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -72,6 +72,8 @@ cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_ eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER # Transfer gas to custodial gas provider adddress +_CONFINI_DIR=$CONFINI_DIR +unset CONFINI_DIR >&2 echo gift gas to gas gifter >&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount @@ -100,6 +102,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS #echo -n 0 > $init_level_file +CONFINI_DIR=$_CONFINI_DIR # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration cic-eth-ctl -i :: unlock INIT cic-eth-ctl -i :: unlock SEND