From 8240e58c0aada7a550995f69b7adce84fdb7fa70 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Thu, 4 Mar 2021 15:06:14 +0000 Subject: [PATCH] cic-eth: Introduce transfer authorization contract --- apps/cic-eth/cic_eth/admin/debug.py | 23 ++- apps/cic-eth/cic_eth/api/api_task.py | 6 +- .../versions/f738d9962fdf_debug_output.py | 32 +++ .../versions/f738d9962fdf_debug_output.py | 32 +++ apps/cic-eth/cic_eth/db/models/debug.py | 23 +++ apps/cic-eth/cic_eth/db/models/nonce.py | 2 +- apps/cic-eth/cic_eth/db/models/otx.py | 2 +- apps/cic-eth/cic_eth/eth/request.py | 194 ------------------ apps/cic-eth/cic_eth/eth/tx.py | 29 +-- apps/cic-eth/cic_eth/queue/tx.py | 3 +- .../runnable/daemons/filters/__init__.py | 1 + .../cic_eth/runnable/daemons/filters/gas.py | 4 +- .../runnable/daemons/filters/transferauth.py | 89 ++++++++ .../{server_agent.py => daemons/server.py} | 59 +----- .../cic_eth/runnable/daemons/tasker.py | 2 +- .../cic_eth/runnable/daemons/tracker.py | 4 + apps/cic-eth/cic_eth/runnable/view.py | 9 +- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/requirements.txt | 6 +- apps/cic-eth/tests/fixtures_celery.py | 2 +- apps/cic-eth/tests/tasks/test_debug.py | 29 +++ .../tests/tasks/test_transfer_approval.py | 76 ------- apps/cic-eth/tests/unit/db/test_debug.py | 16 ++ apps/contract-migration/reset.sh | 2 +- .../scripts/import_balance.py | 6 +- .../scripts/requirements.txt | 4 +- apps/contract-migration/seed_cic_eth.sh | 13 +- apps/requirements.txt | 2 +- docker-compose.yml | 6 +- 29 files changed, 312 insertions(+), 366 deletions(-) create mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py create mode 100644 apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py create mode 100644 apps/cic-eth/cic_eth/db/models/debug.py delete mode 100644 apps/cic-eth/cic_eth/eth/request.py create mode 100644 apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py rename apps/cic-eth/cic_eth/runnable/{server_agent.py => daemons/server.py} (69%) create mode 100644 apps/cic-eth/tests/tasks/test_debug.py delete mode 100644 apps/cic-eth/tests/tasks/test_transfer_approval.py create mode 100644 apps/cic-eth/tests/unit/db/test_debug.py diff --git a/apps/cic-eth/cic_eth/admin/debug.py b/apps/cic-eth/cic_eth/admin/debug.py index 575be54..a8c233e 100644 --- a/apps/cic-eth/cic_eth/admin/debug.py +++ b/apps/cic-eth/cic_eth/admin/debug.py @@ -1,12 +1,25 @@ +# standard imports import datetime +# external imports import celery +# local imports +from cic_eth.db.models.debug import Debug +from cic_eth.db.models.base import SessionBase +from cic_eth.task import CriticalSQLAlchemyTask + celery_app = celery.current_app -@celery_app.task() -def out_tmp(tag, txt): - f = open('/tmp/err.{}.txt'.format(tag), "w") - f.write(txt) - f.close() +@celery_app.task(base=CriticalSQLAlchemyTask) +def alert(chained_input, tag, txt): + session = SessionBase.create_session() + + o = Debug(tag, txt) + session.add(o) + session.commit() + + session.close() + + return chained_input diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 8522ffb..b992e3d 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -489,8 +489,9 @@ class Api: ], queue=self.queue, ) + s_local.link(s_brief) if self.callback_param != None: - s_assemble.link(self.callback_success).on_error(self.callback_error) + s_brief.link(self.callback_success).on_error(self.callback_error) t = None if external_task != None: @@ -515,11 +516,10 @@ class Api: c = celery.chain(s_external_get, s_external_process) t = celery.chord([s_local, c])(s_brief) else: - t = s_local.apply_sync() + t = s_local.apply_async(queue=self.queue) return t - def ping(self, r): """A noop callback ping for testing purposes. diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py new file mode 100644 index 0000000..378f296 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/f738d9962fdf_debug_output.py @@ -0,0 +1,32 @@ +"""debug output + +Revision ID: f738d9962fdf +Revises: ec40ac0974c1 +Create Date: 2021-03-04 08:32:43.281214 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f738d9962fdf' +down_revision = 'ec40ac0974c1' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'debug', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('tag', sa.String, nullable=False), + sa.Column('description', sa.String, nullable=False), + sa.Column('date_created', sa.DateTime, nullable=False), + ) + pass + + +def downgrade(): + op.drop_table('debug') + pass diff --git a/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py new file mode 100644 index 0000000..378f296 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/postgresql/versions/f738d9962fdf_debug_output.py @@ -0,0 +1,32 @@ +"""debug output + +Revision ID: f738d9962fdf +Revises: ec40ac0974c1 +Create Date: 2021-03-04 08:32:43.281214 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f738d9962fdf' +down_revision = 'ec40ac0974c1' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'debug', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('tag', sa.String, nullable=False), + sa.Column('description', sa.String, nullable=False), + sa.Column('date_created', sa.DateTime, nullable=False), + ) + pass + + +def downgrade(): + op.drop_table('debug') + pass diff --git a/apps/cic-eth/cic_eth/db/models/debug.py b/apps/cic-eth/cic_eth/db/models/debug.py new file mode 100644 index 0000000..04682c9 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/models/debug.py @@ -0,0 +1,23 @@ +# standard imports +import datetime +import logging + +# external imports +from sqlalchemy import Column, String, DateTime + +# local imports +from .base import SessionBase + + +class Debug(SessionBase): + + __tablename__ = 'debug' + + date_created = Column(DateTime, default=datetime.datetime.utcnow) + tag = Column(String) + description = Column(String) + + + def __init__(self, tag, description): + self.tag = tag + self.description = description diff --git a/apps/cic-eth/cic_eth/db/models/nonce.py b/apps/cic-eth/cic_eth/db/models/nonce.py index a029d81..3b474a9 100644 --- a/apps/cic-eth/cic_eth/db/models/nonce.py +++ b/apps/cic-eth/cic_eth/db/models/nonce.py @@ -88,7 +88,7 @@ class Nonce(SessionBase): #session.execute('UNLOCK TABLE nonce') #conn.close() session.commit() - session.commit() +# session.commit() SessionBase.release_session(session) return nonce diff --git a/apps/cic-eth/cic_eth/db/models/otx.py b/apps/cic-eth/cic_eth/db/models/otx.py index 523ae2e..ec88fa4 100644 --- a/apps/cic-eth/cic_eth/db/models/otx.py +++ b/apps/cic-eth/cic_eth/db/models/otx.py @@ -2,7 +2,7 @@ import datetime import logging -# third-party imports +# external imports from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method diff --git a/apps/cic-eth/cic_eth/eth/request.py b/apps/cic-eth/cic_eth/eth/request.py deleted file mode 100644 index 8f1c6b9..0000000 --- a/apps/cic-eth/cic_eth/eth/request.py +++ /dev/null @@ -1,194 +0,0 @@ -# standard imports -import logging - -# third-party imports -import web3 -import celery -from erc20_approval_escrow import TransferApproval -from cic_registry import CICRegistry -from cic_registry.chain import ChainSpec - -# local imports -from cic_eth.db.models.tx import TxCache -from cic_eth.db.models.base import SessionBase -from cic_eth.eth import RpcClient -from cic_eth.eth.factory import TxFactory -from cic_eth.eth.task import sign_and_register_tx -from cic_eth.eth.util import unpack_signed_raw_tx -from cic_eth.eth.task import create_check_gas_and_send_task -from cic_eth.error import TokenCountError - -celery_app = celery.current_app -logg = logging.getLogger() - -contract_function_signatures = { - 'request': 'b0addede', - } - - -class TransferRequestTxFactory(TxFactory): - """Factory for creating Transfer request transactions using the TransferApproval contract backend - """ - def request( - self, - token_address, - beneficiary_address, - amount, - chain_spec, - ): - """Create a new TransferApproval.request transaction - - :param token_address: Token to create transfer request for - :type token_address: str, 0x-hex - :param beneficiary_address: Beneficiary of token transfer - :type beneficiary_address: str, 0x-hex - :param amount: Amount of tokens to transfer - :type amount: number - :param chain_spec: Chain spec - :type chain_spec: cic_registry.chain.ChainSpec - :returns: Transaction in standard Ethereum format - :rtype: dict - """ - transfer_approval = CICRegistry.get_contract(chain_spec, 'TransferApproval', 'TransferAuthorization') - fn = transfer_approval.function('createRequest') - tx_approval_buildable = fn(beneficiary_address, token_address, amount) - transfer_approval_gas = transfer_approval.gas('createRequest') - - tx_approval = tx_approval_buildable.buildTransaction({ - 'from': self.address, - 'gas': transfer_approval_gas, - 'gasPrice': self.gas_price, - 'chainId': chain_spec.chain_id(), - 'nonce': self.next_nonce(), - }) - return tx_approval - - -def unpack_transfer_approval_request(data): - """Verifies that a transaction is an "TransferApproval.request" transaction, and extracts call parameters from it. - - :param data: Raw input data from Ethereum transaction. - :type data: str, 0x-hex - :raises ValueError: Function signature does not match AccountRegister.add - :returns: Parsed parameters - :rtype: dict - """ - f = data[2:10] - if f != contract_function_signatures['request']: - raise ValueError('Invalid transfer request data ({})'.format(f)) - - d = data[10:] - return { - 'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]), - 'token': web3.Web3.toChecksumAddress('0x' + d[128-40:128]), - 'amount': int(d[128:], 16) - } - - -@celery_app.task(bind=True) -def transfer_approval_request(self, tokens, holder_address, receiver_address, value, chain_str): - """Creates a new transfer approval - - :param tokens: Token to generate transfer request for - :type tokens: list with single token spec as dict - :param holder_address: Address to generate transfer on behalf of - :type holder_address: str, 0x-hex - :param receiver_address: Address to transfser tokens to - :type receiver_address: str, 0x-hex - :param value: Amount of tokens to transfer - :type value: number - :param chain_spec: Chain spec string representation - :type chain_spec: str - :raises cic_eth.error.TokenCountError: More than one token in tokens argument - :returns: Raw signed transaction - :rtype: list with transaction as only element - """ - - if len(tokens) != 1: - raise TokenCountError - - chain_spec = ChainSpec.from_chain_str(chain_str) - - queue = self.request.delivery_info['routing_key'] - - t = tokens[0] - - c = RpcClient(holder_address) - - txf = TransferRequestTxFactory(holder_address, c) - - tx_transfer = txf.request(t['address'], receiver_address, value, chain_spec) - (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.request.otx_cache_transfer_approval_request') - - gas_budget = tx_transfer['gas'] * tx_transfer['gasPrice'] - - s = create_check_gas_and_send_task( - [tx_signed_raw_hex], - chain_str, - holder_address, - gas_budget, - [tx_hash_hex], - queue, - ) - s.apply_async() - return [tx_signed_raw_hex] - - -@celery_app.task() -def otx_cache_transfer_approval_request( - tx_hash_hex, - tx_signed_raw_hex, - chain_str, - ): - """Generates and commits transaction cache metadata for an TransferApproval.request transaction - - :param tx_hash_hex: Transaction hash - :type tx_hash_hex: str, 0x-hex - :param tx_signed_raw_hex: Raw signed transaction - :type tx_signed_raw_hex: str, 0x-hex - :param chain_str: Chain spec string representation - :type chain_str: str - :returns: Transaction hash and id of cache element in storage backend, respectively - :rtype: tuple - """ - chain_spec = ChainSpec.from_chain_str(chain_str) - tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:]) - tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id()) - logg.debug('in otx acche transfer approval request') - (txc, cache_id) = cache_transfer_approval_request_data(tx_hash_hex, tx) - return txc - - -@celery_app.task() -def cache_transfer_approval_request_data( - tx_hash_hex, - tx, - ): - """Helper function for otx_cache_transfer_approval_request - - :param tx_hash_hex: Transaction hash - :type tx_hash_hex: str, 0x-hex - :param tx: Signed raw transaction - :type tx: str, 0x-hex - :returns: Transaction hash and id of cache element in storage backend, respectively - :rtype: tuple - """ - tx_data = unpack_transfer_approval_request(tx['data']) - logg.debug('tx approval request data {}'.format(tx_data)) - logg.debug('tx approval request {}'.format(tx)) - - session = SessionBase.create_session() - tx_cache = TxCache( - tx_hash_hex, - tx['from'], - tx_data['to'], - tx_data['token'], - tx_data['token'], - tx_data['amount'], - tx_data['amount'], - ) - session.add(tx_cache) - session.commit() - cache_id = tx_cache.id - session.close() - return (tx_hash_hex, cache_id) diff --git a/apps/cic-eth/cic_eth/eth/tx.py b/apps/cic-eth/cic_eth/eth/tx.py index 1c34acc..4de0973 100644 --- a/apps/cic-eth/cic_eth/eth/tx.py +++ b/apps/cic-eth/cic_eth/eth/tx.py @@ -69,7 +69,6 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non for i in range(len(tx_hashes)): o = get_tx(tx_hashes[i]) txs.append(o['signed_tx']) - logg.debug('ooooo {}'.format(o)) if address == None: address = o['address'] @@ -178,12 +177,7 @@ class ParityNodeHandler: def handle(self, exception, tx_hash_hex, tx_hex): meth = self.handle_default if isinstance(exception, (ValueError)): -# s_debug = celery.signature( -# 'cic_eth.admin.debug.out_tmp', -# [tx_hash_hex, '{}: {}'.format(tx_hash_hex, exception)], -# queue=queue, -# ) -# s_debug.apply_async() + earg = exception.args[0] if earg['code'] == -32010: logg.debug('skipping lock for code {}'.format(earg['code'])) @@ -191,14 +185,15 @@ class ParityNodeHandler: elif earg['code'] == -32602: meth = self.handle_invalid_encoding else: + # TODO: move to status log db comment field meth = self.handle_invalid elif isinstance(exception, (requests.exceptions.ConnectionError)): meth = self.handle_connection - (t, e_fn, message) = meth(tx_hash_hex, tx_hex) + (t, e_fn, message) = meth(tx_hash_hex, tx_hex, str(exception)) return (t, e_fn, '{} {}'.format(message, exception)) - def handle_connection(self, tx_hash_hex, tx_hex): + def handle_connection(self, tx_hash_hex, tx_hex, debugstr=None): s_set_sent = celery.signature( 'cic_eth.queue.tx.set_sent_status', [ @@ -211,7 +206,7 @@ class ParityNodeHandler: return (t, TemporaryTxError, 'Sendfail {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid_encoding(self, tx_hash_hex, tx_hex): + def handle_invalid_encoding(self, tx_hash_hex, tx_hex, debugstr=None): tx_bytes = bytes.fromhex(tx_hex[2:]) tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id()) s_lock = celery.signature( @@ -258,7 +253,7 @@ class ParityNodeHandler: return (t, PermanentTxError, 'Reject invalid encoding {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid_parameters(self, tx_hash_hex, tx_hex): + def handle_invalid_parameters(self, tx_hash_hex, tx_hex, debugstr=None): s_sync = celery.signature( 'cic_eth.eth.tx.sync_tx', [ @@ -271,7 +266,7 @@ class ParityNodeHandler: return (t, PermanentTxError, 'Reject invalid parameters {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) - def handle_invalid(self, tx_hash_hex, tx_hex): + def handle_invalid(self, tx_hash_hex, tx_hex, debugstr=None): tx_bytes = bytes.fromhex(tx_hex[2:]) tx = unpack_signed_raw_tx(tx_bytes, self.chain_spec.chain_id()) s_lock = celery.signature( @@ -289,6 +284,16 @@ class ParityNodeHandler: [], queue=self.queue, ) + s_debug = celery.signature( + 'cic_eth.admin.debug.alert', + [ + tx_hash_hex, + tx_hash_hex, + debugstr, + ], + queue=queue, + ) + s_set_reject.link(s_debug) s_lock.link(s_set_reject) t = s_lock.apply_async() return (t, PermanentTxError, 'Reject invalid {}'.format(tx_hex_string(tx_hex, self.chain_spec.chain_id()))) diff --git a/apps/cic-eth/cic_eth/queue/tx.py b/apps/cic-eth/cic_eth/queue/tx.py index e4a8f81..c1fe0a3 100644 --- a/apps/cic-eth/cic_eth/queue/tx.py +++ b/apps/cic-eth/cic_eth/queue/tx.py @@ -30,6 +30,7 @@ from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.eth.util import unpack_signed_raw_tx # TODO: should not be in same sub-path as package that imports queue.tx from cic_eth.error import NotLocalTxError from cic_eth.error import LockedError +from cic_eth.db.enum import status_str celery_app = celery.current_app #logg = celery_app.log.get_default_logger() @@ -405,7 +406,7 @@ def get_tx_cache(tx_hash): 'tx_hash': otx.tx_hash, 'signed_tx': otx.signed_tx, 'nonce': otx.nonce, - 'status': StatusEnum(otx.status).name, + 'status': status_str(otx.status), 'status_code': otx.status, 'source_token': txc.source_token_address, 'destination_token': txc.destination_token_address, diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py index feed42c..d2a9823 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py @@ -2,3 +2,4 @@ from .callback import CallbackFilter from .tx import TxFilter from .gas import GasFilter from .register import RegistrationFilter +from .transferauth import TransferAuthFilter diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py index 697b30f..3fee3d9 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py @@ -1,7 +1,7 @@ # standard imports import logging -# third-party imports +# external imports from cic_registry.chain import ChainSpec from hexathon import add_0x @@ -24,7 +24,7 @@ class GasFilter(SyncFilter): self.chain_spec = chain_spec - def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None): + def filter(self, conn, block, tx, session): tx_hash_hex = add_0x(tx.hash) if tx.value > 0: logg.debug('gas refill tx {}'.format(tx_hash_hex)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py new file mode 100644 index 0000000..ad921b4 --- /dev/null +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py @@ -0,0 +1,89 @@ +# standard imports +import logging + +# external imports +import celery +from hexathon import ( + strip_0x, + add_0x, + ) +from chainlib.eth.address import to_checksum +from .base import SyncFilter + + +logg = logging.getLogger(__name__) + +transfer_request_signature = 'ed71262a' + +def unpack_create_request(data): + + data = strip_0x(data) + cursor = 0 + f = data[cursor:cursor+8] + cursor += 8 + + if f != transfer_request_signature: + raise ValueError('Invalid create request data ({})'.format(f)) + + o = {} + o['sender'] = data[cursor+24:cursor+64] + cursor += 64 + o['recipient'] = data[cursor+24:cursor+64] + cursor += 64 + o['token'] = data[cursor+24:cursor+64] + cursor += 64 + o['value'] = int(data[cursor:], 16) + return o + + +class TransferAuthFilter(SyncFilter): + + def __init__(self, registry, chain_spec, queue=None): + self.queue = queue + self.chain_spec = chain_spec + self.transfer_request_contract = registry.get_contract(self.chain_spec, 'TransferAuthorization') + + + def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None): + + if tx.payload == None: + logg.debug('no payload') + return False + + payloadlength = len(tx.payload) + if payloadlength != 8+256: + logg.debug('{} below minimum length for a transfer auth call'.format(payloadlength)) + logg.debug('payload {}'.format(tx.payload)) + return False + + recipient = tx.inputs[0] + if recipient != self.transfer_request_contract.address(): + logg.debug('not our transfer auth contract address {}'.format(recipient)) + return False + + o = unpack_create_request(tx.payload) + + sender = add_0x(to_checksum(o['sender'])) + recipient = add_0x(to_checksum(recipient)) + token = add_0x(to_checksum(o['token'])) + s = celery.signature( + 'cic_eth.eth.token.approve', + [ + [ + { + 'address': token, + }, + ], + sender, + recipient, + o['value'], + str(self.chain_spec), + ], + queue=self.queue, + ) + t = s.apply_async() + return True + + + def __str__(self): + return 'cic-eth transfer auth filter' diff --git a/apps/cic-eth/cic_eth/runnable/server_agent.py b/apps/cic-eth/cic_eth/runnable/daemons/server.py similarity index 69% rename from apps/cic-eth/cic_eth/runnable/server_agent.py rename to apps/cic-eth/cic_eth/runnable/daemons/server.py index 5f89f12..875c581 100644 --- a/apps/cic-eth/cic_eth/runnable/server_agent.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/server.py @@ -55,62 +55,25 @@ SessionBase.connect(dsn) celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) queue = args.q -re_transfer_approval_request = r'^/transferrequest/?' +re_something = r'^/something/?' chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) -def process_transfer_approval_request(session, env): - r = re.match(re_transfer_approval_request, env.get('PATH_INFO')) +def process_something(session, env): + r = re.match(re_something, env.get('PATH_INFO')) if not r: return None - if env.get('CONTENT_TYPE') != 'application/json': - raise AttributeError('content type') + #if env.get('CONTENT_TYPE') != 'application/json': + # raise AttributeError('content type') - if env.get('REQUEST_METHOD') != 'POST': - raise AttributeError('method') + #if env.get('REQUEST_METHOD') != 'POST': + # raise AttributeError('method') - post_data = json.load(env.get('wsgi.input')) - token_address = web3.Web3.toChecksumAddress(post_data['token_address']) - holder_address = web3.Web3.toChecksumAddress(post_data['holder_address']) - beneficiary_address = web3.Web3.toChecksumAddress(post_data['beneficiary_address']) - value = int(post_data['value']) - - logg.debug('transfer approval request token {} to {} from {} value {}'.format( - token_address, - beneficiary_address, - holder_address, - value, - ) - ) - - s = celery.signature( - 'cic_eth.eth.request.transfer_approval_request', - [ - [ - { - 'address': token_address, - }, - ], - holder_address, - beneficiary_address, - value, - config.get('CIC_CHAIN_SPEC'), - ], - queue=queue, - ) - t = s.apply_async() - r = t.get() - tx_raw_bytes = bytes.fromhex(r[0][2:]) - tx = unpack_signed_raw_tx(tx_raw_bytes, chain_spec.chain_id()) - for r in t.collect(): - logg.debug('result {}'.format(r)) - - if not t.successful(): - raise RuntimeError(tx['hash']) - - return ('text/plain', tx['hash'].encode('utf-8'),) + #post_data = json.load(env.get('wsgi.input')) + + #return ('text/plain', 'foo'.encode('utf-8'),) # uwsgi application @@ -125,7 +88,7 @@ def application(env, start_response): session = SessionBase.create_session() for handler in [ - process_transfer_approval_request, + process_something, ]: try: r = handler(session, env) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 9a4bb2c..ab54997 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -26,7 +26,6 @@ from cic_eth.eth import bancor from cic_eth.eth import token from cic_eth.eth import tx from cic_eth.eth import account -from cic_eth.eth import request from cic_eth.admin import debug from cic_eth.admin import ctrl from cic_eth.eth.rpc import RpcClient @@ -40,6 +39,7 @@ from cic_eth.callbacks import redis from cic_eth.db.models.base import SessionBase from cic_eth.db.models.otx import Otx from cic_eth.db import dsn_from_config +from cic_eth.ext import tx logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 3bb58b7..bd05d2b 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -58,6 +58,7 @@ from cic_eth.runnable.daemons.filters import ( GasFilter, TxFilter, RegistrationFilter, + TransferAuthFilter, ) script_dir = os.path.realpath(os.path.dirname(__file__)) @@ -146,6 +147,8 @@ def main(): gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE')) + transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) + i = 0 for syncer in syncers: logg.debug('running syncer index {}'.format(i)) @@ -153,6 +156,7 @@ def main(): syncer.add_filter(registration_filter) # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break syncer.add_filter(tx_filter) + syncer.add_filter(transfer_auth_filter) for cf in callback_filters: syncer.add_filter(cf) diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index a9d964c..40e10cc 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -23,8 +23,11 @@ from hexathon import add_0x # local imports from cic_eth.api import AdminApi from cic_eth.eth.rpc import RpcClient -from cic_eth.db.enum import StatusEnum -from cic_eth.db.enum import LockEnum +from cic_eth.db.enum import ( + StatusEnum, + status_str, + LockEnum, +) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -119,7 +122,7 @@ def render_tx(o, **kwargs): for v in o.get('status_log', []): d = datetime.datetime.fromisoformat(v[0]) - e = StatusEnum(v[1]).name + e = status_str(v[1]) content += '{}: {}\n'.format(d, e) return content diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 06391aa..99c2750 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 10, 0, - 'alpha.36', + 'alpha.37', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 1c9e50c..bf832fb 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,6 +1,7 @@ +cic-base~=0.1.1a10 web3==5.12.2 celery==4.4.7 -crypto-dev-signer~=0.4.13rc3 +crypto-dev-signer~=0.4.13rc4 confini~=0.3.6rc3 cic-registry~=0.5.3a22 cic-bancor~=0.0.6 @@ -9,7 +10,7 @@ alembic==1.4.2 websockets==8.1 requests~=2.24.0 eth_accounts_index~=0.0.10a10 -erc20-approval-escrow~=0.3.0a5 +erc20-transfer-authorization~=0.3.0a10 erc20-single-shot-faucet~=0.2.0a6 rlp==2.0.1 uWSGI==2.0.19.1 @@ -21,4 +22,3 @@ eth-address-index~=0.1.0a8 chainlib~=0.0.1a19 hexathon~=0.0.1a3 chainsyncer~=0.0.1a19 -cic-base==0.1.1a10 diff --git a/apps/cic-eth/tests/fixtures_celery.py b/apps/cic-eth/tests/fixtures_celery.py index 5d4f712..e8a322a 100644 --- a/apps/cic-eth/tests/fixtures_celery.py +++ b/apps/cic-eth/tests/fixtures_celery.py @@ -13,13 +13,13 @@ def celery_includes(): return [ 'cic_eth.eth.bancor', 'cic_eth.eth.token', - 'cic_eth.eth.request', 'cic_eth.eth.tx', 'cic_eth.ext.tx', 'cic_eth.queue.tx', 'cic_eth.queue.balance', 'cic_eth.admin.ctrl', 'cic_eth.admin.nonce', + 'cic_eth.admin.debug', 'cic_eth.eth.account', 'cic_eth.callbacks.noop', 'cic_eth.callbacks.http', diff --git a/apps/cic-eth/tests/tasks/test_debug.py b/apps/cic-eth/tests/tasks/test_debug.py new file mode 100644 index 0000000..65d8eed --- /dev/null +++ b/apps/cic-eth/tests/tasks/test_debug.py @@ -0,0 +1,29 @@ +# external imports +import celery + +# local imports +from cic_eth.db.models.debug import Debug + + +def test_debug_alert( + init_database, + celery_session_worker, + ): + + s = celery.signature( + 'cic_eth.admin.debug.alert', + [ + 'foo', + 'bar', + 'baz', + ], + queue=None, + ) + t = s.apply_async() + r = t.get() + assert r == 'foo' + + q = init_database.query(Debug) + q = q.filter(Debug.tag=='bar') + o = q.first() + assert o.description == 'baz' diff --git a/apps/cic-eth/tests/tasks/test_transfer_approval.py b/apps/cic-eth/tests/tasks/test_transfer_approval.py deleted file mode 100644 index 574cd4c..0000000 --- a/apps/cic-eth/tests/tasks/test_transfer_approval.py +++ /dev/null @@ -1,76 +0,0 @@ -# standard imports -import logging -import time - -# third-party imports -from erc20_approval_escrow import TransferApproval -import celery -import sha3 - -# local imports -from cic_eth.eth.token import TokenTxFactory - -logg = logging.getLogger() - - -# BUG: transaction receipt only found sometimes -def test_transfer_approval( - default_chain_spec, - transfer_approval, - bancor_tokens, - w3_account_roles, - eth_empty_accounts, - cic_registry, - init_database, - celery_session_worker, - init_eth_tester, - init_w3, - ): - - s = celery.signature( - 'cic_eth.eth.request.transfer_approval_request', - [ - [ - { - 'address': bancor_tokens[0], - }, - ], - w3_account_roles['eth_account_sarafu_owner'], - eth_empty_accounts[0], - 1024, - str(default_chain_spec), - ], - ) - - s_send = celery.signature( - 'cic_eth.eth.tx.send', - [ - str(default_chain_spec), - ], - - ) - s.link(s_send) - t = s.apply_async() - - tx_signed_raws = t.get() - for r in t.collect(): - logg.debug('result {}'.format(r)) - - assert t.successful() - - init_eth_tester.mine_block() - - h = sha3.keccak_256() - tx_signed_raw = tx_signed_raws[0] - tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw[2:]) - h.update(tx_signed_raw_bytes) - tx_hash = h.digest() - rcpt = init_w3.eth.getTransactionReceipt(tx_hash) - - assert rcpt.status == 1 - - a = TransferApproval(init_w3, transfer_approval) - assert a.last_serial() == 1 - - logg.debug('requests {}'.format(a.requests(1)['serial'])) - diff --git a/apps/cic-eth/tests/unit/db/test_debug.py b/apps/cic-eth/tests/unit/db/test_debug.py new file mode 100644 index 0000000..776247d --- /dev/null +++ b/apps/cic-eth/tests/unit/db/test_debug.py @@ -0,0 +1,16 @@ +# local imports +from cic_eth.db.models.debug import Debug + + +def test_debug( + init_database, + ): + + o = Debug('foo', 'bar') + init_database.add(o) + init_database.commit() + + q = init_database.query(Debug) + q = q.filter(Debug.tag=='foo') + o = q.first() + assert o.description == 'bar' diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index d3dc955..d76d244 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -35,7 +35,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then CIC_ACCOUNTS_INDEX_ADDRESS=`eth-accounts-index-deploy -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $keystore_file --writer $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER -vv -w` - CIC_REGISTRY_ADDRESS=`cic-registry-deploy -i $CIC_CHAIN_SPEC -y $keystore_file -k CICRegistry -k BancorRegistry -k AccountRegistry -k TokenRegistry -k AddressDeclarator -k Faucet -k TransferApproval -p $ETH_PROVIDER -vv -w` + CIC_REGISTRY_ADDRESS=`cic-registry-deploy -i $CIC_CHAIN_SPEC -y $keystore_file -k CICRegistry -k BancorRegistry -k AccountRegistry -k TokenRegistry -k AddressDeclarator -k Faucet -k TransferAuthorization -p $ETH_PROVIDER -vv -w` cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -k CICRegistry -p $ETH_PROVIDER $CIC_REGISTRY_ADDRESS -vv #cic-registry-set -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -k BancorRegistry -p $ETH_PROVIDER $BANCOR_REGISTRY_ADDRESS -vv cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -k AccountRegistry -p $ETH_PROVIDER $CIC_ACCOUNTS_INDEX_ADDRESS -vv diff --git a/apps/contract-migration/scripts/import_balance.py b/apps/contract-migration/scripts/import_balance.py index 658ab0d..d9347f5 100644 --- a/apps/contract-migration/scripts/import_balance.py +++ b/apps/contract-migration/scripts/import_balance.py @@ -148,7 +148,11 @@ class Handler: return u = Person.deserialize(o) original_address = u.identities[old_chain_spec.engine()]['{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())][0] - balance = self.balances[original_address] + try: + balance = self.balances[original_address] + except KeyError as e: + logg.error('balance get fail orig {} new {}'.format(original_address, recipient)) + return # TODO: store token object in handler ,get decimals from there multiplier = 10**6 diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index d311e7a..1cb6cbb 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,3 +1,3 @@ -cic-base[full_graph]==0.1.1a10 -cic-eth==0.10.0a36 +cic-base[full_graph]==0.1.1a12 +cic-eth==0.10.0a37 cic-types==0.1.0a8 diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index b8b3002..b0ca84d 100644 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -31,7 +31,8 @@ set -e set -a # We need to not install these here... -pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a36 chainlib==0.0.1a19 cic-contracts==0.0.2a2 +pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a37 chainlib==0.0.1a19 cic-contracts==0.0.2a2 +pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL --force-reinstall erc20-transfer-authorization==0.3.0a10 >&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER @@ -89,13 +90,13 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >&2 echo "deploy transfer authorization contract" -#CIC_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-approval-escrow-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --approver $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER -w $debug` -#echo CIC_APPROVAL_ESCROW_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS >> $env_out_file -#export CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS +CIC_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --approver $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER -w $debug` +echo CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS >> $env_out_file +export CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS # Register transfer approval contract -#>&2 echo "add transfer approval request contract to registry" -#>&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TransferApproval -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $CIC_TRANSFER_AUTHORIZATION_ADDRESS +>&2 echo "add transfer authorization request contract to registry" +>&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TransferAuthorization -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $CIC_TRANSFER_AUTHORIZATION_ADDRESS # Deploy one-time token faucet for newly created token diff --git a/apps/requirements.txt b/apps/requirements.txt index ed354d8..97ffa2b 100644 --- a/apps/requirements.txt +++ b/apps/requirements.txt @@ -3,7 +3,7 @@ alembic==1.4.2 bcrypt==3.2.0 celery==4.4.7 confini==0.3.6rc3 -crypto-dev-signer==0.4.13rc3 +crypto-dev-signer==0.4.13rc4 cryptography==3.2.1 ecuth==0.4.5a1 eth-accounts-index==0.0.10a10 diff --git a/docker-compose.yml b/docker-compose.yml index a9da111..28762eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -161,7 +161,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - /usr/local/bin/cic-cache-tracker -v + /usr/local/bin/cic-cache-tracker -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -192,7 +192,7 @@ services: if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \ - --http :80 \ + --http :8000 \ --pyargv -vv cic-eth-tasker: @@ -237,7 +237,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tasker.sh -q cic-eth -vv + ./start_tasker.sh -q cic-eth -v # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] cic-eth-tracker: