diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index dec5cf52..5c2936d8 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -23,7 +23,7 @@ def upgrade(): op.create_table( 'lock', sa.Column('id', sa.Integer, primary_key=True), - sa.Column("address", sa.String(42), nullable=True), + sa.Column("address", sa.String, nullable=True), sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py new file mode 100644 index 00000000..cc1b0e09 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py @@ -0,0 +1,31 @@ +"""Add gas cache + +Revision ID: c91cafc3e0c1 +Revises: aee12aeb47ec +Create Date: 2021-10-28 20:45:34.239865 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c91cafc3e0c1' +down_revision = 'aee12aeb47ec' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'gas_cache', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column("address", sa.String, nullable=False), + sa.Column("tx_hash", sa.String, nullable=True), + sa.Column("method", sa.String, nullable=True), + sa.Column("value", sa.BIGINT(), nullable=False), + ) + + +def downgrade(): + op.drop_table('gas_cache') diff --git a/apps/cic-eth/cic_eth/db/models/gas_cache.py b/apps/cic-eth/cic_eth/db/models/gas_cache.py new file mode 100644 index 00000000..af514e1e --- /dev/null +++ b/apps/cic-eth/cic_eth/db/models/gas_cache.py @@ -0,0 +1,27 @@ +# standard imports +import logging + +# external imports +from sqlalchemy import Column, String, NUMERIC + +# local imports +from .base import SessionBase + +logg = logging.getLogger(__name__) + + +class GasCache(SessionBase): + """Provides gas budget cache for token operations + """ + __tablename__ = 'gas_cache' + + address = Column(String()) + tx_hash = Column(String()) + method = Column(String()) + value = Column(NUMERIC()) + + def __init__(self, address, method, value, tx_hash): + self.address = address + self.tx_hash = tx_hash + self.method = method + self.value = value diff --git a/apps/cic-eth/cic_eth/db/models/nonce.py b/apps/cic-eth/cic_eth/db/models/nonce.py index 280e94ca..71f404f6 100644 --- a/apps/cic-eth/cic_eth/db/models/nonce.py +++ b/apps/cic-eth/cic_eth/db/models/nonce.py @@ -12,7 +12,7 @@ from cic_eth.error import ( IntegrityError, ) -logg = logging.getLogger() +logg = logging.getLogger(__name__) class Nonce(SessionBase): @@ -21,7 +21,7 @@ class Nonce(SessionBase): __tablename__ = 'nonce' nonce = Column(Integer) - address_hex = Column(String(42)) + address_hex = Column(String(40)) @staticmethod diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 435949fc..c00ee05b 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache from chainqueue.db.models.otx import Otx # local imports +from cic_eth.db.models.gas_cache import GasCache from cic_eth.db.models.role import AccountRole from cic_eth.db.models.base import SessionBase from cic_eth.error import ( @@ -78,6 +79,34 @@ class MaxGasOracle: return MAXIMUM_FEE_UNITS +@celery_app.task(base=CriticalSQLAlchemyTask) +def apply_gas_value_cache(address, method, value, tx_hash): + return apply_gas_value_cache_local(address, method, value, tx_hash) + + +def apply_gas_value_cache_local(address, method, value, tx_hash, session=None): + address = strip_0x(address) + tx_hash = strip_0x(tx_hash) + value = int(value) + + session = SessionBase.bind_session(session) + q = session.query(GasCache) + q = q.filter(GasCache.address==address) + q = q.filter(GasCache.method==method) + o = q.first() + + if o == None: + o = GasCache(address, method, value, tx_hash) + elif tx.gas_used > o.value: + o.value = value + o.tx_hash = strip_0x(tx_hash) + + session.add(o) + session.commit() + + SessionBase.release_session(session) + + def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher. diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py new file mode 100644 index 00000000..6c8ace88 --- /dev/null +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py @@ -0,0 +1,43 @@ +# external imports +from eth_erc20 import ERC20 +from chainlib.eth.contract import ( + ABIContractEncoder, + ABIContractType, + ) +import celery + +# local imports +from .base import SyncFilter + + +class TokenFilter(SyncFilter): + + def __init__(self, chain_spec, queue): + self.queue = queue + self.chain_spec = chain_spec + + + def filter(self, conn, block, tx, db_session=None): + if not tx.payload: + return (None, None) + + try: + r = ERC20.parse_transfer_request(tx.payload) + except RequestMismatchException: + return (None, None) + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + s = celery.signature( + 'cic_eth.eth.gas.apply_gas_value_cache', + [ + tx.inputs[0], + method, + tx.gas_used, + tx.hash, + ], + queue=self.queue, + ) + return s.apply_async() diff --git a/apps/cic-eth/tests/filters/test_token_filter.py b/apps/cic-eth/tests/filters/test_token_filter.py new file mode 100644 index 00000000..10cb8a42 --- /dev/null +++ b/apps/cic-eth/tests/filters/test_token_filter.py @@ -0,0 +1,80 @@ +# external imports +from eth_erc20 import ERC20 +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainlib.eth.tx import ( + TxFormat, + receipt, + raw, + unpack, + Tx, + ) +from chainlib.eth.block import ( + Block, + block_latest, + block_by_number, + ) +from chainlib.eth.contract import ABIContractEncoder +from hexathon import strip_0x + +# local imports +from cic_eth.runnable.daemons.filters.token import TokenFilter +from cic_eth.db.models.gas_cache import GasCache +from cic_eth.db.models.base import SessionBase + + +def test_filter_gas( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + token_roles, + foo_token, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED) + o = raw(tx_signed_raw_hex) + eth_rpc.do(o) + o = receipt(tx_hash_hex) + rcpt = eth_rpc.do(o) + assert rcpt['status'] == 1 + + fltr = TokenFilter(default_chain_spec, queue=None) + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + tx.apply_receipt(rcpt) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + r = t.get_leaf() + assert t.successful() + + q = init_database.query(GasCache) + q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex)) + o = q.first() + + assert o.address == strip_0x(foo_token) + assert o.value > 0 + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + assert o.method == method diff --git a/apps/contract-migration/4_init_custodial.sh b/apps/contract-migration/4_init_custodial.sh index 5dc17d72..84bf96f8 100644 --- a/apps/contract-migration/4_init_custodial.sh +++ b/apps/contract-migration/4_init_custodial.sh @@ -18,6 +18,18 @@ fi must_address "$CIC_REGISTRY_ADDRESS" "registry" must_eth_rpc + + +default_token=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw DefaultToken` +h=`erc20-transfer -u -e $default_token -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -y $WALLET_KEY_FILE -s 10` +r=`eth-receipt + +set +e +set +a + +exit 0 + + # get required addresses from registries token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry` accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`