From fb953d031812ad57cb07078fa3ca4c837d15f351 Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 28 Oct 2021 21:45:47 +0200 Subject: [PATCH 1/3] Add gas cache backend, test, filter --- .../default/versions/75d4767b3031_lock.py | 2 +- .../versions/c91cafc3e0c1_add_gas_cache.py | 31 +++++++ apps/cic-eth/cic_eth/db/models/gas_cache.py | 27 +++++++ apps/cic-eth/cic_eth/db/models/nonce.py | 4 +- apps/cic-eth/cic_eth/eth/gas.py | 29 +++++++ .../cic_eth/runnable/daemons/filters/token.py | 43 ++++++++++ .../tests/filters/test_token_filter.py | 80 +++++++++++++++++++ apps/contract-migration/4_init_custodial.sh | 12 +++ 8 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py create mode 100644 apps/cic-eth/cic_eth/db/models/gas_cache.py create mode 100644 apps/cic-eth/cic_eth/runnable/daemons/filters/token.py create mode 100644 apps/cic-eth/tests/filters/test_token_filter.py diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index dec5cf52..5c2936d8 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -23,7 +23,7 @@ def upgrade(): op.create_table( 'lock', sa.Column('id', sa.Integer, primary_key=True), - sa.Column("address", sa.String(42), nullable=True), + sa.Column("address", sa.String, nullable=True), sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py new file mode 100644 index 00000000..cc1b0e09 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py @@ -0,0 +1,31 @@ +"""Add gas cache + +Revision ID: c91cafc3e0c1 +Revises: aee12aeb47ec +Create Date: 2021-10-28 20:45:34.239865 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c91cafc3e0c1' +down_revision = 'aee12aeb47ec' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'gas_cache', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column("address", sa.String, nullable=False), + sa.Column("tx_hash", sa.String, nullable=True), + sa.Column("method", sa.String, nullable=True), + sa.Column("value", sa.BIGINT(), nullable=False), + ) + + +def downgrade(): + op.drop_table('gas_cache') diff --git a/apps/cic-eth/cic_eth/db/models/gas_cache.py b/apps/cic-eth/cic_eth/db/models/gas_cache.py new file mode 100644 index 00000000..af514e1e --- /dev/null +++ b/apps/cic-eth/cic_eth/db/models/gas_cache.py @@ -0,0 +1,27 @@ +# standard imports +import logging + +# external imports +from sqlalchemy import Column, String, NUMERIC + +# local imports +from .base import SessionBase + +logg = logging.getLogger(__name__) + + +class GasCache(SessionBase): + """Provides gas budget cache for token operations + """ + __tablename__ = 'gas_cache' + + address = Column(String()) + tx_hash = Column(String()) + method = Column(String()) + value = Column(NUMERIC()) + + def __init__(self, address, method, value, tx_hash): + self.address = address + self.tx_hash = tx_hash + self.method = method + self.value = value diff --git a/apps/cic-eth/cic_eth/db/models/nonce.py b/apps/cic-eth/cic_eth/db/models/nonce.py index 280e94ca..71f404f6 100644 --- a/apps/cic-eth/cic_eth/db/models/nonce.py +++ b/apps/cic-eth/cic_eth/db/models/nonce.py @@ -12,7 +12,7 @@ from cic_eth.error import ( IntegrityError, ) -logg = logging.getLogger() +logg = logging.getLogger(__name__) class Nonce(SessionBase): @@ -21,7 +21,7 @@ class Nonce(SessionBase): __tablename__ = 'nonce' nonce = Column(Integer) - address_hex = Column(String(42)) + address_hex = Column(String(40)) @staticmethod diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 435949fc..c00ee05b 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache from chainqueue.db.models.otx import Otx # local imports +from cic_eth.db.models.gas_cache import GasCache from cic_eth.db.models.role import AccountRole from cic_eth.db.models.base import SessionBase from cic_eth.error import ( @@ -78,6 +79,34 @@ class MaxGasOracle: return MAXIMUM_FEE_UNITS +@celery_app.task(base=CriticalSQLAlchemyTask) +def apply_gas_value_cache(address, method, value, tx_hash): + return apply_gas_value_cache_local(address, method, value, tx_hash) + + +def apply_gas_value_cache_local(address, method, value, tx_hash, session=None): + address = strip_0x(address) + tx_hash = strip_0x(tx_hash) + value = int(value) + + session = SessionBase.bind_session(session) + q = session.query(GasCache) + q = q.filter(GasCache.address==address) + q = q.filter(GasCache.method==method) + o = q.first() + + if o == None: + o = GasCache(address, method, value, tx_hash) + elif tx.gas_used > o.value: + o.value = value + o.tx_hash = strip_0x(tx_hash) + + session.add(o) + session.commit() + + SessionBase.release_session(session) + + def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher. diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py new file mode 100644 index 00000000..6c8ace88 --- /dev/null +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py @@ -0,0 +1,43 @@ +# external imports +from eth_erc20 import ERC20 +from chainlib.eth.contract import ( + ABIContractEncoder, + ABIContractType, + ) +import celery + +# local imports +from .base import SyncFilter + + +class TokenFilter(SyncFilter): + + def __init__(self, chain_spec, queue): + self.queue = queue + self.chain_spec = chain_spec + + + def filter(self, conn, block, tx, db_session=None): + if not tx.payload: + return (None, None) + + try: + r = ERC20.parse_transfer_request(tx.payload) + except RequestMismatchException: + return (None, None) + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + s = celery.signature( + 'cic_eth.eth.gas.apply_gas_value_cache', + [ + tx.inputs[0], + method, + tx.gas_used, + tx.hash, + ], + queue=self.queue, + ) + return s.apply_async() diff --git a/apps/cic-eth/tests/filters/test_token_filter.py b/apps/cic-eth/tests/filters/test_token_filter.py new file mode 100644 index 00000000..10cb8a42 --- /dev/null +++ b/apps/cic-eth/tests/filters/test_token_filter.py @@ -0,0 +1,80 @@ +# external imports +from eth_erc20 import ERC20 +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainlib.eth.tx import ( + TxFormat, + receipt, + raw, + unpack, + Tx, + ) +from chainlib.eth.block import ( + Block, + block_latest, + block_by_number, + ) +from chainlib.eth.contract import ABIContractEncoder +from hexathon import strip_0x + +# local imports +from cic_eth.runnable.daemons.filters.token import TokenFilter +from cic_eth.db.models.gas_cache import GasCache +from cic_eth.db.models.base import SessionBase + + +def test_filter_gas( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + agent_roles, + token_roles, + foo_token, + celery_session_worker, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED) + o = raw(tx_signed_raw_hex) + eth_rpc.do(o) + o = receipt(tx_hash_hex) + rcpt = eth_rpc.do(o) + assert rcpt['status'] == 1 + + fltr = TokenFilter(default_chain_spec, queue=None) + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + tx.apply_receipt(rcpt) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + r = t.get_leaf() + assert t.successful() + + q = init_database.query(GasCache) + q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex)) + o = q.first() + + assert o.address == strip_0x(foo_token) + assert o.value > 0 + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + assert o.method == method diff --git a/apps/contract-migration/4_init_custodial.sh b/apps/contract-migration/4_init_custodial.sh index 5dc17d72..84bf96f8 100644 --- a/apps/contract-migration/4_init_custodial.sh +++ b/apps/contract-migration/4_init_custodial.sh @@ -18,6 +18,18 @@ fi must_address "$CIC_REGISTRY_ADDRESS" "registry" must_eth_rpc + + +default_token=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw DefaultToken` +h=`erc20-transfer -u -e $default_token -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -y $WALLET_KEY_FILE -s 10` +r=`eth-receipt + +set +e +set +a + +exit 0 + + # get required addresses from registries token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry` accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry` From d35e144723636ab6ebe36b215c00d1df5c04e176 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 29 Oct 2021 07:00:25 +0200 Subject: [PATCH 2/3] Register gas cache only for registered tokens --- .../cic_eth/runnable/daemons/filters/token.py | 26 ++++++++++++++++--- .../tests/filters/test_token_filter.py | 20 ++++++++++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py index 6c8ace88..b447332b 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py @@ -1,20 +1,32 @@ +# standard imports +import logging + # external imports from eth_erc20 import ERC20 from chainlib.eth.contract import ( ABIContractEncoder, ABIContractType, ) +from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.address import is_same_address +from cic_eth_registry import CICRegistry +from cic_eth_registry.erc20 import ERC20Token +from eth_token_index import TokenUniqueSymbolIndex import celery # local imports from .base import SyncFilter +#logg = logging.getLogger(__name__) +logg = logging.getLogger() + class TokenFilter(SyncFilter): - def __init__(self, chain_spec, queue): + def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS): self.queue = queue self.chain_spec = chain_spec + self.caller_address = call_address def filter(self, conn, block, tx, db_session=None): @@ -25,7 +37,15 @@ class TokenFilter(SyncFilter): r = ERC20.parse_transfer_request(tx.payload) except RequestMismatchException: return (None, None) - + + token_address = tx.inputs[0] + token = ERC20Token(self.chain_spec, conn, token_address) + + registry = CICRegistry(self.chain_spec, conn) + r = registry.by_name(token.symbol, sender_address=self.caller_address) + if is_same_address(r, ZERO_ADDRESS): + return None + enc = ABIContractEncoder() enc.method('transfer') method = enc.get() @@ -33,7 +53,7 @@ class TokenFilter(SyncFilter): s = celery.signature( 'cic_eth.eth.gas.apply_gas_value_cache', [ - tx.inputs[0], + token_address, method, tx.gas_used, tx.hash, diff --git a/apps/cic-eth/tests/filters/test_token_filter.py b/apps/cic-eth/tests/filters/test_token_filter.py index 10cb8a42..04824dfa 100644 --- a/apps/cic-eth/tests/filters/test_token_filter.py +++ b/apps/cic-eth/tests/filters/test_token_filter.py @@ -20,6 +20,7 @@ from chainlib.eth.block import ( ) from chainlib.eth.contract import ABIContractEncoder from hexathon import strip_0x +from eth_token_index import TokenUniqueSymbolIndex # local imports from cic_eth.runnable.daemons.filters.token import TokenFilter @@ -32,10 +33,14 @@ def test_filter_gas( init_database, eth_rpc, eth_signer, + contract_roles, agent_roles, token_roles, foo_token, - celery_session_worker, + token_registry, + register_lookups, + celery_worker, + cic_registry, ): rpc = RPCConnection.connect(default_chain_spec, 'default') @@ -49,7 +54,7 @@ def test_filter_gas( rcpt = eth_rpc.do(o) assert rcpt['status'] == 1 - fltr = TokenFilter(default_chain_spec, queue=None) + fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE']) o = block_latest() r = eth_rpc.do(o) @@ -62,6 +67,17 @@ def test_filter_gas( tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) tx = Tx(tx_src, block=block) tx.apply_receipt(rcpt) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + assert t == None + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) + c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) + (tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token) + eth_rpc.do(o) + o = receipt(tx_hash_hex) + r = eth_rpc.do(o) + assert r['status'] == 1 + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) r = t.get_leaf() assert t.successful() From 1c022e9853a3cb65c4dfa0e07dbb7f1276777b60 Mon Sep 17 00:00:00 2001 From: nolash Date: Fri, 29 Oct 2021 07:33:38 +0200 Subject: [PATCH 3/3] Added changes to wrong branch --- apps/cic-eth/cic_eth/eth/erc20.py | 10 +++++++++- apps/cic-eth/cic_eth/eth/gas.py | 11 ++--------- apps/cic-eth/cic_eth/task.py | 3 +-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 4fbe2e57..52658c4a 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -10,6 +10,9 @@ from chainlib.eth.tx import ( TxFormat, unpack, ) +from chainlib.eth.contract import ( + ABIContractEncoder, + ) from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token from hexathon import ( @@ -155,7 +158,7 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s session = self.create_session() nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) - gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) + gas_oracle = self.create_gas_oracle(rpc, t['address'], MaxGasOracle.gas) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) try: (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED) @@ -226,6 +229,11 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d session = self.create_session() nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) + + enc = ABIContractEncoder() + enc.method('transferFrom') + method = enc.get() + gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) try: diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index c00ee05b..6dab6373 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -71,13 +71,6 @@ celery_app = celery.current_app logg = logging.getLogger() -MAXIMUM_FEE_UNITS = 8000000 - -class MaxGasOracle: - - def gas(code=None): - return MAXIMUM_FEE_UNITS - @celery_app.task(base=CriticalSQLAlchemyTask) def apply_gas_value_cache(address, method, value, tx_hash): @@ -85,8 +78,8 @@ def apply_gas_value_cache(address, method, value, tx_hash): def apply_gas_value_cache_local(address, method, value, tx_hash, session=None): - address = strip_0x(address) - tx_hash = strip_0x(tx_hash) + address = tx_normalize.executable_address(address) + tx_hash = tx_normalize.tx_hash(tx_hash) value = int(value) session = SessionBase.bind_session(session) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 149e1800..77a9fde4 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -10,7 +10,6 @@ from chainlib.chain import ChainSpec from chainlib.connection import RPCConnection from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.nonce import RPCNonceOracle -from chainlib.eth.gas import RPCGasOracle from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError @@ -30,7 +29,7 @@ class BaseTask(celery.Task): call_address = ZERO_ADDRESS trusted_addresses = [] create_nonce_oracle = RPCNonceOracle - create_gas_oracle = RPCGasOracle + create_gas_oracle = CacheGasOracle default_token_address = None default_token_symbol = None default_token_name = None