Add gas cache backend, test, filter

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -23,7 +23,7 @@ def upgrade():
     op.create_table(
             'lock',
             sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String(42), nullable=True),
+            sa.Column("address", sa.String, nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py

@@ -0,0 +1,31 @@
+"""Add gas cache
+
+Revision ID: c91cafc3e0c1
+Revises: aee12aeb47ec
+Create Date: 2021-10-28 20:45:34.239865
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c91cafc3e0c1'
+down_revision = 'aee12aeb47ec'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+            'gas_cache',
+            sa.Column('id', sa.Integer, primary_key=True),
+            sa.Column("address", sa.String, nullable=False),
+            sa.Column("tx_hash", sa.String, nullable=True),
+            sa.Column("method", sa.String, nullable=True),
+            sa.Column("value", sa.BIGINT(), nullable=False),
+            )
+
+
+def downgrade():
+    op.drop_table('gas_cache')

apps/cic-eth/cic_eth/db/models/gas_cache.py

@@ -0,0 +1,27 @@
+# standard imports
+import logging
+
+# external imports
+from sqlalchemy import Column, String, NUMERIC
+
+# local imports
+from .base import SessionBase
+
+logg = logging.getLogger(__name__)
+
+
+class GasCache(SessionBase):
+    """Provides gas budget cache for token operations
+    """
+    __tablename__ = 'gas_cache'
+
+    address = Column(String())
+    tx_hash = Column(String())
+    method = Column(String())
+    value = Column(NUMERIC())
+
+    def __init__(self, address, method, value, tx_hash):
+        self.address = address
+        self.tx_hash = tx_hash
+        self.method = method
+        self.value = value

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -12,7 +12,7 @@ from cic_eth.error import (
         IntegrityError,
         )
 
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
 
 
 class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
     __tablename__ = 'nonce'
 
     nonce = Column(Integer)
-    address_hex = Column(String(42))
+    address_hex = Column(String(40))
 
 
     @staticmethod

apps/cic-eth/cic_eth/eth/gas.py

@@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
+from cic_eth.db.models.gas_cache import GasCache
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 from cic_eth.error import (
@@ -78,6 +79,34 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def apply_gas_value_cache(address, method, value, tx_hash):
+    return apply_gas_value_cache_local(address, method, value, tx_hash)
+
+
+def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
+    address = strip_0x(address)
+    tx_hash = strip_0x(tx_hash)
+    value = int(value)
+
+    session = SessionBase.bind_session(session)
+    q = session.query(GasCache)
+    q = q.filter(GasCache.address==address)
+    q = q.filter(GasCache.method==method)
+    o = q.first()
+
+    if o == None:
+        o = GasCache(address, method, value, tx_hash)
+    elif tx.gas_used > o.value:
+        o.value = value
+        o.tx_hash = strip_0x(tx_hash)
+
+    session.add(o)
+    session.commit()
+
+    SessionBase.release_session(session)
+
+
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 

apps/cic-eth/cic_eth/runnable/daemons/filters/token.py

@@ -0,0 +1,43 @@
+# external imports
+from eth_erc20 import ERC20
+from chainlib.eth.contract import (
+        ABIContractEncoder,
+        ABIContractType,
+        )
+import celery
+
+# local imports
+from .base import SyncFilter
+
+
+class TokenFilter(SyncFilter):
+
+    def __init__(self, chain_spec, queue):
+        self.queue = queue
+        self.chain_spec = chain_spec
+
+
+    def filter(self, conn, block, tx, db_session=None):
+        if not tx.payload:
+            return (None, None)
+
+        try:
+            r = ERC20.parse_transfer_request(tx.payload)
+        except RequestMismatchException:
+            return (None, None)
+        
+        enc = ABIContractEncoder()
+        enc.method('transfer')
+        method = enc.get()
+
+        s = celery.signature(
+                'cic_eth.eth.gas.apply_gas_value_cache',
+                [
+                    tx.inputs[0],
+                    method,
+                    tx.gas_used,
+                    tx.hash,
+                    ],
+                queue=self.queue,
+                )
+        return s.apply_async()

apps/cic-eth/tests/filters/test_token_filter.py

@@ -0,0 +1,80 @@
+# external imports
+from eth_erc20 import ERC20
+from chainlib.connection import RPCConnection
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import (
+        Gas,
+        OverrideGasOracle,
+        )
+from chainlib.eth.tx import (
+        TxFormat,
+        receipt,
+        raw,
+        unpack,
+        Tx,
+        )
+from chainlib.eth.block import (
+        Block,
+        block_latest,
+        block_by_number,
+        )
+from chainlib.eth.contract import ABIContractEncoder
+from hexathon import strip_0x
+
+# local imports
+from cic_eth.runnable.daemons.filters.token import TokenFilter
+from cic_eth.db.models.gas_cache import GasCache
+from cic_eth.db.models.base import SessionBase
+
+
+def test_filter_gas(
+        default_chain_spec,
+        init_database,
+        eth_rpc,
+        eth_signer,
+        agent_roles,
+        token_roles,
+        foo_token,
+        celery_session_worker,
+    ):
+
+    rpc = RPCConnection.connect(default_chain_spec, 'default')
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
+    gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
+    o = raw(tx_signed_raw_hex)
+    eth_rpc.do(o)
+    o = receipt(tx_hash_hex)
+    rcpt = eth_rpc.do(o)
+    assert rcpt['status'] == 1
+
+    fltr = TokenFilter(default_chain_spec, queue=None)
+
+    o = block_latest()
+    r = eth_rpc.do(o)
+    o = block_by_number(r, include_tx=False)
+    r = eth_rpc.do(o)
+    block = Block(r)
+    block.txs = [tx_hash_hex]
+
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
+    tx = Tx(tx_src, block=block)
+    tx.apply_receipt(rcpt)
+    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
+    r = t.get_leaf()
+    assert t.successful()
+
+    q = init_database.query(GasCache)
+    q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex))
+    o = q.first()
+
+    assert o.address == strip_0x(foo_token)
+    assert o.value > 0
+
+    enc = ABIContractEncoder()
+    enc.method('transfer')
+    method = enc.get()
+
+    assert o.method == method

apps/contract-migration/4_init_custodial.sh

@@ -18,6 +18,18 @@ fi
 must_address "$CIC_REGISTRY_ADDRESS" "registry"
 must_eth_rpc
 
+
+
+default_token=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw DefaultToken`
+h=`erc20-transfer -u -e $default_token -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -y $WALLET_KEY_FILE -s 10`
+r=`eth-receipt 
+
+set +e
+set +a
+
+exit 0
+
+
 # get required addresses from registries
 token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry`
 accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`