Merge branch 'lash/tmp-bloxberg-seeding' into lash/bloxberg-seeding

This commit is contained in:
nolash 2021-10-29 07:35:08 +02:00
commit 258ed420b8
11 changed files with 315 additions and 10 deletions

View File

@ -23,7 +23,7 @@ def upgrade():
op.create_table( op.create_table(
'lock', 'lock',
sa.Column('id', sa.Integer, primary_key=True), sa.Column('id', sa.Integer, primary_key=True),
sa.Column("address", sa.String(42), nullable=True), sa.Column("address", sa.String, nullable=True),
sa.Column('blockchain', sa.String), sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

View File

@ -0,0 +1,31 @@
"""Add gas cache
Revision ID: c91cafc3e0c1
Revises: aee12aeb47ec
Create Date: 2021-10-28 20:45:34.239865
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'c91cafc3e0c1'
down_revision = 'aee12aeb47ec'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'gas_cache',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column("address", sa.String, nullable=False),
sa.Column("tx_hash", sa.String, nullable=True),
sa.Column("method", sa.String, nullable=True),
sa.Column("value", sa.BIGINT(), nullable=False),
)
def downgrade():
op.drop_table('gas_cache')

View File

@ -0,0 +1,27 @@
# standard imports
import logging
# external imports
from sqlalchemy import Column, String, NUMERIC
# local imports
from .base import SessionBase
logg = logging.getLogger(__name__)
class GasCache(SessionBase):
"""Provides gas budget cache for token operations
"""
__tablename__ = 'gas_cache'
address = Column(String())
tx_hash = Column(String())
method = Column(String())
value = Column(NUMERIC())
def __init__(self, address, method, value, tx_hash):
self.address = address
self.tx_hash = tx_hash
self.method = method
self.value = value

View File

@ -12,7 +12,7 @@ from cic_eth.error import (
IntegrityError, IntegrityError,
) )
logg = logging.getLogger() logg = logging.getLogger(__name__)
class Nonce(SessionBase): class Nonce(SessionBase):
@ -21,7 +21,7 @@ class Nonce(SessionBase):
__tablename__ = 'nonce' __tablename__ = 'nonce'
nonce = Column(Integer) nonce = Column(Integer)
address_hex = Column(String(42)) address_hex = Column(String(40))
@staticmethod @staticmethod

View File

@ -10,6 +10,9 @@ from chainlib.eth.tx import (
TxFormat, TxFormat,
unpack, unpack,
) )
from chainlib.eth.contract import (
ABIContractEncoder,
)
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token from cic_eth_registry.erc20 import ERC20Token
from hexathon import ( from hexathon import (
@ -155,7 +158,7 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
session = self.create_session() session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) gas_oracle = self.create_gas_oracle(rpc, t['address'], MaxGasOracle.gas)
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
try: try:
(tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED) (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@ -226,6 +229,11 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
session = self.create_session() session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
enc = ABIContractEncoder()
enc.method('transferFrom')
method = enc.get()
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
try: try:

View File

@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx from chainqueue.db.models.otx import Otx
# local imports # local imports
from cic_eth.db.models.gas_cache import GasCache
from cic_eth.db.models.role import AccountRole from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.error import ( from cic_eth.error import (
@ -71,12 +72,33 @@ celery_app = celery.current_app
logg = logging.getLogger() logg = logging.getLogger()
MAXIMUM_FEE_UNITS = 8000000
class MaxGasOracle: @celery_app.task(base=CriticalSQLAlchemyTask)
def apply_gas_value_cache(address, method, value, tx_hash):
return apply_gas_value_cache_local(address, method, value, tx_hash)
def gas(code=None):
return MAXIMUM_FEE_UNITS def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
address = tx_normalize.executable_address(address)
tx_hash = tx_normalize.tx_hash(tx_hash)
value = int(value)
session = SessionBase.bind_session(session)
q = session.query(GasCache)
q = q.filter(GasCache.address==address)
q = q.filter(GasCache.method==method)
o = q.first()
if o == None:
o = GasCache(address, method, value, tx_hash)
elif tx.gas_used > o.value:
o.value = value
o.tx_hash = strip_0x(tx_hash)
session.add(o)
session.commit()
SessionBase.release_session(session)
def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None): def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):

View File

@ -0,0 +1,46 @@
# external imports
from chainlib.eth.gas import RPCGasOracle
# local imports
from cic_eth.db.models.gas_cache import GasCache
MAXIMUM_FEE_UNITS = 8000000
class MaxGasOracle(RPCGasOracle):
def get_fee_units(code=None):
return MAXIMUM_FEE_UNITS
class CacheGasOracle(MaxGasOracle):
"""Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
:todo: instead of max units, connect a pluggable gas heuristics engine.
"""
def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
self.value = None
self.address = address
self.method = method
address = tx_normalize.executable_address(address)
session = SessionBase.bind_session(session)
q = session.query(GasCache)
q = q.filter(GasCache.address=address)
if method != None:
method = strip_0x(method)
q = q.filter(GasCache.method=method)
o = q.first()
if o != None:
self.value = int(o.value)
SessionBase.release_session(session)
def get_fee_units(self, code=None):
if self.value != None:
logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
return self.value
return super(CacheGasOracle, self).get_fee_units(code=code)

View File

@ -0,0 +1,63 @@
# standard imports
import logging
# external imports
from eth_erc20 import ERC20
from chainlib.eth.contract import (
ABIContractEncoder,
ABIContractType,
)
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.address import is_same_address
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from eth_token_index import TokenUniqueSymbolIndex
import celery
# local imports
from .base import SyncFilter
#logg = logging.getLogger(__name__)
logg = logging.getLogger()
class TokenFilter(SyncFilter):
def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
self.queue = queue
self.chain_spec = chain_spec
self.caller_address = call_address
def filter(self, conn, block, tx, db_session=None):
if not tx.payload:
return (None, None)
try:
r = ERC20.parse_transfer_request(tx.payload)
except RequestMismatchException:
return (None, None)
token_address = tx.inputs[0]
token = ERC20Token(self.chain_spec, conn, token_address)
registry = CICRegistry(self.chain_spec, conn)
r = registry.by_name(token.symbol, sender_address=self.caller_address)
if is_same_address(r, ZERO_ADDRESS):
return None
enc = ABIContractEncoder()
enc.method('transfer')
method = enc.get()
s = celery.signature(
'cic_eth.eth.gas.apply_gas_value_cache',
[
token_address,
method,
tx.gas_used,
tx.hash,
],
queue=self.queue,
)
return s.apply_async()

View File

@ -10,13 +10,13 @@ from chainlib.chain import ChainSpec
from chainlib.connection import RPCConnection from chainlib.connection import RPCConnection
from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import RPCGasOracle
from cic_eth_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_eth_registry.error import UnknownContractError from cic_eth_registry.error import UnknownContractError
# local imports # local imports
from cic_eth.error import SeppukuError from cic_eth.error import SeppukuError
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
from cic_eth.eth.util import CacheGasOracle
#logg = logging.getLogger().getChild(__name__) #logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger() logg = logging.getLogger()
@ -30,7 +30,7 @@ class BaseTask(celery.Task):
call_address = ZERO_ADDRESS call_address = ZERO_ADDRESS
trusted_addresses = [] trusted_addresses = []
min_fee_price = 1 min_fee_price = 1
create_nonce_oracle = RPCNonceOracle create_gas_oracle = CacheGasOracle
default_token_address = None default_token_address = None
default_token_symbol = None default_token_symbol = None
default_token_name = None default_token_name = None

View File

@ -0,0 +1,96 @@
# external imports
from eth_erc20 import ERC20
from chainlib.connection import RPCConnection
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.gas import (
Gas,
OverrideGasOracle,
)
from chainlib.eth.tx import (
TxFormat,
receipt,
raw,
unpack,
Tx,
)
from chainlib.eth.block import (
Block,
block_latest,
block_by_number,
)
from chainlib.eth.contract import ABIContractEncoder
from hexathon import strip_0x
from eth_token_index import TokenUniqueSymbolIndex
# local imports
from cic_eth.runnable.daemons.filters.token import TokenFilter
from cic_eth.db.models.gas_cache import GasCache
from cic_eth.db.models.base import SessionBase
def test_filter_gas(
default_chain_spec,
init_database,
eth_rpc,
eth_signer,
contract_roles,
agent_roles,
token_roles,
foo_token,
token_registry,
register_lookups,
celery_worker,
cic_registry,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
o = raw(tx_signed_raw_hex)
eth_rpc.do(o)
o = receipt(tx_hash_hex)
rcpt = eth_rpc.do(o)
assert rcpt['status'] == 1
fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE'])
o = block_latest()
r = eth_rpc.do(o)
o = block_by_number(r, include_tx=False)
r = eth_rpc.do(o)
block = Block(r)
block.txs = [tx_hash_hex]
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
tx = Tx(tx_src, block=block)
tx.apply_receipt(rcpt)
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
assert t == None
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
(tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token)
eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
assert r['status'] == 1
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
r = t.get_leaf()
assert t.successful()
q = init_database.query(GasCache)
q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex))
o = q.first()
assert o.address == strip_0x(foo_token)
assert o.value > 0
enc = ABIContractEncoder()
enc.method('transfer')
method = enc.get()
assert o.method == method

View File

@ -18,6 +18,18 @@ fi
must_address "$CIC_REGISTRY_ADDRESS" "registry" must_address "$CIC_REGISTRY_ADDRESS" "registry"
must_eth_rpc must_eth_rpc
default_token=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw DefaultToken`
h=`erc20-transfer -u -e $default_token -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -y $WALLET_KEY_FILE -s 10`
r=`eth-receipt
set +e
set +a
exit 0
# get required addresses from registries # get required addresses from registries
token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry` token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry`
accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry` accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`