From 546256c86aef6bd27c08404b056ec420c81d33af Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 28 Oct 2021 13:34:39 +0200 Subject: [PATCH] Better gas gifting amounts and thresholds estimation, fix broken cic-eth imports --- apps/cic-eth/cic_eth/check/gas.py | 26 +++++++++---------- apps/cic-eth/cic_eth/data/config/cic.ini | 2 +- apps/cic-eth/cic_eth/data/config/eth.ini | 5 +++- apps/cic-eth/cic_eth/enum.py | 6 ++++- apps/cic-eth/cic_eth/error.py | 4 ++- apps/cic-eth/cic_eth/eth/gas.py | 24 +++++++++++++++++ .../cic_eth/runnable/daemons/tasker.py | 16 +++++++++--- apps/cic-eth/cic_eth/task.py | 14 ++++------ apps/contract-migration/config.sh | 4 +++ apps/contract-migration/config/config.ini | 3 +++ apps/contract-migration/requirements.txt | 2 +- apps/data-seeding/cic_eth/import_users.py | 7 ++--- apps/data-seeding/requirements.txt | 2 +- 13 files changed, 80 insertions(+), 35 deletions(-) diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py index 4add981b..56fe23ed 100644 --- a/apps/cic-eth/cic_eth/check/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase from cic_eth.db.enum import LockEnum from cic_eth.error import LockedError from cic_eth.admin.ctrl import check_lock +from cic_eth.eth.gas import have_gas_minimum -logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger(__name__) def health(*args, **kwargs): @@ -31,18 +32,15 @@ def health(*args, **kwargs): return True gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) + min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER')) + if config.get('ETH_MIN_FEE_PRICE'): + min_gas *= int(config.get('ETH_MIN_FEE_PRICE')) + + r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session) + session.close() + + if not r: + logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas)) - rpc = RPCConnection.connect(chain_spec, 'default') - o = balance(gas_provider) - r = rpc.do(o) - try: - r = int(r, 16) - except TypeError: - r = int(r) - gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE')) - if r < gas_min: - logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min)) - return False - - return True + return r diff --git a/apps/cic-eth/cic_eth/data/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini index 103566ff..3de0748a 100644 --- a/apps/cic-eth/cic_eth/data/config/cic.ini +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -2,5 +2,5 @@ registry_address = trust_address = default_token_symbol = -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas +health_modules = cic_eth.check.db,cic_eth.check.start,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas run_dir = /run diff --git a/apps/cic-eth/cic_eth/data/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini index 6252960b..2de2d1bd 100644 --- a/apps/cic-eth/cic_eth/data/config/eth.ini +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -1,3 +1,6 @@ [eth] -gas_gifter_minimum_balance = 10000000000000000000000 +gas_holder_minimum_units = 180000 +gas_holder_refill_units = 15 +gas_holder_refill_threshold = 3 +gas_gifter_refill_buffer = 3 min_fee_price = 1 diff --git a/apps/cic-eth/cic_eth/enum.py b/apps/cic-eth/cic_eth/enum.py index 2ce3eccd..3cc85ff8 100644 --- a/apps/cic-eth/cic_eth/enum.py +++ b/apps/cic-eth/cic_eth/enum.py @@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum): class LockEnum(enum.IntEnum): """ STICKY: When set, reset is not possible + INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance) + START: When set, startup is not possible, regardless of state CREATE: Disable creation of accounts SEND: Disable sending to network QUEUE: Disable queueing new or modified transactions + QUERY: Disable all queue state and transaction queries """ STICKY=1 INIT=2 @@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum): SEND=8 QUEUE=16 QUERY=32 - ALL=int(0xfffffffffffffffe) + START=int(0x80000000) + ALL=int(0x7ffffffe) def status_str(v, bits_only=False): diff --git a/apps/cic-eth/cic_eth/error.py b/apps/cic-eth/cic_eth/error.py index 7e8dbfa6..9c0689a8 100644 --- a/apps/cic-eth/cic_eth/error.py +++ b/apps/cic-eth/cic_eth/error.py @@ -64,8 +64,10 @@ class LockedError(Exception): class SeppukuError(Exception): """Exception base class for all errors that should cause system shutdown - """ + def __init__(self, message, lockdown=False): + self.message = message + self.lockdown = lockdown class SignerError(SeppukuError): diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 435949fc..c27e9478 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -65,6 +65,7 @@ from cic_eth.encode import ( ZERO_ADDRESS_NORMAL, unpack_normal, ) +from cic_eth.error import SeppukuError celery_app = celery.current_app logg = logging.getLogger() @@ -78,6 +79,22 @@ class MaxGasOracle: return MAXIMUM_FEE_UNITS +def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None): + if rpc == None: + rpc = RPCConnection.connect(chain_spec, 'default') + o = balance(add_0x(address)) + r = rpc.do(o) + try: + r = int(r) + except ValueError: + r = strip_0x(r) + r = int(r, 16) + logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas)) + if r < min_gas: + return False + return True + + def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher. @@ -357,6 +374,13 @@ def refill_gas(self, recipient_address, chain_spec_dict): # set up evm RPC connection rpc = RPCConnection.connect(chain_spec, 'default') + # check the gas balance of the gifter + if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount): + raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider)) + + if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance): + logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider)) + # set up transaction builder nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session) gas_oracle = self.create_gas_oracle(rpc) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 4826d410..d2751e4a 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -67,7 +67,10 @@ from cic_eth.registry import ( connect_declarator, connect_token_registry, ) -from cic_eth.task import BaseTask +from cic_eth.task import ( + BaseTask, + CriticalWeb3Task, + ) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -215,8 +218,7 @@ def main(): argv.append('-n') argv.append(config.get('CELERY_QUEUE')) - if config.get('ETH_MIN_FEE_PRICE'): - BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE')) + # TODO: More elegant way of setting queue-wide settings BaseTask.default_token_symbol = default_token_symbol BaseTask.default_token_address = default_token_address default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address)) @@ -224,6 +226,14 @@ def main(): BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name BaseTask.trusted_addresses = trusted_addresses + CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS')) + CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD')) + CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER')) + if config.get('ETH_MIN_FEE_PRICE'): + BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE')) + CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price + CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price + CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price BaseTask.run_dir = config.get('CIC_RUN_DIR') logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 0eaae980..c026ea1f 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -83,19 +83,18 @@ class CriticalWeb3Task(CriticalTask): autoretry_for = ( ConnectionError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 + safe_gas_threshold_amount = 60000 * 3 safe_gas_refill_amount = safe_gas_threshold_amount * 5 + safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100 -class CriticalSQLAlchemyAndWeb3Task(CriticalTask): +class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task): autoretry_for = ( sqlalchemy.exc.DatabaseError, sqlalchemy.exc.TimeoutError, ConnectionError, sqlalchemy.exc.ResourceClosedError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 - safe_gas_refill_amount = safe_gas_threshold_amount * 5 class CriticalSQLAlchemyAndSignerTask(CriticalTask): @@ -105,14 +104,11 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask): sqlalchemy.exc.ResourceClosedError, ) -class CriticalWeb3AndSignerTask(CriticalTask): +class CriticalWeb3AndSignerTask(CriticalWeb3Task): autoretry_for = ( ConnectionError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 - safe_gas_refill_amount = safe_gas_threshold_amount * 5 - - + @celery_app.task() def check_health(self): pass diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index ea69a52b..a5071aa7 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -1,6 +1,7 @@ #!/bin/bash set -a +set -e if [ -z $DEV_DATA_DIR ]; then export DEV_DATA_DIR=`mktemp -d` @@ -33,6 +34,7 @@ else fi rm $bash_debug_flag -f ${DEV_DATA_DIR}/env_reset rm $bash_debug_flag -f $noncefile + export SYNCER_OFFSET=`eth-info --raw block` confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset fi @@ -43,6 +45,7 @@ export CIC_TRUST_ADDRESS=${CIC_TRUST_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} + if [ ! -f $noncefile ]; then nonce=`eth-count -p $RPC_PROVIDER $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER` >&2 echo -e "\033[;96mUsing contract deployer address $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER with nonce $nonce\033[;39m" @@ -55,4 +58,5 @@ fi # Migration variable processing confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset +set +e set +a diff --git a/apps/contract-migration/config/config.ini b/apps/contract-migration/config/config.ini index 78edef76..2d8e93a5 100644 --- a/apps/contract-migration/config/config.ini +++ b/apps/contract-migration/config/config.ini @@ -25,3 +25,6 @@ port = [cic] registry_address = trust_address = + +[syncer] +offset = diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 61ec770d..cd74f5a2 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,5 +1,5 @@ cic-eth[tools]==0.12.4a13 -chainlib-eth>=0.0.10a15,<0.1.0 +chainlib-eth>=0.0.10a17,<0.1.0 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 eth-address-index>=0.2.4a1,<0.3.0 diff --git a/apps/data-seeding/cic_eth/import_users.py b/apps/data-seeding/cic_eth/import_users.py index 1322c638..bf66c382 100644 --- a/apps/data-seeding/cic_eth/import_users.py +++ b/apps/data-seeding/cic_eth/import_users.py @@ -40,7 +40,7 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, help='config override directory') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-f', action='store_true', help='force clear previous state') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec') argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') @@ -227,9 +227,10 @@ if __name__ == '__main__': ) os.makedirs(os.path.dirname(filepath), exist_ok=True) - sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) + sub_old_chain_str = '{}:{}'.format(old_chain_spec.network_id(), old_chain_spec.common_name()) + logg.debug('u id {}'.format(u.identities)) f = open(filepath, 'w') - k = u.identities['evm'][sub_old_chain_str][0] + k = u.identities['evm'][old_chain_spec.fork()][sub_old_chain_str][0] tag_data = {'tags': user_tags[strip_0x(k)]} f.write(json.dumps(tag_data)) f.close() diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index dcec83c8..f8e18738 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -4,7 +4,7 @@ cic-types~=0.2.1a2 funga>=0.5.1a1,<=0.5.15 faker==4.17.1 chainsyncer~=0.0.7a3 -chainlib-eth~=0.0.10a10 +chainlib-eth~=0.0.10a18 eth-address-index~=0.2.4a1 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3