Better gas gifting amounts and thresholds estimation, fix broken cic-eth imports

apps/cic-eth/cic_eth/check/gas.py

@@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.admin.ctrl import check_lock
+from cic_eth.eth.gas import have_gas_minimum
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 
 def health(*args, **kwargs):
@@ -31,18 +32,15 @@ def health(*args, **kwargs):
         return True
 
     gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
+    if config.get('ETH_MIN_FEE_PRICE'):
+        min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
+
+    r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
+
     session.close()
     
-    rpc = RPCConnection.connect(chain_spec, 'default')
+    if not r:
-    o = balance(gas_provider)
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
-    r = rpc.do(o)
-    try: 
-        r = int(r, 16)
-    except TypeError:
-        r = int(r)
-    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
-    if r < gas_min:
-        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
-        return False
 
-    return True
+    return r

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -2,5 +2,5 @@
 registry_address = 
 trust_address =
 default_token_symbol =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
+health_modules = cic_eth.check.db,cic_eth.check.start,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,3 +1,6 @@
 [eth]
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_holder_minimum_units = 180000
+gas_holder_refill_units = 15
+gas_holder_refill_threshold = 3
+gas_gifter_refill_buffer = 3
 min_fee_price = 1

apps/cic-eth/cic_eth/enum.py

@@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum):
 class LockEnum(enum.IntEnum):
     """
     STICKY: When set, reset is not possible
+    INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
+    START: When set, startup is not possible, regardless of state
     CREATE: Disable creation of accounts
     SEND: Disable sending to network
     QUEUE: Disable queueing new or modified transactions
+    QUERY: Disable all queue state and transaction queries
     """
     STICKY=1
     INIT=2
@@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum):
     SEND=8
     QUEUE=16
     QUERY=32
-    ALL=int(0xfffffffffffffffe)
+    START=int(0x80000000)
+    ALL=int(0x7ffffffe)
 
 
 def status_str(v, bits_only=False):

apps/cic-eth/cic_eth/error.py

@@ -64,8 +64,10 @@ class LockedError(Exception):
 
 class SeppukuError(Exception):
     """Exception base class for all errors that should cause system shutdown
-
     """
+    def __init__(self, message, lockdown=False):
+        self.message = message
+        self.lockdown = lockdown
 
 
 class SignerError(SeppukuError):

apps/cic-eth/cic_eth/eth/gas.py

@@ -65,6 +65,7 @@ from cic_eth.encode import (
         ZERO_ADDRESS_NORMAL,
         unpack_normal,
         )
+from cic_eth.error import SeppukuError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -78,6 +79,22 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
+    if rpc == None:
+        rpc = RPCConnection.connect(chain_spec, 'default')
+    o = balance(add_0x(address))
+    r = rpc.do(o)
+    try: 
+        r = int(r)
+    except ValueError:
+        r = strip_0x(r)
+        r = int(r, 16)
+    logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
+    if r < min_gas:
+        return False
+    return True
+
+
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -357,6 +374,13 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # set up evm RPC connection
     rpc = RPCConnection.connect(chain_spec, 'default')
 
+    # check the gas balance of the gifter
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
+        raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
+
+    if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
+        logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
+    
     # set up transaction builder
     nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc)

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -67,7 +67,10 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import BaseTask
+from cic_eth.task import (
+        BaseTask,
+        CriticalWeb3Task,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -215,8 +218,7 @@ def main():
     argv.append('-n')
     argv.append(config.get('CELERY_QUEUE'))
 
-    if config.get('ETH_MIN_FEE_PRICE'):
+    # TODO: More elegant way of setting queue-wide settings
-        BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
     BaseTask.default_token_symbol = default_token_symbol
     BaseTask.default_token_address = default_token_address
     default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
@@ -224,6 +226,14 @@ def main():
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
     BaseTask.trusted_addresses = trusted_addresses
+    CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
+    CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
+    CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
+    if config.get('ETH_MIN_FEE_PRICE'):
+        BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
+        CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
+        CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
+        CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
 
     BaseTask.run_dir = config.get('CIC_RUN_DIR')
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

apps/cic-eth/cic_eth/task.py

@@ -83,19 +83,18 @@ class CriticalWeb3Task(CriticalTask):
     autoretry_for = (
         ConnectionError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
+    safe_gas_threshold_amount = 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
+    safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100
 
 
-class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
+class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
-    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
 class CriticalSQLAlchemyAndSignerTask(CriticalTask):
@@ -105,13 +104,10 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.ResourceClosedError,
         ) 
 
-class CriticalWeb3AndSignerTask(CriticalTask):
+class CriticalWeb3AndSignerTask(CriticalWeb3Task):
     autoretry_for = (
         ConnectionError,
         )
-    safe_gas_threshold_amount = 2000000000 * 60000 * 3
-    safe_gas_refill_amount = safe_gas_threshold_amount * 5 
-
     
 @celery_app.task()
 def check_health(self):

apps/contract-migration/config.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -a
+set -e
 
 if [ -z $DEV_DATA_DIR ]; then
 	export DEV_DATA_DIR=`mktemp -d`
@@ -33,6 +34,7 @@ else
 	fi
 	rm $bash_debug_flag -f ${DEV_DATA_DIR}/env_reset
 	rm $bash_debug_flag -f $noncefile
+	export SYNCER_OFFSET=`eth-info --raw block`
 	confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset
 fi
 
@@ -43,6 +45,7 @@ export CIC_TRUST_ADDRESS=${CIC_TRUST_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL
 export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 
+
 if [ ! -f $noncefile ]; then
 	nonce=`eth-count -p $RPC_PROVIDER $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
 	>&2 echo -e "\033[;96mUsing contract deployer address $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER with nonce $nonce\033[;39m"
@@ -55,4 +58,5 @@ fi
 # Migration variable processing
 confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset
 
+set +e
 set +a

apps/contract-migration/config/config.ini

@@ -25,3 +25,6 @@ port =
 [cic]
 registry_address =
 trust_address =
+
+[syncer]
+offset =

apps/contract-migration/requirements.txt

@@ -1,5 +1,5 @@
 cic-eth[tools]==0.12.4a13
-chainlib-eth>=0.0.10a15,<0.1.0
+chainlib-eth>=0.0.10a17,<0.1.0
 eth-erc20>=0.1.2a3,<0.2.0
 erc20-demurrage-token>=0.0.5a2,<0.1.0
 eth-address-index>=0.2.4a1,<0.3.0

apps/data-seeding/cic_eth/import_users.py

@@ -40,7 +40,7 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, help='config override directory')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
 argparser.add_argument('-f', action='store_true', help='force clear previous state')
-argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec')
 argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
 argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
 argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -227,9 +227,10 @@ if __name__ == '__main__':
                     )
             os.makedirs(os.path.dirname(filepath), exist_ok=True)
            
-            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
+            sub_old_chain_str = '{}:{}'.format(old_chain_spec.network_id(), old_chain_spec.common_name())
+            logg.debug('u id {}'.format(u.identities))
             f = open(filepath, 'w')
-            k = u.identities['evm'][sub_old_chain_str][0]
+            k = u.identities['evm'][old_chain_spec.fork()][sub_old_chain_str][0]
             tag_data = {'tags': user_tags[strip_0x(k)]}
             f.write(json.dumps(tag_data))
             f.close()

apps/data-seeding/requirements.txt

@@ -4,7 +4,7 @@ cic-types~=0.2.1a2
 funga>=0.5.1a1,<=0.5.15
 faker==4.17.1
 chainsyncer~=0.0.7a3
-chainlib-eth~=0.0.10a10
+chainlib-eth~=0.0.10a18
 eth-address-index~=0.2.4a1
 eth-contract-registry~=0.6.3a3
 eth-accounts-index~=0.1.2a3