cic-eth: Add sanity checks for emergency shutdown / liveness tests

2021-04-23 21:02:51 +00:00 · 2021-04-23 21:02:51 +00:00 · 3129a78e06
commit 3129a78e06
parent 6b6ec8659b
23 changed files with 165 additions and 37 deletions
--- a/apps/cic-eth/cic_eth/admin/ctrl.py
+++ b/apps/cic-eth/cic_eth/admin/ctrl.py
@ -32,6 +32,8 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
    :returns: New lock state for address
    :rtype: number
    """
+    chain_str = '::'
+    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
    logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
@ -51,6 +53,8 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
    :returns: New lock state for address
    :rtype: number
    """
+    chain_str = '::'
+    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.reset(chain_str, flags, address=address)
    logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
@ -127,6 +131,8 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):

@celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
+    chain_str = '::'
+    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    session = SessionBase.create_session()
    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
--- a/apps/cic-eth/cic_eth/check/init.py
+++ b/apps/cic-eth/cic_eth/check/init.py
--- a/apps/cic-eth/cic_eth/check/db.py
+++ b/apps/cic-eth/cic_eth/check/db.py
@ -1,5 +1,6 @@
 from cic_eth.db.models.base import SessionBase

+
 def health(*args, **kwargs):
    session = SessionBase.create_session()
    session.execute('SELECT count(*) from alembic_version')
--- a/apps/cic-eth/cic_eth/check/gas.py
+++ b/apps/cic-eth/cic_eth/check/gas.py
@ -0,0 +1,48 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.chain import ChainSpec
+from chainlib.eth.gas import balance
+
+# local imports
+from cic_eth.db.models.role import AccountRole
+from cic_eth.db.models.base import SessionBase
+from cic_eth.db.enum import LockEnum
+from cic_eth.error import LockedError
+from cic_eth.admin.ctrl import check_lock
+
+logg = logging.getLogger().getChild(__name__)
+
+
+def health(*args, **kwargs):
+
+    session = SessionBase.create_session()
+
+    config = kwargs['config']
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
+
+    try:
+        check_lock(None, None, LockEnum.INIT)
+    except LockedError:
+        logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
+        return True
+
+    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    session.close()
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    o = balance(gas_provider)
+    r = rpc.do(o)
+    try: 
+        r = int(r, 16)
+    except TypeError:
+        r = int(r)
+    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
+    if r < gas_min:
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
+        return False
+
+    return True
--- a/apps/cic-eth/cic_eth/check/signer.py
+++ b/apps/cic-eth/cic_eth/check/signer.py
@ -0,0 +1,37 @@
+# standard imports
+import time
+import logging
+from urllib.error import URLError
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.sign import sign_message
+from chainlib.error import JSONRPCException
+
+logg = logging.getLogger().getChild(__name__)
+
+
+def health(*args, **kwargs):
+    blocked = True
+    max_attempts = 5
+    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
+    for i in range(max_attempts):
+        idx = i + 1
+        logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
+        try:
+            conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
+        except FileNotFoundError:
+            pass
+        except ConnectionError:
+            pass
+        except URLError:
+            pass
+        except JSONRPCException:
+            logg.debug('signer connection succeeded')
+            return True
+
+        if idx < max_attempts:
+            time.sleep(0.5)
+
+    return False
--- a/apps/cic-eth/cic_eth/db/enum.py
+++ b/apps/cic-eth/cic_eth/db/enum.py
@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum):
    QUEUE: Disable queueing new or modified transactions
    """
    STICKY=1
-    CREATE=2
-    SEND=4
-    QUEUE=8
-    QUERY=16
+    INIT=2
+    CREATE=4
+    SEND=8
+    QUEUE=16
+    QUERY=32
    ALL=int(0xfffffffffffffffe)


--- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
+++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
@ -5,8 +5,11 @@ Revises: 1f1b3b641d08
 Create Date: 2021-04-02 18:41:20.864265

 """
+import datetime
 from alembic import op
 import sqlalchemy as sa
+from chainlib.eth.constant import ZERO_ADDRESS
+from cic_eth.db.enum import LockEnum


 # revision identifiers, used by Alembic.
@ -23,10 +26,11 @@ def upgrade():
            sa.Column("address", sa.String(42), nullable=True),
            sa.Column('blockchain', sa.String),
            sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
-            sa.Column("date_created", sa.DateTime, nullable=False),
+            sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
            sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
            )
    op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))


 def downgrade():
--- a/apps/cic-eth/cic_eth/runnable/ctrl.py
+++ b/apps/cic-eth/cic_eth/runnable/ctrl.py
@ -59,6 +59,7 @@ args_override = {
        'CIC_CHAIN_SPEC': getattr(args, 'i'),
        }
 # override args
+config.dict_override(args_override, 'cli')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@ -67,7 +68,9 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi

 queue = args.q

-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = None
+if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)


@ -82,6 +85,9 @@ def lock_names_to_flag(s):

 # TODO: move each command to submodule
 def main():
+    chain_spec_dict = None
+    if chain_spec != None:
+        chain_spec_dict = chain_spec.asdict()
    if args.command == 'unlock':
        flags = lock_names_to_flag(args.flags)
        if not is_checksum_address(args.address):
@ -91,7 +97,7 @@ def main():
            'cic_eth.admin.ctrl.unlock',
            [
                None,
-                chain_spec.asdict(),
+                chain_spec_dict,
                args.address,
                flags,
                ],
@ -110,7 +116,7 @@ def main():
            'cic_eth.admin.ctrl.lock',
            [
                None,
-                chain_spec.asdict(),
+                chain_spec_dict,
                args.address,
                flags,
                ],
--- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py
+++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py
@ -15,8 +15,10 @@ from chainlib.connection import RPCConnection
 from chainlib.eth.connection import EthUnixSignerConnection
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
+from cic_eth_registry.error import UnknownContractError
 import liveness.linux

+
 # local imports
 from cic_eth.eth import (
        erc20,
@ -138,11 +140,15 @@ else:

 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
-RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
+#RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')

 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')

-liveness.linux.load(health_modules)
+#import cic_eth.checks.gas
+#if not cic_eth.checks.gas.health(config=config):
+#    raise RuntimeError()
+liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config)

 def main():
    argv = ['worker']
@ -166,7 +172,11 @@ def main():

    rpc = RPCConnection.connect(chain_spec, 'default')

+    try:
        connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    except UnknownContractError as e:
+        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+        sys.exit(1)

    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
    if trusted_addresses_src == None:
@ -175,6 +185,7 @@ def main():
    trusted_addresses = trusted_addresses_src.split(',')
    for address in trusted_addresses:
        logg.info('using trusted address {}'.format(address))
+
    connect_declarator(rpc, chain_spec, trusted_addresses)
    connect_token_registry(rpc, chain_spec)
   
--- a/apps/cic-eth/cic_eth/task.py
+++ b/apps/cic-eth/cic_eth/task.py
@ -94,3 +94,8 @@ class CriticalWeb3AndSignerTask(CriticalTask):
 def hello(self):
    time.sleep(0.1)
    return id(SessionBase.create_session)
+
+
+@celery_app.task()
+def check_health(self):
+    celery.app.control.shutdown()
--- a/apps/cic-eth/cic_eth/version.py
+++ b/apps/cic-eth/cic_eth/version.py
@ -10,7 +10,7 @@ version = (
        0,
        11,
        0,
-        'beta.7',
+        'beta.8',
        )

 version_object = semver.VersionInfo(
--- a/apps/cic-eth/config/cic.ini
+++ b/apps/cic-eth/config/cic.ini
@ -3,4 +3,5 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =
-health_modules = cic_eth.k8s.db
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
+run_dir = /run
--- a/apps/cic-eth/config/docker/cic.ini
+++ b/apps/cic-eth/config/docker/cic.ini
@ -3,3 +3,5 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 tx_retry_delay = 20
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
+run_dir = /run
--- a/apps/cic-eth/config/docker/eth.ini
+++ b/apps/cic-eth/config/docker/eth.ini
@ -1,8 +1,4 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_provider_address = 
-#chain_id = 
-abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
-account_accounts_index_writer = 
+health_modules = cic_eth.check.db,cic_eth.check.gas
+gas_gifter_minimum_balance = 10000000000000000000000
--- a/apps/cic-eth/config/docker/signer.ini
+++ b/apps/cic-eth/config/docker/signer.ini
@ -1,5 +1,5 @@
 [signer]
-socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
+socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
 secret = deedbeef
 database_name = signer_test
 dev_keys_path = 
--- a/apps/cic-eth/config/eth.ini
+++ b/apps/cic-eth/config/eth.ini
@ -1,8 +1,4 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:8545
-gas_provider_address = 
-#chain_id = 
-abi_dir = /usr/local/share/cic/solidity/abi
-account_accounts_index_writer = 
+gas_gifter_minimum_balance = 10000000000000000000000
+health_modules = cic_eth.check.db,cic_eth.check.gas
--- a/apps/cic-eth/requirements.txt
+++ b/apps/cic-eth/requirements.txt
@ -1,6 +1,6 @@
-cic-base==0.1.2a79+build.35e442bc
+cic-base==0.1.2b1
 celery==4.4.7
-crypto-dev-signer~=0.4.14b2
+crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
 cic-eth-registry~=0.5.4a16
 #cic-bancor~=0.0.6
--- a/apps/cic-eth/setup.cfg
+++ b/apps/cic-eth/setup.cfg
@ -38,7 +38,7 @@ packages =
 	cic_eth.runnable.daemons.filters
 	cic_eth.callbacks
 	cic_eth.sync
-	cic_eth.k8s
+	cic_eth.check
 scripts =
 	./scripts/migrate.py

--- a/apps/contract-migration/docker/Dockerfile
+++ b/apps/contract-migration/docker/Dockerfile
@ -57,8 +57,8 @@ WORKDIR /home/grassroots
 USER grassroots

 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a77
-ARG cic_eth_version=0.11.0b6
+ARG cic_base_version=0.1.2a79
+ARG cic_eth_version=0.11.0b8+build.c2286e5c
 ARG sarafu_faucet_version=0.0.2a28
 ARG sarafu_token_version==0.0.1a6
 ARG cic_contracts_version=0.0.2a2
--- a/apps/contract-migration/seed_cic_eth.sh
+++ b/apps/contract-migration/seed_cic_eth.sh
@ -13,6 +13,12 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi
 DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
 CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} 
 ETH_PASSPHRASE=''
+DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
+if [ $DEV_TOKEN_TYPE = 'giftable' ]; then
+	token_symbol='GFT'
+else
+	token_symbol='SRF'
+fi

 # Debug flag
 DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@ -46,6 +52,7 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H
 echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER

+
 >&2 echo "create account for sarafu gifter"
 DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
@ -97,5 +104,10 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS

 #echo -n 0 > $init_level_file

+# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
+cic-eth-ctl -i :: unlock INIT
+cic-eth-ctl -i :: unlock SEND
+cic-eth-ctl -i :: unlock QUEUE
+
 set +a
 set +e
--- a/apps/util/liveness/liveness/linux.py
+++ b/apps/util/liveness/liveness/linux.py
@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs
        checks.append(module)

    for check in checks:
-        r = check.health(args, kwargs)
+        r = check.health(*args, **kwargs)
        if r == False:
            raise RuntimeError('liveness check {} failed'.format(str(check)))
        logg.info('liveness check passed: {}'.format(str(check)))
--- a/apps/util/liveness/setup.py
+++ b/apps/util/liveness/setup.py
@ -1,7 +1,7 @@
 from setuptools import setup
 setup(
        name='liveness',
-        version='0.0.1a6',
+        version='0.0.1a7',
        packages=['liveness'],
        include_package_data=True,
        )
--- a/apps/util/liveness/tests/test_imports.py
+++ b/apps/util/liveness/tests/test_imports.py
@ -116,7 +116,9 @@ class TestImports(unittest.TestCase):

    def test_args(self):
        checks = ['tests.imports.import_args']
-        liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42})
+        aargs=['foo']
+        kwaargs={'bar': 42}
+        liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs)
        f = open(self.pid_path, 'r')
        r = f.read()
        f.close()