From 3129a78e067ac7a9681bd2ae9981a6c149b6bc23 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Fri, 23 Apr 2021 21:02:51 +0000 Subject: [PATCH] cic-eth: Add sanity checks for emergency shutdown / liveness tests --- apps/cic-eth/cic_eth/admin/ctrl.py | 12 +++-- apps/cic-eth/cic_eth/check/__init__.py | 0 apps/cic-eth/cic_eth/{k8s => check}/db.py | 1 + apps/cic-eth/cic_eth/check/gas.py | 48 +++++++++++++++++++ apps/cic-eth/cic_eth/check/signer.py | 37 ++++++++++++++ apps/cic-eth/cic_eth/db/enum.py | 9 ++-- .../default/versions/75d4767b3031_lock.py | 6 ++- apps/cic-eth/cic_eth/runnable/ctrl.py | 12 +++-- .../cic_eth/runnable/daemons/tasker.py | 17 +++++-- apps/cic-eth/cic_eth/task.py | 5 ++ apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/config/cic.ini | 3 +- apps/cic-eth/config/docker/cic.ini | 2 + apps/cic-eth/config/docker/eth.ini | 8 +--- apps/cic-eth/config/docker/signer.ini | 2 +- apps/cic-eth/config/eth.ini | 8 +--- apps/cic-eth/requirements.txt | 4 +- apps/cic-eth/setup.cfg | 2 +- apps/contract-migration/docker/Dockerfile | 4 +- apps/contract-migration/seed_cic_eth.sh | 12 +++++ apps/util/liveness/liveness/linux.py | 2 +- apps/util/liveness/setup.py | 2 +- apps/util/liveness/tests/test_imports.py | 4 +- 23 files changed, 165 insertions(+), 37 deletions(-) create mode 100644 apps/cic-eth/cic_eth/check/__init__.py rename apps/cic-eth/cic_eth/{k8s => check}/db.py (99%) create mode 100644 apps/cic-eth/cic_eth/check/gas.py create mode 100644 apps/cic-eth/cic_eth/check/signer.py diff --git a/apps/cic-eth/cic_eth/admin/ctrl.py b/apps/cic-eth/cic_eth/admin/ctrl.py index c3815d3..0b55ffc 100644 --- a/apps/cic-eth/cic_eth/admin/ctrl.py +++ b/apps/cic-eth/cic_eth/admin/ctrl.py @@ -32,7 +32,9 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL :returns: New lock state for address :rtype: number """ - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash) logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r)) return chained_input @@ -51,7 +53,9 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum. :returns: New lock state for address :rtype: number """ - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.reset(chain_str, flags, address=address) logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r)) return chained_input @@ -127,7 +131,9 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS): @celery_app.task(base=CriticalSQLAlchemyTask) def check_lock(chained_input, chain_spec_dict, lock_flags, address=None): - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) session = SessionBase.create_session() r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session) if address != None: diff --git a/apps/cic-eth/cic_eth/check/__init__.py b/apps/cic-eth/cic_eth/check/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/cic-eth/cic_eth/k8s/db.py b/apps/cic-eth/cic_eth/check/db.py similarity index 99% rename from apps/cic-eth/cic_eth/k8s/db.py rename to apps/cic-eth/cic_eth/check/db.py index 485e1b5..3382a33 100644 --- a/apps/cic-eth/cic_eth/k8s/db.py +++ b/apps/cic-eth/cic_eth/check/db.py @@ -1,5 +1,6 @@ from cic_eth.db.models.base import SessionBase + def health(*args, **kwargs): session = SessionBase.create_session() session.execute('SELECT count(*) from alembic_version') diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py new file mode 100644 index 0000000..b1646fd --- /dev/null +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -0,0 +1,48 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import RPCConnection +from chainlib.chain import ChainSpec +from chainlib.eth.gas import balance + +# local imports +from cic_eth.db.models.role import AccountRole +from cic_eth.db.models.base import SessionBase +from cic_eth.db.enum import LockEnum +from cic_eth.error import LockedError +from cic_eth.admin.ctrl import check_lock + +logg = logging.getLogger().getChild(__name__) + + +def health(*args, **kwargs): + + session = SessionBase.create_session() + + config = kwargs['config'] + chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) + logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec)) + + try: + check_lock(None, None, LockEnum.INIT) + except LockedError: + logg.warning('INIT lock is set, skipping GAS GIFTER balance check.') + return True + + gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) + session.close() + + rpc = RPCConnection.connect(chain_spec, 'default') + o = balance(gas_provider) + r = rpc.do(o) + try: + r = int(r, 16) + except TypeError: + r = int(r) + gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE')) + if r < gas_min: + logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min)) + return False + + return True diff --git a/apps/cic-eth/cic_eth/check/signer.py b/apps/cic-eth/cic_eth/check/signer.py new file mode 100644 index 0000000..87b7785 --- /dev/null +++ b/apps/cic-eth/cic_eth/check/signer.py @@ -0,0 +1,37 @@ +# standard imports +import time +import logging +from urllib.error import URLError + +# external imports +from chainlib.connection import RPCConnection +from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.sign import sign_message +from chainlib.error import JSONRPCException + +logg = logging.getLogger().getChild(__name__) + + +def health(*args, **kwargs): + blocked = True + max_attempts = 5 + conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer') + for i in range(max_attempts): + idx = i + 1 + logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts)) + try: + conn.do(sign_message(ZERO_ADDRESS, '0x2a')) + except FileNotFoundError: + pass + except ConnectionError: + pass + except URLError: + pass + except JSONRPCException: + logg.debug('signer connection succeeded') + return True + + if idx < max_attempts: + time.sleep(0.5) + + return False diff --git a/apps/cic-eth/cic_eth/db/enum.py b/apps/cic-eth/cic_eth/db/enum.py index 67a50f0..2ce3ecc 100644 --- a/apps/cic-eth/cic_eth/db/enum.py +++ b/apps/cic-eth/cic_eth/db/enum.py @@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum): QUEUE: Disable queueing new or modified transactions """ STICKY=1 - CREATE=2 - SEND=4 - QUEUE=8 - QUERY=16 + INIT=2 + CREATE=4 + SEND=8 + QUEUE=16 + QUERY=32 ALL=int(0xfffffffffffffffe) diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index 656fdd0..cfbbff0 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -5,8 +5,11 @@ Revises: 1f1b3b641d08 Create Date: 2021-04-02 18:41:20.864265 """ +import datetime from alembic import op import sqlalchemy as sa +from chainlib.eth.constant import ZERO_ADDRESS +from cic_eth.db.enum import LockEnum # revision identifiers, used by Alembic. @@ -23,10 +26,11 @@ def upgrade(): sa.Column("address", sa.String(42), nullable=True), sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), - sa.Column("date_created", sa.DateTime, nullable=False), + sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True), ) op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) + op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE)) def downgrade(): diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index 8702085..8f9df08 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -59,6 +59,7 @@ args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), } # override args +config.dict_override(args_override, 'cli') config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) @@ -67,7 +68,9 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi queue = args.q -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = None +if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::': + chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) admin_api = AdminApi(None) @@ -82,6 +85,9 @@ def lock_names_to_flag(s): # TODO: move each command to submodule def main(): + chain_spec_dict = None + if chain_spec != None: + chain_spec_dict = chain_spec.asdict() if args.command == 'unlock': flags = lock_names_to_flag(args.flags) if not is_checksum_address(args.address): @@ -91,7 +97,7 @@ def main(): 'cic_eth.admin.ctrl.unlock', [ None, - chain_spec.asdict(), + chain_spec_dict, args.address, flags, ], @@ -110,7 +116,7 @@ def main(): 'cic_eth.admin.ctrl.lock', [ None, - chain_spec.asdict(), + chain_spec_dict, args.address, flags, ], diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 2d9d043..3868924 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -15,8 +15,10 @@ from chainlib.connection import RPCConnection from chainlib.eth.connection import EthUnixSignerConnection from chainlib.chain import ChainSpec from chainqueue.db.models.otx import Otx +from cic_eth_registry.error import UnknownContractError import liveness.linux + # local imports from cic_eth.eth import ( erc20, @@ -138,11 +140,15 @@ else: chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default') -RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection) +#RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection) +RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer') Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') -liveness.linux.load(health_modules) +#import cic_eth.checks.gas +#if not cic_eth.checks.gas.health(config=config): +# raise RuntimeError() +liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config) def main(): argv = ['worker'] @@ -166,7 +172,11 @@ def main(): rpc = RPCConnection.connect(chain_spec, 'default') - connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + try: + connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + except UnknownContractError as e: + logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) + sys.exit(1) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: @@ -175,6 +185,7 @@ def main(): trusted_addresses = trusted_addresses_src.split(',') for address in trusted_addresses: logg.info('using trusted address {}'.format(address)) + connect_declarator(rpc, chain_spec, trusted_addresses) connect_token_registry(rpc, chain_spec) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index e95f482..45ba12b 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -94,3 +94,8 @@ class CriticalWeb3AndSignerTask(CriticalTask): def hello(self): time.sleep(0.1) return id(SessionBase.create_session) + + +@celery_app.task() +def check_health(self): + celery.app.control.shutdown() diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 1624e64..4740b58 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 11, 0, - 'beta.7', + 'beta.8', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/config/cic.ini b/apps/cic-eth/config/cic.ini index 10ec9cd..20f1fe8 100644 --- a/apps/cic-eth/config/cic.ini +++ b/apps/cic-eth/config/cic.ini @@ -3,4 +3,5 @@ registry_address = chain_spec = evm:bloxberg:8996 tx_retry_delay = trust_address = -health_modules = cic_eth.k8s.db +health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas +run_dir = /run diff --git a/apps/cic-eth/config/docker/cic.ini b/apps/cic-eth/config/docker/cic.ini index 50032aa..45bd47e 100644 --- a/apps/cic-eth/config/docker/cic.ini +++ b/apps/cic-eth/config/docker/cic.ini @@ -3,3 +3,5 @@ registry_address = chain_spec = evm:bloxberg:8996 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C tx_retry_delay = 20 +health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas +run_dir = /run diff --git a/apps/cic-eth/config/docker/eth.ini b/apps/cic-eth/config/docker/eth.ini index d162a5e..9d6b660 100644 --- a/apps/cic-eth/config/docker/eth.ini +++ b/apps/cic-eth/config/docker/eth.ini @@ -1,8 +1,4 @@ [eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 provider = http://localhost:63545 -gas_provider_address = -#chain_id = -abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis -account_accounts_index_writer = +health_modules = cic_eth.check.db,cic_eth.check.gas +gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/docker/signer.ini b/apps/cic-eth/config/docker/signer.ini index 69df88d..fe72206 100644 --- a/apps/cic-eth/config/docker/signer.ini +++ b/apps/cic-eth/config/docker/signer.ini @@ -1,5 +1,5 @@ [signer] -socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc +socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc secret = deedbeef database_name = signer_test dev_keys_path = diff --git a/apps/cic-eth/config/eth.ini b/apps/cic-eth/config/eth.ini index 1a6935e..3b41af9 100644 --- a/apps/cic-eth/config/eth.ini +++ b/apps/cic-eth/config/eth.ini @@ -1,8 +1,4 @@ [eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 provider = http://localhost:8545 -gas_provider_address = -#chain_id = -abi_dir = /usr/local/share/cic/solidity/abi -account_accounts_index_writer = +gas_gifter_minimum_balance = 10000000000000000000000 +health_modules = cic_eth.check.db,cic_eth.check.gas diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 3ab4137..44d8b27 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,6 +1,6 @@ -cic-base==0.1.2a79+build.35e442bc +cic-base==0.1.2b1 celery==4.4.7 -crypto-dev-signer~=0.4.14b2 +crypto-dev-signer~=0.4.14b3 confini~=0.3.6rc3 cic-eth-registry~=0.5.4a16 #cic-bancor~=0.0.6 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index cc0d554..55bbb4c 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -38,7 +38,7 @@ packages = cic_eth.runnable.daemons.filters cic_eth.callbacks cic_eth.sync - cic_eth.k8s + cic_eth.check scripts = ./scripts/migrate.py diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index c370006..f0d82cb 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,8 +57,8 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a77 -ARG cic_eth_version=0.11.0b6 +ARG cic_base_version=0.1.2a79 +ARG cic_eth_version=0.11.0b8+build.c2286e5c ARG sarafu_faucet_version=0.0.2a28 ARG sarafu_token_version==0.0.1a6 ARG cic_contracts_version=0.0.2a2 diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index 5df672a..42db0ca 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -13,6 +13,12 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"} CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} ETH_PASSPHRASE='' +DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable} +if [ $DEV_TOKEN_TYPE = 'giftable' ]; then + token_symbol='GFT' +else + token_symbol='SRF' +fi # Debug flag DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C @@ -46,6 +52,7 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER + >&2 echo "create account for sarafu gifter" DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file @@ -97,5 +104,10 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS #echo -n 0 > $init_level_file +# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration +cic-eth-ctl -i :: unlock INIT +cic-eth-ctl -i :: unlock SEND +cic-eth-ctl -i :: unlock QUEUE + set +a set +e diff --git a/apps/util/liveness/liveness/linux.py b/apps/util/liveness/liveness/linux.py index 24bb59e..7ff754f 100644 --- a/apps/util/liveness/liveness/linux.py +++ b/apps/util/liveness/liveness/linux.py @@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs checks.append(module) for check in checks: - r = check.health(args, kwargs) + r = check.health(*args, **kwargs) if r == False: raise RuntimeError('liveness check {} failed'.format(str(check))) logg.info('liveness check passed: {}'.format(str(check))) diff --git a/apps/util/liveness/setup.py b/apps/util/liveness/setup.py index 3c8b3f2..ed75fca 100644 --- a/apps/util/liveness/setup.py +++ b/apps/util/liveness/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup( name='liveness', - version='0.0.1a6', + version='0.0.1a7', packages=['liveness'], include_package_data=True, ) diff --git a/apps/util/liveness/tests/test_imports.py b/apps/util/liveness/tests/test_imports.py index 02e6be7..3af935a 100644 --- a/apps/util/liveness/tests/test_imports.py +++ b/apps/util/liveness/tests/test_imports.py @@ -116,7 +116,9 @@ class TestImports(unittest.TestCase): def test_args(self): checks = ['tests.imports.import_args'] - liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42}) + aargs=['foo'] + kwaargs={'bar': 42} + liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs) f = open(self.pid_path, 'r') r = f.read() f.close()