From c2286e5c25daa17e3c54655b1759055655a88ade Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 21 Apr 2021 23:11:48 +0200 Subject: [PATCH] Permit empty chain spec in ctl --- apps/cic-eth/cic_eth/admin/ctrl.py | 12 +++++++++--- apps/cic-eth/cic_eth/{k8s => check}/db.py | 0 apps/cic-eth/cic_eth/{checks => check}/gas.py | 12 ++++++++++-- apps/cic-eth/cic_eth/db/enum.py | 9 +++++---- .../migrations/default/versions/75d4767b3031_lock.py | 6 +++++- apps/cic-eth/cic_eth/runnable/ctrl.py | 12 +++++++++--- apps/cic-eth/cic_eth/runnable/daemons/tasker.py | 2 +- apps/cic-eth/config/cic.ini | 2 +- apps/cic-eth/config/docker/eth.ini | 3 ++- apps/cic-eth/config/eth.ini | 8 ++------ apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/setup.cfg | 2 +- apps/contract-migration/seed_cic_eth.sh | 5 +++++ apps/util/liveness/liveness/linux.py | 2 +- apps/util/liveness/setup.py | 2 +- apps/util/liveness/tests/test_imports.py | 4 +++- 16 files changed, 56 insertions(+), 27 deletions(-) rename apps/cic-eth/cic_eth/{k8s => check}/db.py (100%) rename apps/cic-eth/cic_eth/{checks => check}/gas.py (72%) diff --git a/apps/cic-eth/cic_eth/admin/ctrl.py b/apps/cic-eth/cic_eth/admin/ctrl.py index c3815d3b..0b55ffc5 100644 --- a/apps/cic-eth/cic_eth/admin/ctrl.py +++ b/apps/cic-eth/cic_eth/admin/ctrl.py @@ -32,7 +32,9 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL :returns: New lock state for address :rtype: number """ - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash) logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r)) return chained_input @@ -51,7 +53,9 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum. :returns: New lock state for address :rtype: number """ - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) r = Lock.reset(chain_str, flags, address=address) logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r)) return chained_input @@ -127,7 +131,9 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS): @celery_app.task(base=CriticalSQLAlchemyTask) def check_lock(chained_input, chain_spec_dict, lock_flags, address=None): - chain_str = str(ChainSpec.from_dict(chain_spec_dict)) + chain_str = '::' + if chain_spec_dict != None: + chain_str = str(ChainSpec.from_dict(chain_spec_dict)) session = SessionBase.create_session() r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session) if address != None: diff --git a/apps/cic-eth/cic_eth/k8s/db.py b/apps/cic-eth/cic_eth/check/db.py similarity index 100% rename from apps/cic-eth/cic_eth/k8s/db.py rename to apps/cic-eth/cic_eth/check/db.py diff --git a/apps/cic-eth/cic_eth/checks/gas.py b/apps/cic-eth/cic_eth/check/gas.py similarity index 72% rename from apps/cic-eth/cic_eth/checks/gas.py rename to apps/cic-eth/cic_eth/check/gas.py index fae873d5..b1646fd6 100644 --- a/apps/cic-eth/cic_eth/checks/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -9,6 +9,9 @@ from chainlib.eth.gas import balance # local imports from cic_eth.db.models.role import AccountRole from cic_eth.db.models.base import SessionBase +from cic_eth.db.enum import LockEnum +from cic_eth.error import LockedError +from cic_eth.admin.ctrl import check_lock logg = logging.getLogger().getChild(__name__) @@ -17,9 +20,15 @@ def health(*args, **kwargs): session = SessionBase.create_session() - logg.info('kwargs {} {}'.format(kwargs, args)) config = kwargs['config'] chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) + logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec)) + + try: + check_lock(None, None, LockEnum.INIT) + except LockedError: + logg.warning('INIT lock is set, skipping GAS GIFTER balance check.') + return True gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) session.close() @@ -36,5 +45,4 @@ def health(*args, **kwargs): logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min)) return False - return True diff --git a/apps/cic-eth/cic_eth/db/enum.py b/apps/cic-eth/cic_eth/db/enum.py index 67a50f0d..2ce3eccd 100644 --- a/apps/cic-eth/cic_eth/db/enum.py +++ b/apps/cic-eth/cic_eth/db/enum.py @@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum): QUEUE: Disable queueing new or modified transactions """ STICKY=1 - CREATE=2 - SEND=4 - QUEUE=8 - QUERY=16 + INIT=2 + CREATE=4 + SEND=8 + QUEUE=16 + QUERY=32 ALL=int(0xfffffffffffffffe) diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index 656fdd06..cfbbff09 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -5,8 +5,11 @@ Revises: 1f1b3b641d08 Create Date: 2021-04-02 18:41:20.864265 """ +import datetime from alembic import op import sqlalchemy as sa +from chainlib.eth.constant import ZERO_ADDRESS +from cic_eth.db.enum import LockEnum # revision identifiers, used by Alembic. @@ -23,10 +26,11 @@ def upgrade(): sa.Column("address", sa.String(42), nullable=True), sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), - sa.Column("date_created", sa.DateTime, nullable=False), + sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True), ) op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) + op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE)) def downgrade(): diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index 87020851..8f9df080 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -59,6 +59,7 @@ args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), } # override args +config.dict_override(args_override, 'cli') config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) @@ -67,7 +68,9 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi queue = args.q -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = None +if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::': + chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) admin_api = AdminApi(None) @@ -82,6 +85,9 @@ def lock_names_to_flag(s): # TODO: move each command to submodule def main(): + chain_spec_dict = None + if chain_spec != None: + chain_spec_dict = chain_spec.asdict() if args.command == 'unlock': flags = lock_names_to_flag(args.flags) if not is_checksum_address(args.address): @@ -91,7 +97,7 @@ def main(): 'cic_eth.admin.ctrl.unlock', [ None, - chain_spec.asdict(), + chain_spec_dict, args.address, flags, ], @@ -110,7 +116,7 @@ def main(): 'cic_eth.admin.ctrl.lock', [ None, - chain_spec.asdict(), + chain_spec_dict, args.address, flags, ], diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index d0c2ba91..b0c736f8 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -145,7 +145,7 @@ Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') #import cic_eth.checks.gas #if not cic_eth.checks.gas.health(config=config): # raise RuntimeError() -liveness.linux.load(health_modules) +liveness.linux.load(health_modules, config=config) def main(): argv = ['worker'] diff --git a/apps/cic-eth/config/cic.ini b/apps/cic-eth/config/cic.ini index 10ec9cdc..d97826d4 100644 --- a/apps/cic-eth/config/cic.ini +++ b/apps/cic-eth/config/cic.ini @@ -3,4 +3,4 @@ registry_address = chain_spec = evm:bloxberg:8996 tx_retry_delay = trust_address = -health_modules = cic_eth.k8s.db +health_modules = cic_eth.check.db,cic_eth.check.gas diff --git a/apps/cic-eth/config/docker/eth.ini b/apps/cic-eth/config/docker/eth.ini index a7905917..9d6b6607 100644 --- a/apps/cic-eth/config/docker/eth.ini +++ b/apps/cic-eth/config/docker/eth.ini @@ -1,3 +1,4 @@ [eth] provider = http://localhost:63545 -gas_gifter_minimum_balance = 1000000000000000 +health_modules = cic_eth.check.db,cic_eth.check.gas +gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/eth.ini b/apps/cic-eth/config/eth.ini index 1a6935e1..3b41af90 100644 --- a/apps/cic-eth/config/eth.ini +++ b/apps/cic-eth/config/eth.ini @@ -1,8 +1,4 @@ [eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 provider = http://localhost:8545 -gas_provider_address = -#chain_id = -abi_dir = /usr/local/share/cic/solidity/abi -account_accounts_index_writer = +gas_gifter_minimum_balance = 10000000000000000000000 +health_modules = cic_eth.check.db,cic_eth.check.gas diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 3ab41375..953eea77 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ -cic-base==0.1.2a79+build.35e442bc +cic-base==0.1.2a79+build.d412618d celery==4.4.7 crypto-dev-signer~=0.4.14b2 confini~=0.3.6rc3 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index cc0d5549..55bbb4c9 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -38,7 +38,7 @@ packages = cic_eth.runnable.daemons.filters cic_eth.callbacks cic_eth.sync - cic_eth.k8s + cic_eth.check scripts = ./scripts/migrate.py diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index 84d1258c..9eccf75c 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -46,6 +46,11 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER +# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration +cic-eth-ctl -i :: unlock INIT +cic-eth-ctl -i :: unlock SEND +cic-eth-ctl -i :: unlock QUEUE + >&2 echo "create account for sarafu gifter" DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file diff --git a/apps/util/liveness/liveness/linux.py b/apps/util/liveness/liveness/linux.py index 24bb59e6..7ff754f0 100644 --- a/apps/util/liveness/liveness/linux.py +++ b/apps/util/liveness/liveness/linux.py @@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs checks.append(module) for check in checks: - r = check.health(args, kwargs) + r = check.health(*args, **kwargs) if r == False: raise RuntimeError('liveness check {} failed'.format(str(check))) logg.info('liveness check passed: {}'.format(str(check))) diff --git a/apps/util/liveness/setup.py b/apps/util/liveness/setup.py index 3c8b3f2e..ed75fca1 100644 --- a/apps/util/liveness/setup.py +++ b/apps/util/liveness/setup.py @@ -1,7 +1,7 @@ from setuptools import setup setup( name='liveness', - version='0.0.1a6', + version='0.0.1a7', packages=['liveness'], include_package_data=True, ) diff --git a/apps/util/liveness/tests/test_imports.py b/apps/util/liveness/tests/test_imports.py index 02e6be79..3af935ad 100644 --- a/apps/util/liveness/tests/test_imports.py +++ b/apps/util/liveness/tests/test_imports.py @@ -116,7 +116,9 @@ class TestImports(unittest.TestCase): def test_args(self): checks = ['tests.imports.import_args'] - liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42}) + aargs=['foo'] + kwaargs={'bar': 42} + liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs) f = open(self.pid_path, 'r') r = f.read() f.close()