diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 62a90080..6fc0e81a 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -204,6 +204,7 @@ def main(): BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) + BaseTask.run_dir = config.get('CIC_RUN_DIR') logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) liveness.linux.set(rundir=config.get('CIC_RUN_DIR')) diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 24802185..3fe201f6 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -10,12 +10,13 @@ import sqlalchemy from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.gas import RPCGasOracle +import liveness.linux # local imports from cic_eth.error import SeppukuError from cic_eth.db.models.base import SessionBase -logg = logging.getLogger(__name__) +logg = logging.getLogger().getChild(__name__) celery_app = celery.current_app @@ -28,6 +29,7 @@ class BaseTask(celery.Task): create_gas_oracle = RPCGasOracle default_token_address = None default_token_symbol = None + run_dir = '/run' def create_session(self): return BaseTask.session_func() @@ -40,6 +42,7 @@ class BaseTask(celery.Task): def on_failure(self, exc, task_id, args, kwargs, einfo): if isinstance(exc, SeppukuError): + liveness.linux.reset(rundir=self.run_dir) logg.critical(einfo) msg = 'received critical exception {}, calling shutdown'.format(str(exc)) s = celery.signature( diff --git a/apps/util/liveness/liveness/linux.py b/apps/util/liveness/liveness/linux.py index 7ff754f0..52d4b24c 100644 --- a/apps/util/liveness/liveness/linux.py +++ b/apps/util/liveness/liveness/linux.py @@ -42,6 +42,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs def set(error=0, namespace=default_namespace, rundir='/run'): + logg.info('liveness SET error {} for namespace {}'.format(error, namespace)) app_rundir = os.path.join(rundir, namespace) f = open(os.path.join(app_rundir, 'error'), 'w') f.write(str(error)) @@ -49,6 +50,13 @@ def set(error=0, namespace=default_namespace, rundir='/run'): def reset(namespace=default_namespace, rundir='/run'): + logg.info('liveness RESET for namespace {}'.format(namespace)) app_rundir = os.path.join(rundir, namespace) - os.unlink(os.path.join(app_rundir, 'pid')) - os.unlink(os.path.join(app_rundir, 'error')) + try: + os.unlink(os.path.join(app_rundir, 'pid')) + except FileNotFoundError: + pass + try: + os.unlink(os.path.join(app_rundir, 'error')) + except FileNotFoundError: + pass