diff --git a/apps/cic-cache/docker/start_tracker.sh b/apps/cic-cache/docker/start_tracker.sh index cca1cb1a..0f722537 100644 --- a/apps/cic-cache/docker/start_tracker.sh +++ b/apps/cic-cache/docker/start_tracker.sh @@ -2,4 +2,9 @@ . ./db.sh +if [ $? -ne "0" ]; then + >&2 echo db migrate fail + exit 1 +fi + /usr/local/bin/cic-cache-trackerd $@ diff --git a/apps/util/liveness/health.sh b/apps/util/liveness/health.sh new file mode 100644 index 00000000..7046de42 --- /dev/null +++ b/apps/util/liveness/health.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +rundir=${CIC_RUNDIR:-/run} + +read p < $rundir/$CIC_UNIT/pid + +if [ -z $p ]; then + >&2 echo unit $CIC_UNIT has no pid + exit 1 +fi + +if [ ! -d /proc/$p ]; then + >&2 echo unit $CIC_UNIT reports non-existent pid $p + exit 1 +fi + +>&2 echo unit $CIC_UNIT has pid $p + +if [ ! -f $rundir/$CIC_UNIT/error ]; then + >&2 echo unit $CIC_UNIT has unspecified state + exit 1 +fi + +read e 2> /dev/null < $rundir/$CIC_UNIT/error +if [ -z $e ]; then + >&2 echo unit $CIC_UNIT has unspecified state + exit 1 +fi + +>&2 echo unit $CIC_UNIT has error $e + +if [ $e -gt 0 ]; then + exit 1; +fi diff --git a/apps/util/liveness/liveness/linux.py b/apps/util/liveness/liveness/linux.py new file mode 100644 index 00000000..250ddfad --- /dev/null +++ b/apps/util/liveness/liveness/linux.py @@ -0,0 +1,42 @@ +# standard imports +import importlib +import sys +import os +import logging + +logg = logging.getLogger().getChild(__name__) + +pid = os.getpid() + + +def load(namespace, check_strs, rundir='/run'): + logg.info('pid ' + str(pid)) + + checks = [] + for m in check_strs: + logg.debug('added liveness check module {}'.format(str(m))) + module = importlib.import_module(m) + checks.append(module) + + for check in checks: + r = check.health() + if r == False: + raise RuntimeError('check {} failed'.format(str(check))) + + app_rundir = os.path.join(rundir, namespace) + os.makedirs(app_rundir) # should not already exist + f = open(os.path.join(app_rundir, 'pid'), 'w') + f.write(str(pid)) + f.close() + + +def set(namespace, error=0, rundir='/run'): + app_rundir = os.path.join(rundir, namespace) + f = open(os.path.join(app_rundir, 'error'), 'w') + f.write(str(error)) + f.close() + + +def reset(namespace, rundir='/run'): + app_rundir = os.path.join(rundir, namespace) + os.unlink(os.path.join(app_rundir, 'error')) diff --git a/apps/util/liveness/test_health.sh b/apps/util/liveness/test_health.sh new file mode 100644 index 00000000..898a05d2 --- /dev/null +++ b/apps/util/liveness/test_health.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +export CIC_RUNDIR=`realpath ./tests/testdata/run` +t=`mktemp -d -p $CIC_RUNDIR` +export CIC_UNIT=`basename $t` + +>&2 echo test pid $$ +echo $$ > $t/pid +echo 0 > $t/error + +. health.sh + +echo 1 > $t/error +#unlink $t/error +. health.sh + +echo if error this is not printed diff --git a/apps/util/liveness/tests/imports/__init__.py b/apps/util/liveness/tests/imports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/util/liveness/tests/imports/import_false.py b/apps/util/liveness/tests/imports/import_false.py new file mode 100644 index 00000000..cc5a26fb --- /dev/null +++ b/apps/util/liveness/tests/imports/import_false.py @@ -0,0 +1,2 @@ +def health(): + return False diff --git a/apps/util/liveness/tests/imports/import_true.py b/apps/util/liveness/tests/imports/import_true.py new file mode 100644 index 00000000..c33f7db9 --- /dev/null +++ b/apps/util/liveness/tests/imports/import_true.py @@ -0,0 +1,2 @@ +def health(): + return True diff --git a/apps/util/liveness/tests/test_imports.py b/apps/util/liveness/tests/test_imports.py new file mode 100644 index 00000000..745123c6 --- /dev/null +++ b/apps/util/liveness/tests/test_imports.py @@ -0,0 +1,94 @@ +# standard imports +import os +import unittest +import logging +import tempfile + +# local imports +import liveness.linux + +## test imports +import tests.imports + + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() +script_dir = os.path.realpath(os.path.dirname(__file__)) +data_dir = os.path.join(script_dir, 'testdata') +run_base_dir = os.path.join(data_dir, 'run') + + + +class TestImports(unittest.TestCase): + + def setUp(self): + os.makedirs(run_base_dir, exist_ok=True) + self.run_dir = tempfile.mkdtemp(dir=run_base_dir) + self.unit_dir = os.path.join(self.run_dir, 'unittest') + self.pid_path = os.path.join(self.unit_dir, 'pid') + self.error_path = os.path.join(self.unit_dir, 'error') + + def test_no_import(self): + liveness.linux.load('unittest', [], rundir=self.run_dir) + f = open(self.pid_path, 'r') + r = f.read() + f.close() + self.assertEqual(str(os.getpid()), r) + + + def test_import_single_true(self): + checks = ['tests.imports.import_true'] + liveness.linux.load('unittest', checks, rundir=self.run_dir) + f = open(self.pid_path, 'r') + r = f.read() + f.close() + self.assertEqual(str(os.getpid()), r) + + + def test_import_single_false(self): + checks = ['tests.imports.import_false'] + with self.assertRaises(RuntimeError): + liveness.linux.load('unittest', checks, rundir=self.run_dir) + with self.assertRaises(FileNotFoundError): + os.stat(self.pid_path) + + + def test_import_false_then_true(self): + checks = ['tests.imports.import_false', 'tests.imports.import_true'] + with self.assertRaises(RuntimeError): + liveness.linux.load('unittest', checks, rundir=self.run_dir) + with self.assertRaises(FileNotFoundError): + os.stat(self.pid_path) + + + def test_import_multiple_true(self): + checks = ['tests.imports.import_true', 'tests.imports.import_true'] + liveness.linux.load('unittest', checks, rundir=self.run_dir) + f = open(self.pid_path, 'r') + r = f.read() + f.close() + self.assertEqual(str(os.getpid()), r) + + + def test_set(self): + liveness.linux.load('unittest', [], rundir=self.run_dir) + liveness.linux.set('unittest', rundir=self.run_dir) + f = open(self.error_path, 'r') + r = f.read() + f.close() + self.assertEqual('0', r) + + liveness.linux.set('unittest', 42, rundir=self.run_dir) + f = open(self.error_path, 'r') + r = f.read() + f.close() + self.assertEqual('42', r) + + liveness.linux.reset('unittest', rundir=self.run_dir) + with self.assertRaises(FileNotFoundError): + os.stat(self.error_path) + + + +if __name__ == '__main__': + unittest.main()