Add health python helper, shell tester

apps/cic-cache/docker/start_tracker.sh

@@ -2,4 +2,9 @@
 
 . ./db.sh
 
+if [ $? -ne "0" ]; then
+	>&2 echo db migrate fail
+	exit 1
+fi
+
 /usr/local/bin/cic-cache-trackerd $@

apps/util/liveness/health.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+rundir=${CIC_RUNDIR:-/run}
+
+read p < $rundir/$CIC_UNIT/pid
+
+if [ -z $p ]; then
+	>&2 echo unit $CIC_UNIT has no pid
+	exit 1
+fi
+
+if [ ! -d /proc/$p ]; then
+	>&2 echo unit $CIC_UNIT reports non-existent pid $p
+	exit 1
+fi	
+
+>&2 echo unit $CIC_UNIT has pid $p
+
+if [ ! -f $rundir/$CIC_UNIT/error ]; then
+	>&2 echo unit $CIC_UNIT has unspecified state
+	exit 1
+fi
+
+read e 2> /dev/null < $rundir/$CIC_UNIT/error 
+if [ -z $e ]; then
+	>&2 echo unit $CIC_UNIT has unspecified state
+	exit 1
+fi
+
+>&2 echo unit $CIC_UNIT has error $e
+
+if [ $e -gt 0 ]; then
+	exit 1;
+fi

apps/util/liveness/liveness/linux.py

@@ -0,0 +1,42 @@
+# standard imports
+import importlib
+import sys
+import os
+import logging
+
+logg = logging.getLogger().getChild(__name__)
+
+pid = os.getpid()
+
+
+def load(namespace, check_strs, rundir='/run'):
+    logg.info('pid ' + str(pid))
+
+    checks = []
+    for m in check_strs:
+        logg.debug('added liveness check module {}'.format(str(m)))
+        module = importlib.import_module(m)
+        checks.append(module)
+
+    for check in checks:
+        r = check.health()
+        if r == False:
+            raise RuntimeError('check {} failed'.format(str(check)))
+
+    app_rundir = os.path.join(rundir, namespace)
+    os.makedirs(app_rundir) # should not already exist
+    f = open(os.path.join(app_rundir, 'pid'), 'w')
+    f.write(str(pid))
+    f.close()
+
+
+def set(namespace, error=0, rundir='/run'):
+    app_rundir = os.path.join(rundir, namespace)
+    f = open(os.path.join(app_rundir, 'error'), 'w')
+    f.write(str(error))
+    f.close()
+
+
+def reset(namespace, rundir='/run'):
+    app_rundir = os.path.join(rundir, namespace)
+    os.unlink(os.path.join(app_rundir, 'error'))

apps/util/liveness/test_health.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export CIC_RUNDIR=`realpath ./tests/testdata/run`
+t=`mktemp -d -p $CIC_RUNDIR`
+export CIC_UNIT=`basename $t`
+
+>&2 echo test pid $$
+echo $$ > $t/pid
+echo 0 > $t/error
+
+. health.sh
+
+echo 1 > $t/error
+#unlink $t/error
+. health.sh
+
+echo if error this is not printed

apps/util/liveness/tests/imports/import_false.py

@@ -0,0 +1,2 @@
+def health():
+    return False

apps/util/liveness/tests/imports/import_true.py

@@ -0,0 +1,2 @@
+def health():
+    return True

apps/util/liveness/tests/test_imports.py

@@ -0,0 +1,94 @@
+# standard imports
+import os
+import unittest
+import logging
+import tempfile
+
+# local imports
+import liveness.linux 
+
+## test imports
+import tests.imports
+
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+script_dir = os.path.realpath(os.path.dirname(__file__))
+data_dir = os.path.join(script_dir, 'testdata')
+run_base_dir = os.path.join(data_dir, 'run')
+
+
+
+class TestImports(unittest.TestCase):
+
+    def setUp(self):
+        os.makedirs(run_base_dir, exist_ok=True)
+        self.run_dir = tempfile.mkdtemp(dir=run_base_dir)
+        self.unit_dir = os.path.join(self.run_dir, 'unittest')
+        self.pid_path = os.path.join(self.unit_dir, 'pid')
+        self.error_path = os.path.join(self.unit_dir, 'error')
+
+    def test_no_import(self):
+        liveness.linux.load('unittest', [], rundir=self.run_dir)
+        f = open(self.pid_path, 'r')
+        r = f.read()
+        f.close()
+        self.assertEqual(str(os.getpid()), r)
+
+    
+    def test_import_single_true(self):
+        checks = ['tests.imports.import_true']
+        liveness.linux.load('unittest', checks, rundir=self.run_dir)
+        f = open(self.pid_path, 'r')
+        r = f.read()
+        f.close()
+        self.assertEqual(str(os.getpid()), r)
+
+
+    def test_import_single_false(self):
+        checks = ['tests.imports.import_false']
+        with self.assertRaises(RuntimeError):
+            liveness.linux.load('unittest', checks, rundir=self.run_dir)
+        with self.assertRaises(FileNotFoundError):
+            os.stat(self.pid_path)
+
+
+    def test_import_false_then_true(self):
+        checks = ['tests.imports.import_false', 'tests.imports.import_true']
+        with self.assertRaises(RuntimeError):
+            liveness.linux.load('unittest', checks, rundir=self.run_dir)
+        with self.assertRaises(FileNotFoundError):
+            os.stat(self.pid_path)
+
+
+    def test_import_multiple_true(self):
+        checks = ['tests.imports.import_true', 'tests.imports.import_true']
+        liveness.linux.load('unittest', checks, rundir=self.run_dir)
+        f = open(self.pid_path, 'r')
+        r = f.read()
+        f.close()
+        self.assertEqual(str(os.getpid()), r)
+
+
+    def test_set(self):
+        liveness.linux.load('unittest', [], rundir=self.run_dir)
+        liveness.linux.set('unittest', rundir=self.run_dir)
+        f = open(self.error_path, 'r')
+        r = f.read()
+        f.close()
+        self.assertEqual('0', r)
+
+        liveness.linux.set('unittest', 42, rundir=self.run_dir)
+        f = open(self.error_path, 'r')
+        r = f.read()
+        f.close()
+        self.assertEqual('42', r)
+
+        liveness.linux.reset('unittest', rundir=self.run_dir)
+        with self.assertRaises(FileNotFoundError):
+            os.stat(self.error_path)
+            
+
+
+if __name__ == '__main__':
+    unittest.main()