diff --git a/apps/cic-eth/cic_eth/k8s/db.py b/apps/cic-eth/cic_eth/k8s/db.py index d849a9aa..485e1b56 100644 --- a/apps/cic-eth/cic_eth/k8s/db.py +++ b/apps/cic-eth/cic_eth/k8s/db.py @@ -1,6 +1,6 @@ from cic_eth.db.models.base import SessionBase -def health(): +def health(*args, **kwargs): session = SessionBase.create_session() session.execute('SELECT count(*) from alembic_version') session.close() diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index b52ff20a..2d9d0434 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -142,7 +142,7 @@ RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 's Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') -liveness.linux.load(health_modules, namespace='cic-eth') +liveness.linux.load(health_modules) def main(): argv = ['worker'] @@ -178,9 +178,9 @@ def main(): connect_declarator(rpc, chain_spec, trusted_addresses) connect_token_registry(rpc, chain_spec) - liveness.linux.set('cic-eth') + liveness.linux.set() current_app.worker_main(argv) - liveness.linux.reset('cic-eth') + liveness.linux.reset() @celery.signals.eventlet_pool_postshutdown.connect diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index c4a9f4d8..3ab41375 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ -cic-base==0.1.2a79+build.7b0b61c9 +cic-base==0.1.2a79+build.35e442bc celery==4.4.7 crypto-dev-signer~=0.4.14b2 confini~=0.3.6rc3 diff --git a/apps/util/liveness/Makefile b/apps/util/liveness/Makefile new file mode 100644 index 00000000..257775f8 --- /dev/null +++ b/apps/util/liveness/Makefile @@ -0,0 +1,10 @@ +docs: + mkdir -p doc/texinfo/html + makeinfo doc/texinfo/index.texi --html -o doc/texinfo/html/ + +markdown: doc + pandoc -f html -t markdown --standalone doc/texinfo/html/liveness.html -o README.md + + +.PHONY dist: + python setup.py sdist diff --git a/apps/util/liveness/README.md b/apps/util/liveness/README.md new file mode 100644 index 00000000..7081db01 --- /dev/null +++ b/apps/util/liveness/README.md @@ -0,0 +1,105 @@ +--- +description: liveness (Untitled Document) +distribution: global +Generator: makeinfo +keywords: liveness (Untitled Document) +lang: en +resource-type: document +title: liveness (Untitled Document) +--- + +[]{#liveness}[]{#liveness-1} + +## 1 liveness {#liveness .chapter} + +[]{#ilveness_005foverview}[]{#Overview} + +### 1.1 Overview {#overview .section} + +This is a cluster-specific convenience setup for enabling a +Kubernetes-style liveness/readiness test as outlined in +. + +Conceptually, it provides an application with means to: + +- Run a collection of functions to validate sanity of the environment +- Set a no-error state before execution of the main routine +- Modify the error state during execution +- Invalidating all state when execution ends + +[]{#Python-module} + +### 1.2 Python module {#python-module .section} + +Three python methods are provided. + +[]{#load} + +#### 1.2.1 load {#load .subsection} + +This is meant to be called after configurations and environment has been +set up, but before the execution logic has commenced. + +It receives a list of externally defined fully-qualified python modules. +Each of these modules must implement the method `health(*args,**kwargs)` +in its global namespace. + +Any module returning `False` will cause a `RuntimeException`. + +The component will not trap any other exception from the modules. + +If successful, it will write the `pid` of the application to the +specified run data folder. By default this is `/run/`, but the +path can be modified if desired. + +[]{#set} + +#### 1.2.2 set {#set .subsection} + +This is meant to be called during the execution of the main program +routine begins. + +[]{#at-startup} + +#### 1.2.2.1 at startup {#at-startup .subsubsection} + +It should be called once at the *start* of execution of the main program +routine. + +For one-shot routines, this would mean the start of any code only run +when the module name is `__main__`. + +For daemons, it would be just before handing over execution to the main +loop. + +[]{#during-execution} + +#### 1.2.2.2 during execution {#during-execution .subsubsection} + +Call `set(error_code=, ...` any time the health state temporarily +changes. Any `error` value other than `0` is considered an unhealthy +state. + +[]{#at-shutdown} + +#### 1.2.2.3 at shutdown {#at-shutdown .subsubsection} + +Call `reset(...)`, which will indicate that the state is to be +considered the same as at startup. + +[]{#shell} + +### 1.3 shell {#shell .section} + +A bash script is provided for *Kubernetes* to perform the health check. + +It performs the following checks: + +1. A numeric value exists in `//pid`{.sample}. +2. The numeric value is a directory in `/proc`{.sample} (a valid pid) +3. The file `//error`{.sample} contains \"0\" + +If any of these checks fail should inditcate that the container is +unhealthy. + +------------------------------------------------------------------------ diff --git a/apps/util/liveness/doc/texinfo/index.texi b/apps/util/liveness/doc/texinfo/index.texi new file mode 100644 index 00000000..82972d98 --- /dev/null +++ b/apps/util/liveness/doc/texinfo/index.texi @@ -0,0 +1,71 @@ +@node liveness +@chapter liveness + +@anchor{ilveness_overview} +@section Overview + +This is a cluster-specific convenience setup for enabling a Kubernetes-style liveness/readiness test as outlined in @url{https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/}. + +Conceptually, it provides an application with means to: + +@itemize +@item Run a collection of functions to validate sanity of the environment +@item Set a no-error state before execution of the main routine +@item Modify the error state during execution +@item Invalidating all state when execution ends +@end itemize + + +@section Python module + +Three python methods are provided. + +@subsection load + +This is meant to be called after configurations and environment has been set up, but before the execution logic has commenced. + +It receives a list of externally defined fully-qualified python modules. Each of these modules must implement the method @code{health(*args,**kwargs)} in its global namespace. + +Any module returning @code{False} will cause a @code{RuntimeException}. + +The component will not trap any other exception from the modules. + +If successful, it will write the @code{pid} of the application to the specified run data folder. By default this is @code{/run/}, but the path can be modified if desired. + + +@subsection set + +This is meant to be called during the execution of the main program routine begins. + +@subsubsection at startup + +It should be called once at the @emph{start} of execution of the main program routine. + +For one-shot routines, this would mean the start of any code only run when the module name is @code{__main__}. + +For daemons, it would be just before handing over execution to the main loop. + + +@subsubsection during execution + +Call @code{set(error_code=, ...} any time the health state temporarily changes. Any @code{error} value other than @code{0} is considered an unhealthy state. + + +@subsubsection at shutdown + +Call @code{reset(...)}, which will indicate that the state is to be considered the same as at startup. + + +@section shell + +A bash script is provided for @emph{Kubernetes} to perform the health check. + +It performs the following checks: + +@enumerate +@item A numeric value exists in @file{//pid}. +@item The numeric value is a directory in @file{/proc} (a valid pid) +@item The file @file{//error} contains "0" +@end enumerate + +If any of these checks fail should inditcate that the container is unhealthy. diff --git a/apps/util/liveness/tests/imports/import_args.py b/apps/util/liveness/tests/imports/import_args.py new file mode 100644 index 00000000..a68003b1 --- /dev/null +++ b/apps/util/liveness/tests/imports/import_args.py @@ -0,0 +1,8 @@ +a = ['foo'] +kw = { + 'bar': 42, + } + +def health(*args, **kwargs): + args[0] == a[0] + kwargs['bar'] = kw['bar']