Add docstrings for config, test for config

This commit is contained in:
nolash 2021-08-23 08:27:48 +02:00
parent f1ffff8b90
commit 96e0a97c3b
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
8 changed files with 251 additions and 24 deletions

View File

@ -17,6 +17,13 @@ logg = logging.getLogger(__name__)
def stdin_arg(): def stdin_arg():
"""Retreive input arguments from stdin if they exist.
Method does not block, and expects arguments to be ready on stdin before being called.
:rtype: str
:returns: Input arguments string
"""
h = select.select([sys.stdin], [], [], 0) h = select.select([sys.stdin], [], [], 0)
if len(h[0]) > 0: if len(h[0]) > 0:
v = h[0][0].read() v = h[0][0].read()
@ -25,6 +32,23 @@ def stdin_arg():
class ArgumentParser(argparse.ArgumentParser): class ArgumentParser(argparse.ArgumentParser):
"""Extends the standard library argument parser to construct arguments based on configuration flags.
The extended class is set up to facilitate piping of single positional arguments via stdin. For this reason, positional arguments should be added using the locally defined add_positional method instead of add_argument.
Calls chainlib.cli.args.ArgumentParser.process_flags with arg_flags and env arguments, see the method's documentation for further details.
:param arg_flags: Argument flag bit vector to generate configuration values for.
:type arg_flags: chainlib.cli.Flag
:param env: Environment variables
:type env: dict
:param usage: Usage string, passed to parent
:type usage: str
:param description: Description string, passed to parent
:type description: str
:param epilog: Epilog string, passed to parent
:type epilog: str
"""
def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs): def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs):
super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog) super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog)
@ -33,10 +57,34 @@ class ArgumentParser(argparse.ArgumentParser):
def add_positional(self, name, type=str, help=None, required=True): def add_positional(self, name, type=str, help=None, required=True):
"""Add a positional argument.
Stdin piping will only be possible in the event a single positional argument is defined.
If the "required" is set, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line.
:param name: Attribute name of argument
:type name: str
:param type: Argument type
:type type: str
:param help: Help string
:type help: str
:param required: If true, argument will be set to required
:type required: bool
"""
self.pos_args.append((name, type, help, required,)) self.pos_args.append((name, type, help, required,))
def parse_args(self, argv=sys.argv[1:]): def parse_args(self, argv=sys.argv[1:]):
"""Overrides the argparse.ArgumentParser.parse_args method.
Implements reading arguments from stdin if a single positional argument is defined (and not set to required).
If the "required" was set for the single positional argument, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line.
:param argv: Argument vector to process
:type argv: list
"""
if len(self.pos_args) == 1: if len(self.pos_args) == 1:
arg = self.pos_args[0] arg = self.pos_args[0]
self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2]) self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
@ -62,6 +110,20 @@ class ArgumentParser(argparse.ArgumentParser):
def process_flags(self, arg_flags, env): def process_flags(self, arg_flags, env):
"""Configures the arguments of the parser using the provided flags.
Environment variables are used for default values for:
CONFINI_DIR: -c, --config
CONFINI_ENV_PREFIX: --env-prefix
This method is called by the constructor, and is not intended to be called directly.
:param arg_flags: Argument flag bit vector to generate configuration values for.
:type arg_flags: chainlib.cli.Flag
:param env: Environment variables
:type env: dict
"""
if arg_flags & Flag.VERBOSE: if arg_flags & Flag.VERBOSE:
self.add_argument('-v', action='store_true', help='Be verbose') self.add_argument('-v', action='store_true', help='Be verbose')
self.add_argument('-vv', action='store_true', help='Be more verbose') self.add_argument('-vv', action='store_true', help='Be more verbose')

View File

@ -11,22 +11,87 @@ from .base import (
default_config_dir as default_parent_config_dir, default_config_dir as default_parent_config_dir,
) )
#logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
logg = logging.getLogger()
def logcallback(config): def logcallback(config):
"""Callback to dump config contents to log after completed config load
:param config: Config object
:type config: confini.Config
"""
logg.debug('config loaded:\n{}'.format(config)) logg.debug('config loaded:\n{}'.format(config))
class Config(confini.Config): class Config(confini.Config):
"""Extends confini.Config.
Processes argument parser attributes to configuration variables.
Provides sane configuration overrides and fallbacks.
"""
default_base_config_dir = default_parent_config_dir default_base_config_dir = default_parent_config_dir
default_fee_limit = 0 default_fee_limit = 0
@classmethod @classmethod
def from_args(cls, args, arg_flags, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback): def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback):
"""Parses arguments in argparse.ArgumentParser instance, then match and override configuration values that match them.
The method processes all known argument flags from chainlib.cli.Flag passed in the "args" argument.
All entries in extra_args may be used to associate arguments not defined in the argument flags with configuration variables, in the following manner:
- The value of argparser.ArgumentParser instance attribute with the dictionary key string is looked up.
- If the value is None (defined but empty), any existing value for the configuration directive will be kept.
- If the value of the extra_args dictionary entry is None, then the value will be stored in the configuration under the upper-case value of the key string, prefixed with "_" ("foo_bar" becomes "_FOO_BAR")
- If the value of the extra_args dictionary entries is a string, then the value will be stored in the configuration under that literal string.
Missing attributes defined by both the "args" and "extra_args" arguments will both raise an AttributeError.
The python package "confini" is used to process and render the configuration.
The confini config schema is determined in the following manner:
- If nothing is set, only the config folder in chainlib.data.config will be used as schema.
- If base_config_dir is a string or list, the config directives from the path(s) will be added to the schema.
The global override config directories are determined in the following manner:
- If no default_config_dir is defined, the environment variable CONFINI_DIR will be used.
- If default_config_dir is a string or list, values from the config directives from the path(s) will override those defined in the schema(s).
The user override config directories work the same way as the global ones, but the namespace - if defined - are dependent on them. They are only applied if the CONFIG arg flag is set. User override config directories are determined in the following manner:
- If --config argument is not defined and the pyxdg module is present, the first available xdg basedir is used.
- If --config argument is defined, the directory defined by its value will be used.
The namespace, if defined, will be stored under the CONFIG_USER_NAMESPACE configuration key.
:param args: Argument parser object
:type args: argparse.ArgumentParser
:param arg_flags: Argument flags defining which arguments to process into configuration.
:type arg_flags: confini.cli.args.ArgumentParser
:param env: Environment variables selection
:type env: dict
:param extra_args: Extra arguments to process and override.
:type extra_args: dict
:param base_config_dir: Path(s) to one or more directories extending the base chainlib config schema.
:type base_config_dir: list or str
:param default_config_dir: Path(s) to one or more directories overriding the defaults defined in the schema config directories.
:type default_config_dir: list or str
:param user_config_dir: User xdg config basedir, with namespace
:type user_config_dir: str
:param default_fee_limit: Default value for fee limit argument
:type default_fee_limit: int
:param logger: Logger instance to use during argument processing (will use package namespace logger if None)
:type logger: logging.Logger
:param load_callback: Callback receiving config instance as argument after config processing and load completes.
:type load_callback: function
:raises AttributeError: Attribute defined in flag not found in parsed arguments
:rtype: confini.Config
:return: Processed configuation
"""
if logger == None: if logger == None:
logger = logging.getLogger() logger = logging.getLogger()
@ -58,7 +123,7 @@ class Config(confini.Config):
# confini dir env var will be used for override configs only in this case # confini dir env var will be used for override configs only in this case
if default_config_dir == None: if default_config_dir == None:
default_config_dir = os.environ.get('CONFINI_DIR') default_config_dir = env.get('CONFINI_DIR')
if default_config_dir != None: if default_config_dir != None:
if isinstance(default_config_dir, str): if isinstance(default_config_dir, str):
default_config_dir = [default_config_dir] default_config_dir = [default_config_dir]
@ -67,32 +132,27 @@ class Config(confini.Config):
# process config command line arguments # process config command line arguments
if arg_flags & Flag.CONFIG: if arg_flags & Flag.CONFIG:
effective_user_config_dir = getattr(args, 'config', None) effective_user_config_dir = getattr(args, 'config', None)
if effective_user_config_dir == None: if effective_user_config_dir == None:
effective_user_config_dir = user_config_dir effective_user_config_dir = user_config_dir
if effective_user_config_dir != None: if effective_user_config_dir != None:
if config_dir == None:
if getattr(args, 'namespace', None) != None: if getattr(args, 'namespace', None) != None:
arg_config_dir = os.path.join(effective_user_config_dir, args.namespace) effective_user_config_dir = os.path.join(effective_user_config_dir, args.namespace)
config_dir = [cls.default_base_config_dir, effective_user_config_dir] #if config_dir == None:
logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir)) # config_dir = [cls.default_base_config_dir, effective_user_config_dir]
else: # logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir))
if getattr(args, 'namespace', None) != None: #else:
arg_config_dir = os.path.join(effective_user_config_dir, args.namespace)
override_config_dirs.append(effective_user_config_dir) override_config_dirs.append(effective_user_config_dir)
logg.debug('using config arg as config override {}'.format(effective_user_config_dir)) logg.debug('using config arg as config override {}'.format(effective_user_config_dir))
#if config_dir == None:
if config_dir == None: # if default_config_dir == None:
if default_config_dir == None: # default_config_dir = default_parent_config_dir
default_config_dir = default_parent_config_dir # config_dir = default_config_dir
config_dir = default_config_dir # override_config_dirs = []
override_config_dirs = []
env_prefix = getattr(args, 'env_prefix', None) env_prefix = getattr(args, 'env_prefix', None)
config = confini.Config(config_dir, env_prefix=args.env_prefix, override_dirs=override_config_dirs) config = confini.Config(config_dir, env_prefix=env_prefix, override_dirs=override_config_dirs)
config.process() config.process()
args_override = {} args_override = {}
@ -140,6 +200,9 @@ class Config(confini.Config):
config.add(getattr(args, 'raw'), '_RAW') config.add(getattr(args, 'raw'), '_RAW')
if arg_flags & Flag.CONFIG:
config.add(getattr(args, 'namespace'), 'CONFIG_USER_NAMESPACE')
for k in extra_args.keys(): for k in extra_args.keys():
v = extra_args[k] v = extra_args[k]
if v == None: if v == None:

View File

@ -2,7 +2,7 @@
set -e set -e
set -x set -x
export PYTHONPATH=${PYTHONPATH:.} export PYTHONPATH=$PYTHONPATH:.
for f in `ls tests/*.py`; do for f in `ls tests/*.py`; do
python $f python $f
done done

94
tests/test_cli.py Normal file
View File

@ -0,0 +1,94 @@
# standard imports
import unittest
import os
# local imports
import chainlib.cli
from chainlib.cli.base import argflag_std_base
script_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(script_dir, 'testdata')
config_dir = os.path.join(data_dir, 'config')
class TestCli(unittest.TestCase):
def test_args_process_single(self):
ap = chainlib.cli.arg.ArgumentParser()
argv = [
'-vv',
'-n',
'foo',
]
args = ap.parse_args(argv)
config = chainlib.cli.config.Config.from_args(args)
self.assertEqual(config.get('CONFIG_USER_NAMESPACE'), 'foo')
def test_args_process_schema_override(self):
ap = chainlib.cli.arg.ArgumentParser()
args = ap.parse_args([])
config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir)
self.assertEqual(config.get('FOO_BAR'), 'baz')
def test_args_process_arg_override(self):
ap = chainlib.cli.arg.ArgumentParser()
argv = [
'-c',
config_dir,
'-n',
'foo',
]
args = ap.parse_args(argv)
config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir)
self.assertEqual(config.get('FOO_BAR'), 'bazbazbaz')
def test_args_process_internal_override(self):
ap = chainlib.cli.arg.ArgumentParser()
args = ap.parse_args()
default_config_dir = os.path.join(config_dir, 'default')
config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir)
self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo')
user_config_dir = os.path.join(default_config_dir, 'user')
config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=user_config_dir)
self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo')
config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir)
self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo')
ap = chainlib.cli.arg.ArgumentParser()
argv = [
'-n',
'user',
]
args = ap.parse_args(argv)
config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir)
self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo')
def test_args_process_extra(self):
ap = chainlib.cli.arg.ArgumentParser()
ap.add_argument('--foo', type=str)
argv = [
'--foo',
'bar',
]
args = ap.parse_args(argv)
extra_args = {
'foo': None,
}
config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args)
self.assertEqual(config.get('_FOO'), 'bar')
extra_args = {
'foo': 'FOOFOO',
}
config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args)
self.assertEqual(config.get('FOOFOO'), 'bar')
if __name__ == '__main__':
unittest.main()

2
tests/testdata/config/config.ini vendored Normal file
View File

@ -0,0 +1,2 @@
[foo]
bar = baz

View File

@ -0,0 +1,2 @@
[chain]
spec = baz:bar:13:foo

View File

@ -0,0 +1,2 @@
[chain]
spec = foo:foo:666:foo

2
tests/testdata/config/foo/config.ini vendored Normal file
View File

@ -0,0 +1,2 @@
[foo]
bar = bazbazbaz