Bump vesrion

Reviewed-on: #2

CHANGELOG

@@ -1,14 +1,36 @@
-- 0.0.5-pending
+- 0.0.23
+	* Configuration variable descriptions
+	* Arg flags to names listing method
+	* New flags preset for reads without wallet
+	* Remove pesistent wallet arg flag bug
+	* Rename arg flag reset to flag_reset
+- 0.0.22
+	* Man page generator script
+- 0.0.21
+	* Log rpc reply before json parse
+- 0.0.20
+	* Add edit option for some long flag aliases of short flags: -a, -e, -s and -y
+- 0.0.19
+	* Passphrase file option to unlock keyfile for CLI tooling
+- 0.0.18
+	* Upgrade hexathon skipping buggy compact hex method
+- 0.0.17
+	* Add loglevel environment variable
+- 0.0.16
+	* Allow query string in query url
+- 0.0.14
+	* Add option to skip ssl verification on rpc
+- 0.0.5
 	* Move eth code to separate package
-- 0.0.4-unreleased
+- 0.0.4
 	* Add pack tx from already signed tx struct
 	* Add http auth handling for jsonrpc connections
 	* Add customizable jsonrpc id generator (to allow for buggy server id handling)
-- 0.0.3-unreleased
+- 0.0.3
 	* Remove erc20 module (to new external package)
-- 0.0.2-unreleased
+- 0.0.2
 	* 
-- 0.0.1-unreleased
+- 0.0.1
 	* Add eth tx decode
 	* Add eth balance query with erc20 option
 	* Add eth checksum address

MANIFEST.in

@@ -1 +1 @@
-include *requirements.txt LICENSE chainlib/data/config/*
+include *requirements.txt LICENSE chainlib/data/config/* chainlib/data/env/*

chainlib/chain.py

@@ -1,5 +1,15 @@
 # standard imports
 import copy
+import re
+
+
+def is_valid_label(v, alpha_only=False):
+    re_m = None
+    if alpha_only:
+        re_m = r'^[a-zA-Z]+$'
+    else:
+        re_m = r'^[a-zA-Z0-9]+$'
+    return re.match(re_m, v)
 
 
 class ChainSpec:
@@ -16,13 +26,37 @@ class ChainSpec:
     :param tag: Descriptive tag
     :type tag: str
     """
-    def __init__(self, arch, fork, network_id, common_name=None):
+    def __init__(self, arch, fork, network_id, common_name=None, custom=[], safe=True):
+        if custom == None:
+            custom = []
+        elif not isinstance(custom, list):
+            raise ValueError('custom value must be list')
+
         self.o = {
-                'arch': arch,
-                'fork': fork,
-                'network_id': network_id,
-                'common_name': common_name,
-                }
+        'arch': arch,
+        'fork': fork,
+        'network_id': network_id,
+        'common_name': common_name,
+        'custom': custom,
+        }
+
+        if safe:
+            self.validate()
+
+
+    def validate(self):
+        self.o['network_id'] = int(self.o['network_id'])
+        if not is_valid_label(self.o['arch'], alpha_only=True):
+            raise ValueError('arch: ' + self.o['arch'])
+        if not is_valid_label(self.o['fork'], alpha_only=True):
+            raise ValueError('fork: ' + self.o['fork'])
+        if self.o.get('common_name') and not is_valid_label(self.o['common_name']):
+            raise ValueError('common_name: ' + self.o['common_name'])
+        if self.o.get('custom'):
+            for i, v in enumerate(self.o['custom']):
+                if not is_valid_label(v):
+                    raise ValueError('common_name {}: {}'.format(i, v))
+
 
     def network_id(self):
         """Returns the network id part of the spec.
@@ -43,6 +77,12 @@ class ChainSpec:
 
 
     def engine(self):
+        """Alias of self.arch()
+        """
+        return self.arch()
+
+
+    def arch(self):
         """Returns the chain architecture part of the spec
 
         :rtype: str
@@ -51,6 +91,15 @@ class ChainSpec:
         return self.o['arch']
 
 
+    def fork(self):
+        """Returns the fork part of the spec
+
+        :rtype: str
+        :returns: fork
+        """
+        return self.o['fork']
+
+
     def common_name(self):
         """Returns the common name part of the spec
 
@@ -60,6 +109,21 @@ class ChainSpec:
         return self.o['common_name']
 
 
+    def is_same_as(self, chain_spec_cmp, use_common_name=False, use_custom=False):
+        a = ['arch', 'fork', 'network_id']
+        if use_common_name:
+            a += ['common_name']
+        if use_custom:
+            a += ['custom']
+        try:
+            for k in a:
+                assert(chain_spec_cmp.o[k] == self.o[k])
+        except AssertionError:
+            return False
+        return True
+                
+
+
     @staticmethod
     def from_chain_str(chain_str):
         """Create a new ChainSpec object from a colon-separated string, as output by the string representation of the ChainSpec object.
@@ -79,9 +143,13 @@ class ChainSpec:
         if len(o) < 3:
             raise ValueError('Chain string must have three sections, got {}'.format(len(o)))
         common_name = None
-        if len(o) == 4:
+        if len(o) > 3:
             common_name = o[3]
-        return ChainSpec(o[0], o[1], int(o[2]), common_name)
+        custom = []
+        if len(o) > 4:
+            for i in range(4, len(o)):
+                custom.append(o[i])
+        return ChainSpec(o[0], o[1], int(o[2]), common_name=common_name, custom=custom)
 
 
     @staticmethod
@@ -100,20 +168,35 @@ class ChainSpec:
         :rtype: chainlib.chain.ChainSpec
         :returns: Resulting chain spec
         """
-        return ChainSpec(o['arch'], o['fork'], o['network_id'], common_name=o['common_name'])
+        return ChainSpec(o['arch'], o['fork'], o['network_id'], common_name=o.get('common_name'), custom=o.get('custom'))
 
   
-    def asdict(self):
+    def asdict(self, use_common_name=True, use_custom=True):
         """Create a dictionary representation of the chain spec.
 
         :rtype: dict
         :returns: Chain spec dictionary
         """
-        return copy.copy(self.o)
+        r = copy.copy(self.o)
+        if not use_common_name:
+            del r['common_name']
+            del r['custom']
+        if not use_custom:
+            del r['custom']
+        return r
+
+
+    def as_string(self, skip_optional=False):
+        s = '{}:{}:{}'.format(self.o['arch'], self.o['fork'], self.o['network_id'])
+        if skip_optional:
+            return s
+
+        if self.o.get('common_name'):
+            s += ':' + self.o['common_name']
+        if self.o.get('custom'):
+            s += ':' + ':'.join(self.o['custom'])
+        return s
 
 
     def __str__(self):
-        s = '{}:{}:{}'.format(self.o['arch'], self.o['fork'], self.o['network_id'])
-        if self.o['common_name'] != None:
-            s += ':' + self.o['common_name']
-        return s
+        return self.as_string()        

chainlib/cli/__init__.py

@@ -3,7 +3,9 @@ from .base import (
         argflag_std_read,
         argflag_std_write,
         argflag_std_base,
-        reset,
+        argflag_std_base_read,
+        flag_reset,
+        flag_set,
         )
 from .arg import ArgumentParser
 from .config import Config

chainlib/cli/arg.py

@@ -5,6 +5,7 @@ import enum
 import os
 import select
 import sys
+import re
 
 # local imports
 from .base import (
@@ -30,16 +31,35 @@ def stdin_arg():
         return v.rstrip()
     return None
 
+_default_long_args = {
+        '-a': '--recipient',
+        '-e': '--executable-address',
+        '-s': '--send',
+        '-y': '--key-file',
+                }
+
+_default_dest = {
+        '-a': 'recipient',
+        '-e': 'executable_address',
+        }
+
+
+_default_fmt = 'human'
+
 
 class ArgumentParser(argparse.ArgumentParser):
     """Extends the standard library argument parser to construct arguments based on configuration flags.
 
     The extended class is set up to facilitate piping of single positional arguments via stdin. For this reason, positional arguments should be added using the locally defined add_positional method instead of add_argument.
 
+    Long flag aliases for short flags are editable using the arg_long argument. Editing a non-existent short flag will produce no error and have no effect. Adding a long flag for a short flag that does not have an alias will also not have effect.
+
     Calls chainlib.cli.args.ArgumentParser.process_flags with arg_flags and env arguments, see the method's documentation for further details.
 
     :param arg_flags: Argument flag bit vector to generate configuration values for.
     :type arg_flags: chainlib.cli.Flag
+    :param arg_long: Change long flag alias for given short flags. Example value: {'-a': '--addr', '-e': '--contract'}
+    :type arg_long: dict
     :param env: Environment variables
     :type env: dict
     :param usage: Usage string, passed to parent
@@ -50,14 +70,27 @@ class ArgumentParser(argparse.ArgumentParser):
     :type epilog: str
     """
 
-    def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs):
+    def __init__(self, arg_flags=0x0f, arg_long={}, env=os.environ, usage=None, description=None, epilog=None, default_format=_default_fmt, *args, **kwargs):
         super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter, *args, **kwargs)
     
-        self.process_flags(arg_flags, env)
         self.pos_args = []
+        self.long_args = _default_long_args
+        self.arg_dest = _default_dest
+        self.default_format = default_format
+
+        re_long = r'^--[a-z\-]+$'
+        for k in arg_long.keys():
+            if re.match(re_long, arg_long[k]) == None:
+                raise ValueError('invalid long arg {}'.format(arg_long[k]))
+            self.long_args[k] = arg_long[k]
+            dest = arg_long[k][2:]
+            dest = dest.replace('-', '_')
+            self.arg_dest[k] = dest
+
+        self.process_flags(arg_flags, env)
 
 
-    def add_positional(self, name, type=str, help=None, required=True):
+    def add_positional(self, name, type=str, help=None, append=False, required=True):
         """Add a positional argument.
 
         Stdin piping will only be possible in the event a single positional argument is defined.
@@ -73,7 +106,7 @@ class ArgumentParser(argparse.ArgumentParser):
         :param required: If true, argument will be set to required
         :type required: bool
         """
-        self.pos_args.append((name, type, help, required,))
+        self.pos_args.append((name, type, help, required, append,))
 
 
     def parse_args(self, argv=sys.argv[1:]):
@@ -88,16 +121,25 @@ class ArgumentParser(argparse.ArgumentParser):
         """
         if len(self.pos_args) == 1:
             arg = self.pos_args[0]
-            self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
+            if arg[4]:
+                self.add_argument(arg[0], nargs='*', type=arg[1], default=stdin_arg(), help=arg[2])
+            else:
+                self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
         else:
             for arg in self.pos_args:
                 if arg[3]:
-                    self.add_argument(arg[0], type=arg[1], help=arg[2])
+                    if arg[4]:
+                        self.add_argument(arg[0], nargs='+', type=arg[1], help=arg[2])
+                    else:
+                        self.add_argument(arg[0], type=arg[1], help=arg[2])
                 else:
-                    self.add_argument(arg[0], nargs='?', type=arg[1], help=arg[2])
+                    if arg[4]:
+                        self.add_argument(arg[0], nargs='*', type=arg[1], help=arg[2])
+                    else:
+                        self.add_argument(arg[0], type=arg[1], help=arg[2])
         args = super(ArgumentParser, self).parse_args(args=argv)
 
-        if args.dumpconfig:
+        if getattr(args, 'dumpconfig', None) != None:
             return args
 
         if len(self.pos_args) == 1:
@@ -129,12 +171,13 @@ class ArgumentParser(argparse.ArgumentParser):
         :type env: dict
         """
         if arg_flags & Flag.VERBOSE:
+            self.add_argument('--no-logs', dest='no_logs',action='store_true', help='Turn off all logging')
             self.add_argument('-v', action='store_true', help='Be verbose')
             self.add_argument('-vv', action='store_true', help='Be more verbose')
         if arg_flags & Flag.CONFIG:
             self.add_argument('-c', '--config', type=str, default=env.get('CONFINI_DIR'), help='Configuration directory')
             self.add_argument('-n', '--namespace', type=str, help='Configuration namespace')
-            self.add_argument('--dumpconfig', action='store_true', help='Output configuration and quit. Use with --raw to omit values and output schema only.')
+            self.add_argument('--dumpconfig', type=str, choices=['env', 'ini'], help='Output configuration and quit. Use with --raw to omit values and output schema only.')
         if arg_flags & Flag.WAIT:
             self.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
             self.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')
@@ -143,7 +186,8 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & Flag.PROVIDER:
             self.add_argument('-p', '--rpc-provider', dest='p', type=str, help='RPC HTTP(S) provider url')
             self.add_argument('--rpc-dialect', dest='rpc_dialect', type=str, help='RPC HTTP(S) backend dialect')
-            self.add_argument('--height', default='latest', help='Block height to execute against')
+            if arg_flags & Flag.NO_TARGET == 0:
+                self.add_argument('--height', default='latest', help='Block height to execute against')
             if arg_flags & Flag.RPC_AUTH:
                 self.add_argument('--rpc-auth', dest='rpc_auth', type=str, help='RPC autentication scheme')
                 self.add_argument('--rpc-credentials', dest='rpc_credentials', type=str, help='RPC autentication credential values')
@@ -154,9 +198,10 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & Flag.SEQ:
             self.add_argument('--seq', action='store_true', help='Use sequential rpc ids')
         if arg_flags & Flag.KEY_FILE:
-            self.add_argument('-y', '--key-file', dest='y', type=str, help='Keystore file to use for signing or address')
+            self.add_argument('-y', self.long_args['-y'], dest='y', type=str, help='Keystore file to use for signing or address')
+            self.add_argument('--passphrase-file', dest='passphrase_file', type=str, help='File containing passphrase for keystore')
         if arg_flags & Flag.SEND:
-            self.add_argument('-s', '--send', dest='s', action='store_true', help='Send to network')
+            self.add_argument('-s', self.long_args['-s'], dest='s', action='store_true', help='Send to network')
         if arg_flags & Flag.RAW:
             self.add_argument('--raw', action='store_true', help='Do not decode output')
         if arg_flags & (Flag.SIGN | Flag.NONCE):
@@ -164,9 +209,19 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & (Flag.SIGN | Flag.FEE):
             self.add_argument('--fee-price', dest='fee_price', type=int, help='override fee price')
             self.add_argument('--fee-limit', dest='fee_limit', type=int, help='override fee limit')
-        if arg_flags & argflag_std_target == 0:
-            arg_flags |= Flag.WALLET
+        # wtf?
+        #if arg_flags & argflag_std_target == 0:
+        #    arg_flags |= Flag.WALLET
         if arg_flags & Flag.EXEC:
-            self.add_argument('-e', '--exectuable-address', dest='executable_address', type=str, help='contract address')
+            self.add_argument('-e', self.long_args['-e'], dest=self.arg_dest['-e'], type=str, help='contract address')
         if arg_flags & Flag.WALLET:
-            self.add_argument('-a', '--recipient', dest='recipient', type=str, help='recipient address')
+            self.add_argument('-a', self.long_args['-a'], dest=self.arg_dest['-a'], type=str, help='recipient address')
+        if arg_flags & (Flag.FMT_HUMAN | Flag.FMT_WIRE | Flag.FMT_RPC):
+            format_choices = []
+            if arg_flags & Flag.FMT_HUMAN:
+                format_choices.append('human')
+            if arg_flags & Flag.FMT_WIRE:
+                format_choices.append('bin')
+            if arg_flags & Flag.FMT_RPC:
+                format_choices.append('rpc')
+            self.add_argument('-f', '--format', type=str, choices=format_choices, help='output formatting (default: {})'.format(self.default_format))

chainlib/cli/base.py

@@ -33,14 +33,40 @@ class Flag(enum.IntEnum):
     SEND = 262144
     # rpc extras - nibble 6
     RPC_AUTH = 1048576
+    # formatting - nibble 7
+    FMT_HUMAN = 16777216
+    FMT_WIRE = 33554432
+    FMT_RPC = 67108864
+    # upper bound
+    MAX = 1048576
 
-argflag_std_read = 0x23ff 
-argflag_std_write = 0xff31ff
-argflag_std_base = 0x200f
-argflag_std_target = 0x00e000
-argflag_all = 0xffffff
+argflag_std_read = 0x000023ff
+argflag_std_write = 0x001731ff
+argflag_std_base = 0x0000200f
+argflag_std_base_read = 0x000000bf
+argflag_std_target = 0x0000e000
+argflag_all = 0x0317f7ff
 
-def reset(flags, v):
+
+def flag_reset(flags, v):
     mask = ~(argflag_all & v)
     r = flags & mask
     return r
+
+
+def flag_set(flags, v):
+    return flags | v
+
+
+def flag_names(flags):
+    flags_debug = []
+    i = Flag.MAX
+    while True:
+        if flags & i > 0:
+            v = Flag(i)
+            flags_debug.append(v.name)
+        i >>= 1
+        if i == 0:
+            break
+    flags_debug.reverse()
+    return flags_debug

chainlib/cli/config.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+import stat
 
 # external imports
 import confini
@@ -45,7 +46,7 @@ class Config(confini.Config):
 
 
     @classmethod
-    def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback, dump_writer=sys.stdout):
+    def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=None, dump_writer=sys.stdout):
         """Parses arguments in argparse.ArgumentParser instance, then match and override configuration values that match them.
 
         The method processes all known argument flags from chainlib.cli.Flag passed in the "args" argument. 
@@ -102,14 +103,51 @@ class Config(confini.Config):
         :rtype: confini.Config
         :return: Processed configuation
         """
+        env_prefix = getattr(args, 'env_prefix', None)
+        env_prefix_str = env_prefix
+        if env_prefix_str == None:
+            env_prefix_str = ''
+        else:
+            env_prefix_str += '_'
+
+        env_loglevel_key_str = env_prefix_str + 'LOGLEVEL'
+        env_loglevel = os.environ.get(env_loglevel_key_str)
+
         if logger == None:
             logger = logging.getLogger()
 
-        if arg_flags & Flag.CONFIG:
+        if env_loglevel != None:
+            env_loglevel = env_loglevel.lower()
+            if env_loglevel == '0' or env_loglevel == 'no' or env_loglevel == 'none' or env_loglevel == 'disable' or env_loglevel == 'disabled' or env_loglevel == 'off':
+                logging.disable()
+            elif env_loglevel == '1' or env_loglevel == 'err' or env_loglevel == 'error':
+                logger.setLevel(logging.ERROR)
+            elif env_loglevel == '2' or env_loglevel == 'warning' or env_loglevel == 'warn':
+                logger.setLevel(logging.WARNING)
+            elif env_loglevel == '3' or env_loglevel == 'info':
+                logger.setLevel(logging.INFO)
+            else:
+                valid_level = False
+                try:
+                    num_loglevel = int(env_loglevel)
+                    valid_level = True
+                except:
+                    if env_loglevel == 'debug':
+                        valid_level = True
+
+                if not valid_level:
+                    raise ValueError('unknown loglevel {} set in environment variable {}'.format(env_loglevel, env_loglevel_key_str))
+
+                logger.setLevel(logging.DEBUG)
+
+
+        if arg_flags & Flag.VERBOSE:
             if args.vv:
                 logger.setLevel(logging.DEBUG)
             elif args.v:
                 logger.setLevel(logging.INFO)
+            if args.no_logs:
+                logging.disable()
    
         override_config_dirs = []
         config_dir = [cls.default_base_config_dir]
@@ -160,7 +198,6 @@ class Config(confini.Config):
         #        default_config_dir = default_parent_config_dir
         #    config_dir = default_config_dir
         #    override_config_dirs = []
-        env_prefix = getattr(args, 'env_prefix', None)
 
         config = confini.Config(config_dir, env_prefix=env_prefix, override_dirs=override_config_dirs)
         config.process()
@@ -170,17 +207,24 @@ class Config(confini.Config):
         args_override = {}
    
         if arg_flags & Flag.PROVIDER:
-            args_override['RPC_HTTP_PROVIDER'] = getattr(args, 'p')
             args_override['RPC_PROVIDER'] = getattr(args, 'p')
             args_override['RPC_DIALECT'] = getattr(args, 'rpc_dialect')
         if arg_flags & Flag.CHAIN_SPEC:
             args_override['CHAIN_SPEC'] = getattr(args, 'i')
         if arg_flags & Flag.KEY_FILE:
             args_override['WALLET_KEY_FILE'] = getattr(args, 'y')
+            fp = getattr(args, 'passphrase_file')
+            if fp != None:
+                st = os.stat(fp)
+                if stat.S_IMODE(st.st_mode) & (stat.S_IRWXO | stat.S_IRWXG) > 0:
+                    logg.warning('others than owner have access on password file')
+                f = open(fp, 'r')
+                args_override['WALLET_PASSPHRASE'] = f.read()
+                f.close()
+            config.censor('PASSPHRASE', 'WALLET')
+        config.dict_override(args_override, 'cli args', allow_empty=True)
 
-        config.dict_override(args_override, 'cli args')
-
-        if arg_flags & Flag.PROVIDER:
+        if arg_flags & (Flag.PROVIDER | Flag.NO_TARGET) == Flag.PROVIDER:
             config.add(getattr(args, 'height'), '_HEIGHT')
         if arg_flags & Flag.UNSAFE:
             config.add(getattr(args, 'u'), '_UNSAFE')
@@ -225,6 +269,7 @@ class Config(confini.Config):
             config.add(getattr(args, 'rpc_credentials'), 'RPC_CREDENTIALS')
 
         for k in extra_args.keys():
+            logg.debug('extra_agrs {}'.format(k))
             v = extra_args[k]
             if v == None:
                 v = '_' + k.upper()
@@ -236,20 +281,29 @@ class Config(confini.Config):
                 pass
             if existing_r == None or r != None:
                 config.add(r, v, exists_ok=True)
+                logg.debug('added {} to {}'.format(r, v))
 
-        if getattr(args, 'dumpconfig'):
-            config_keys = config.all()
-            with_values = not config.get('_RAW')
-            for k in config_keys:
-                if k[0] == '_':
-                    continue
-                s = k + '='
-                if with_values:
-                    v = config.get(k)
-                    if v != None:
-                        s += str(v)
-                s += '\n'
-                dump_writer.write(s)
+        if getattr(args, 'dumpconfig', None):
+            if args.dumpconfig == 'ini':
+                from confini.export import ConfigExporter
+                exporter = ConfigExporter(config, target=sys.stdout, doc=False)
+                exporter.export(exclude_sections=['config'])
+            elif args.dumpconfig == 'env':
+                from confini.env import export_env
+                export_env(config)
+
+#            config_keys = config.all()
+#            with_values = not config.get('_RAW')
+#            for k in config_keys:
+#                if k[0] == '_':
+#                    continue
+#                s = k + '='
+#                if with_values:
+#                    v = config.get(k)
+#                    if v != None:
+#                        s += str(v)
+#                s += '\n'
+#                dump_writer.write(s)
             sys.exit(0)
 
         if load_callback != None:

chainlib/cli/man.py

@@ -0,0 +1,357 @@
+# standard imports
+import os
+
+# external imports
+import confini
+
+# local imports
+from .base import (
+        Flag,
+        argflag_std_target,
+        )
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+data_dir = os.path.join(script_dir, '..', 'data')
+
+
+def apply_groff(collection, v, arg=None, typ='arg'):
+    s = ''
+    for flag in collection:
+        if len(s) > 0:
+            s += ', '
+        s += format_groff(flag, v, arg=arg, typ=typ)
+    s = "\n.TP\n" + s + "\n" + v
+    return s
+
+
+def format_groff(k, v, arg=None, typ='arg'):
+    s = ''
+    if typ == 'env':
+        s += '\\fI'
+    else:
+        s += '\\fB'
+    s += k
+    if arg != None:
+        s += ' \\fI' + arg
+    s += '\\fP'
+    return s
+
+
+class DocEntry:
+
+    def __init__(self, *args, typ='arg'):
+        self.flags = args
+        self.render = self.get_empty
+        self.groff = None
+        self.typ = typ
+        self.v = None
+
+
+    def __check_line_default(self, m):
+        if self.render == self.get_empty:
+            self.render = m
+
+
+    def get_empty(self):
+        s = ''
+        for flag in self.flags:
+            if len(s) > 0:
+                s += ', '
+            s += flag
+   
+        s += '\n\t(undefined)\n'
+        return s
+
+
+    def set_groff(self, v):
+        self.__check_line_default(self.get_groff)
+        self.groff = v
+
+
+    def set_groff_argvalue(self, argvalue):
+        self.v = '\\fI' + argvalue + '\\fP'
+
+
+    def get_groff(self):
+        v = self.groff
+        if v == None:
+            v = self.plain
+        s = apply_groff(self.flags, v, arg=self.v, typ=self.typ) 
+        return s
+
+
+    def __str__(self):
+        return self.render()
+   
+
+class DocGenerator:
+
+#    def __init__(self, arg_flags, config):
+    def __init__(self, arg_flags):
+        #self.config = config
+        self.arg_flags = arg_flags
+        self.docs = {}
+        self.envs = {}
+#        self.envs = {}
+
+
+    def __str__(self):
+        s = ''
+        ks = list(self.docs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.docs[k]) 
+            env = self.envs.get(k)
+            if env != None:
+                s += ' Overrides the \\fI' + env + '\\fP configuration setting.'
+            s += "\n" 
+        return s
+
+
+    def get_args(self):
+        s = ''
+        ks = list(self.docs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.docs[k]) + "\n" 
+        return s
+
+
+    def set_arg(self, k, v, flags, argvalue=None):
+        o = DocEntry(*flags)
+        o.set_groff_argvalue(argvalue)
+        o.set_groff(v)
+        self.docs[k] = o
+
+
+    def override_arg(self, k, v, args, argvalue=None):
+        o = self.docs[k]
+        #g.docs[v[0]].groff = v[1].rstrip()
+        o.set_groff(v)
+        if argvalue != None:
+            o.set_groff_argvalue(argvalue)
+        l = len(args)
+        if l > 0:
+            o.flags = []
+        for i in range(l):
+            o.flags.append(args[i])
+
+#
+#    def process_env(self):
+#        for k in self.config.all():
+#            if k[0] == '_':
+#                continue
+#            self.envs[k] = None
+
+
+    def process_arg(self):
+
+        if self.arg_flags & Flag.VERBOSE:
+            o = DocEntry('--no-logs')
+            o.set_groff('Turn of logging completely. Negates \\fB-v\\fP and \\fB-vv\\fP')
+            self.docs['nologs'] = o
+
+            o = DocEntry('-v')
+            o.set_groff('Verbose. Show logs for important state changes.')
+            self.docs['v'] = o
+
+            o = DocEntry('-vv')
+            o.set_groff('Very verbose. Show logs with debugging information.')
+            self.docs['vv'] = o
+
+        if self.arg_flags & Flag.CONFIG:
+            o = DocEntry('-c', '--config')
+            o.set_groff('Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.')
+            o.set_groff_argvalue('config_dir')
+            self.docs['c'] = o
+
+            o = DocEntry('-n', '--namespace')
+            o.set_groff('Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.')
+            o.set_groff_argvalue('namespace')
+            self.docs['n'] = o
+           
+            o = DocEntry('--dumpconfig')
+            o.set_groff('Output configuration settings rendered from environment and inputs. Valid arguments are \\fIini\\fP for ini file output, and \\fIenv\\fP for environment variable output. See \\fBCONFIGURATION\\fP.')
+            o.set_groff_argvalue('format')
+            self.docs['dumpconfig'] = o
+
+
+        if self.arg_flags & Flag.WAIT:
+            o = DocEntry('-w')
+            o.set_groff('Wait for the last transaction to be confirmed on the network. Will generate an error if the EVM execution fails.')
+            self.docs['w'] = o
+
+            o = DocEntry('-ww')
+            o.set_groff('Wait for \\fIall\\fP transactions sequentially to be confirmed on the network. Will generate an error if EVM execution fails for any of the transactions.')
+            self.docs['ww'] = o
+
+
+        if self.arg_flags & Flag.ENV_PREFIX:
+            o = DocEntry('--env-prefix')
+            o.set_groff('Environment prefix for variables to overwrite configuration. Example: If \\fB--env-prefix\\fP is set to \\fBFOO\\fP then configuration variable \\fBBAR_BAZ\\fP would be set by environment variable \\fBFOO_BAZ_BAR\\fP. Also see \\fBENVIRONMENT\\fP.')
+            self.docs['envprefix'] = o
+
+        
+        if self.arg_flags & Flag.PROVIDER:
+            o = DocEntry('-p', '--rpc-provider')
+            o.set_groff('Fully-qualified URL of RPC provider.')
+            self.docs['p'] = o
+            self.envs['p'] = 'RPC_PROVIDER'
+
+            o = DocEntry('--rpc-dialect')
+            o.set_groff('RPC backend dialect. If specified it \\fImay\\fP help with encoding and decoding issues.')
+            self.docs['rpcdialect'] = o
+            self.envs['rpcdialect'] = 'RPC_DIALECT'
+
+            if self.arg_flags & Flag.NO_TARGET == 0:
+                o = DocEntry('--height')
+                o.set_groff('Block height at which to query state for. Does not apply to transactions.')
+                self.docs['height'] = o
+
+            if self.arg_flags & Flag.RPC_AUTH:
+                o = DocEntry('--rpc-auth')
+                o.set_groff('RPC endpoint authentication method, e.g. how to handle a HTTP WWW-Authenticate header.')
+                self.docs['rpcauth'] = o
+                self.envs['rpcauth'] = 'RPC_AUTH'
+
+                o = DocEntry('--rpc-credentials')
+                o.set_groff('RPC endpoint authentication data. Format depends on the authentication method defined in \\fB--rpc-auth\\fP.')
+                self.docs['rpccredentials'] = o
+                self.envs['rpccredentials'] = 'RPC_CREDENTIALS'
+
+
+        if self.arg_flags & Flag.CHAIN_SPEC:
+            o = DocEntry('-i', '--chain-spec')
+            o.set_groff('Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum".')
+            o.set_groff_argvalue('chain_spec')
+            self.docs['i'] = o
+            self.envs['i'] = 'RPC_CREDENTIALS'
+
+
+        if self.arg_flags & Flag.UNSAFE:
+            o = DocEntry('-u', '--unsafe')
+            o.set_groff('Allow addresses that do not pass checksum.')
+            self.docs['u'] = o
+
+        
+        if self.arg_flags & Flag.SEQ:
+            o = DocEntry('--seq')
+            o.set_groff('Use numeric sequencial jsonrpc query ids. Useful for buggy server implementations who expects such.')
+            self.docs['seq'] = o
+
+
+        if self.arg_flags & Flag.KEY_FILE:
+            o = DocEntry('-y', '--key-path')
+            o.set_groff('Path to signing key.')
+            o.set_groff_argvalue('path')
+            self.docs['y'] = o
+            self.envs['y'] = 'WALLET_KEY_FILE'
+
+            o = DocEntry('--passphrase-file')
+            o.set_groff('Path to file containing password to unlock key file')
+            o.set_groff_argvalue('path')
+            self.docs['passphrasefile'] = o
+
+
+        if self.arg_flags & Flag.SEND:
+            o = DocEntry('-s')
+            o.set_groff('Send to network. If set, network state may change. This means tokens may be spent and so on. Use with care. Only applies to transactions.')
+            self.docs['s'] = o
+
+
+        if self.arg_flags & Flag.RAW:
+            o = DocEntry('--raw')
+            o.set_groff('Produce output most optimized for machines.')
+            self.docs['raw'] = o
+
+        if self.arg_flags & (Flag.SIGN | Flag.NONCE):
+            o = DocEntry('--nonce')
+            o.set_groff('Explicitly set nonce to use for transaction.')
+            self.docs['nonce'] = o
+
+
+        if self.arg_flags & (Flag.SIGN | Flag.FEE):
+            o = DocEntry('--fee-price')
+            o.set_groff('Set fee unit price to offer for the transaction. If used with \\fB-s\\fP this may incur actual network token cost.')
+            self.docs['feeprice'] = o
+
+            o = DocEntry('--fee-limit')
+            o.set_groff('Set the limit of execution units for the transaction. If used with \\fB-s\\fP this may incur actual network token cost. If \\fB--fee-price\\fP is not explicitly set, the price \\fImay\\fP be retrieved from the network, and multiplied with this value to define the cost.')
+            self.docs['feelimit'] = o
+
+
+#        # TODO: this manipulation should be DRYd
+#        if self.arg_flags & argflag_std_target == 0:
+#            self.arg_flags |= Flag.WALLET
+
+
+        if self.arg_flags & Flag.EXEC:
+            o = DocEntry('-e', '--executable-address')
+            o.set_groff('Address of an executable code point on the network.')
+            self.docs['e'] = o
+
+        
+        if self.arg_flags & Flag.WALLET:
+            o = DocEntry('-a', '--recipient-address')
+            o.set_groff('Network wallet address to operate on. For read calls, this will be the wallet address for which the query is anchored. For transaction calls, it will be the wallet address for which state will be changed.')
+            self.docs['a'] = o
+
+
+    def process(self):
+        self.process_arg()
+#        self.process_env()
+
+
+class EnvDocGenerator:
+
+    def __init__(self, arg_flags, override=None):
+        self.arg_flags = arg_flags
+        self.envs = {}
+        env_dir = os.path.join(data_dir, 'env')
+        self.config = confini.Config(env_dir, override_dirs=override)
+        self.config.process()
+
+
+    def __add(self, k):
+        #v = format_groff(k, self.config.get(k), None, typ='env')
+        v = apply_groff([k], self.config.get(k), None, typ='env')
+        self.envs[k] = v
+
+
+    def process(self):
+        ks = []
+        if self.arg_flags & Flag.PROVIDER:
+            ks += [
+                    'RPC_PROVIDER',
+                    'RPC_DIALECT',
+                    ]
+            if self.arg_flags & Flag.RPC_AUTH:
+                ks += [
+                        'RPC_AUTH',
+                        'RPC_CREDENTIALS',
+                        ]
+
+        if self.arg_flags & Flag.CHAIN_SPEC:
+            ks.append('CHAIN_SPEC')
+
+        if self.arg_flags & Flag.KEY_FILE:
+            ks += [
+                'WALLET_KEY_FILE',
+                'WALLET_PASSPHRASE',
+                    ]
+
+        for k in ks:
+            self.__add(k)
+
+
+    def __len__(self):
+        return len(self.envs)
+
+
+    def __str__(self):
+        s = ''
+        ks = list(self.envs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.envs[k]) + "\n"  
+        return s

chainlib/cli/rpc.py

@@ -61,7 +61,7 @@ class Rpc:
             self.id_generator = IntSequenceGenerator()
 
         self.chain_spec = config.get('CHAIN_SPEC')
-        self.conn = self.constructor(url=config.get('RPC_PROVIDER'), chain_spec=self.chain_spec, auth=auth)
+        self.conn = self.constructor(url=config.get('RPC_PROVIDER'), chain_spec=self.chain_spec, auth=auth, verify_identity=config.true('RPC_VERIFY'))
 
         return self.conn
 

chainlib/cli/wallet.py

@@ -1,9 +1,6 @@
 # standard imports
 import logging
 
-# external imports
-from crypto_dev_signer.keystore.dict import DictKeystore
-
 logg = logging.getLogger(__name__)
 
 
@@ -19,7 +16,7 @@ class Wallet:
     :todo: sign_transaction_to_rlp from chainlib-eth must be renamed to sign_transaction_to_wire, and included as part of signer interface
     """
     
-    def __init__(self, signer_cls, keystore=DictKeystore(), checksummer=None):
+    def __init__(self, signer_cls, keystore=None, checksummer=None):
         self.signer_constructor = signer_cls
         self.keystore = keystore
         self.signer = None
@@ -30,6 +27,10 @@ class Wallet:
         self.use_checksum = False
 
 
+    def init(self):
+        self.signer = self.signer_constructor(self.keystore)
+
+
     def from_config(self, config):
         """Instantiates a signer from the registered signer class, using parameters from a processed configuration.
 

chainlib/connection.py

@@ -23,7 +23,10 @@ from .jsonrpc import (
         ErrorParser,
         )
 from .http import PreemptiveBasicAuthHandler
-from .error import JSONRPCException
+from .error import (
+        JSONRPCException,
+        RPCException,
+        )
 from .auth import Auth
 
 logg = logging.getLogger(__name__)
@@ -99,10 +102,14 @@ class RPCConnection:
         }
     __constructors_for_chains = {}
 
-    def __init__(self, url=None, chain_spec=None, auth=None):
+    def __init__(self, url=None, chain_spec=None, auth=None, verify_identity=True, timeout=1.0):
+        self.timeout = timeout
         self.chain_spec = chain_spec
         self.location = None
         self.basic = None
+        self.verify_identity = verify_identity
+        if not self.verify_identity:
+            logg.warning('RPC host identity verification is OFF. Beware, you will be easy to cheat')
         if url == None:
             return
         self.auth = auth
@@ -127,6 +134,9 @@ class RPCConnection:
         self.location = os.path.join('{}://'.format(url_parsed.scheme), location)
         self.location = urljoin(self.location, url_parsed.path)
 
+        if url_parsed.query != '':
+            self.location = urljoin(self.location, '?' + url_parsed.query)
+
         logg.debug('parsed url {} to location {}'.format(url, self.location))
 
 
@@ -284,6 +294,11 @@ class JSONRPCHTTPConnection(HTTPConnection):
         :returns: Result value part of JSON RPC response
         :todo: Invalid response exception from invalid json response
         """
+        ssl_ctx = None
+        if not self.verify_identity:
+            import ssl
+            ssl_ctx = ssl.SSLContext()
+            ssl_ctx.verify_mode = ssl.CERT_NONE
         req = Request(
                 self.location,
                 method='POST',
@@ -309,10 +324,20 @@ class JSONRPCHTTPConnection(HTTPConnection):
             ho = build_opener(handler)
             install_opener(ho)
        
-        r = urlopen(req, data=data.encode('utf-8'))
+        r = None
+        try: 
+            r = urlopen(
+                req,
+                data=data.encode('utf-8'),
+                context=ssl_ctx,
+                timeout=self.timeout,
+                )
+        except URLError as e:
+            raise RPCException(e)
 
-        result = json.load(r)
-        logg.debug('(HTTP) recv {}'.format(result))
+        resp = r.read()
+        logg.debug('(HTTP) recv {}'.format(resp.decode('utf-8')))
+        result = json.loads(resp)
         if o['id'] != result['id']:
             raise ValueError('RPC id mismatch; sent {} received {}'.format(o['id'], result['id']))
         return jsonrpc_result(result, error_parser)

chainlib/data/config/config.ini

@@ -4,6 +4,7 @@ auth =
 credentials =
 dialect = default
 scheme = http
+verify = 1
 
 [chain]
 spec = 

chainlib/data/env/env.ini

@@ -0,0 +1,14 @@
+[rpc]
+provider = Fully-qualified URL to the RPC endpoint of the blockchain node.
+auth = Authentication method to use for the \fIRPC_PROVIDER\fP. Currently only \fIbasic\fP is supported.
+credentials = Authentication credentials to use for \fIRPC_AUTH\fP. For \fIbasic\fP authentication the value must be given as \fI<user>:<pass>\fP.
+dialect = Enables translations of EVM node specific formatting and response codes.
+scheme = (needs content)
+verify = (needs content)
+
+[chain]
+spec = String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+[wallet]
+key_file = The wallet key file containing private key to use for transaction signing. Overridden by \fB-y\fP. 
+passphrase = Passphrase to unlock wallet. \fBWARNING:\fP it is \fBunsafe\fP to pass the passphrase as an environment variable. If the key unlocks something of value, the passphrase should rather be in a configuration file, preferably as an encrypted entry. Alternatively, a passphrase can be read from file using the \fB--passphrase-file\fP option. Files containing passphrases should only be accessible by the owner.

chainlib/encode.py

@@ -30,9 +30,7 @@ class TxHexNormalizer:
 
 
     def __hex_normalize(self, data, context):
-        #r = add_0x(hex_uniform(strip_0x(data)))
         r = hex_uniform(strip_0x(data))
-        logg.debug('normalize {} {} -> {}'.format(context, data, r))
         return r
 
 

chainlib/settings.py

@@ -0,0 +1,26 @@
+# local imports
+from .chain import ChainSpec
+
+
+class ChainSettings:
+
+    def __init__(self, include_sync=False, include_queue=False):
+        self.o = {}
+        self.get = self.o.get
+
+
+    def process_common(self, config):
+        self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+
+    def process(self, config):
+        self.process_common(config)
+
+
+    def __str__(self):
+        ks = list(self.o.keys())
+        ks.sort()
+        s = ''
+        for k in ks:
+            s += '{}: {}\n'.format(k, self.o.get(k))
+        return s

chainlib/tx.py

@@ -8,7 +8,7 @@ class Tx:
     """
 
     def __init__(self, src, block=None):
-        self.txs = []
         self.src = src
         self.block = block
         self.block_src = None
+        self.index = None

doc/texinfo/cli.texi

@@ -1,5 +1,3 @@
-@node chainlib-cli
-
 @section Command line interface provisions
 
 The base CLI provisions of @code{chainlib} simplifies the generation of a some base object instances by command line arguments, environment variables and configuration schemas.

doc/texinfo/code.texi

@@ -1,5 +1,3 @@
-@node chainlib-lib
-
 @section Base library contents
 
 

doc/texinfo/config.texi

@@ -1,5 +1,4 @@
-@node chainlib-config
-
+@anchor{chainlib-config}
 @section Rendering configurations
 
 Configurations in @code{chainlib} are processed, rendered and interfaced using the @code{confini} python package.

doc/texinfo/content.texi

@@ -1,5 +1,4 @@
-@top chainlib
-
+@node chainlib
 @chapter Chainlib
 
 @include intro.texi

doc/texinfo/intro.texi

@@ -1,4 +1,4 @@
-@node chainlib-intro
+@section Overview
 
 Chainlib is an attempt at employing a universal interface to manipulate and access blockchains regardless of underlying architecture.
 

requirements.txt

@@ -1,3 +1,4 @@
-crypto-dev-signer>=0.4.15rc2,<=0.4.15
+funga~=0.5.2
 pysha3==1.0.2
-hexathon~=0.0.1a8
+hexathon~=0.1.5
+confini~=0.6.0

scripts/chainlib-man.py

@@ -0,0 +1,232 @@
+#!/usr/bin/python3
+
+import logging
+import os
+import sys
+import argparse
+import tempfile
+import shutil
+
+from hexathon import strip_0x, add_0x
+
+from chainlib.cli.man import (
+        EnvDocGenerator,
+        DocGenerator,
+        apply_groff,
+        )
+from chainlib.cli.base import (
+        argflag_std_base,
+        flag_names,
+        )
+from chainlib.cli.arg import ArgumentParser as ChainlibArgumentParser
+from chainlib.cli.config import Config
+        
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+configuration_description = """
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \\fB-c\\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \\fB--dumpconfig\\fP \\fIformat\\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \\fIsection\\fP and \\fIkey\\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \\fIFOO_BAZ_BAR\\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \\fBENVIRONMENT\\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \\fB-n\\fP and \\fB--env-prefix\\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \\fBOPTIONS\\fP section.
+
+"""
+
+seealso_description = """
+.SH SEE ALSO
+
+.BP
+confini-dump(1), eth-keyfile(1)
+
+"""
+
+legal_description = """
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+"""
+
+source_description = """
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+
+"""
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-b', default=add_0x(hex(argflag_std_base)), help='argument flag bitmask')
+argparser.add_argument('-c', help='config override directory')
+argparser.add_argument('-n', required=True, help='tool name to use for man filename')
+argparser.add_argument('-d', default='.', help='output directory')
+argparser.add_argument('-v', action='store_true', help='turn on debug logging')
+#argparser.add_argument('--overrides-dir', dest='overrides_dir', help='load options description override from file')
+argparser.add_argument('--overrides-env-dir', dest='overrides_env_dir', help='load envionment description override config from directory')
+argparser.add_argument('--overrides-config-file', dest='overrides_config_file', help='load configuration text from file')
+argparser.add_argument('source_dir', help='directory containing sources for the tool man page')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v:
+    logg.setLevel(logging.DEBUG)
+            
+b = bytes.fromhex(strip_0x(args.b))
+flags = int.from_bytes(b, byteorder='big')
+
+flags_debug= flag_names(flags)
+logg.debug('apply arg flags {}: {}'.format(flags, ', '.join(flags_debug)))
+
+g = DocGenerator(flags)
+
+toolname = args.n
+g.process()
+
+def apply_override(g, override_dir):
+    #if args.overrides_dir != None:
+    overrides_file = os.path.join(override_dir, toolname + '.overrides')
+    override = True
+    f = None
+    try:
+        f = open(overrides_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no overrides found for {}'.format(toolname))
+        override = False
+
+    if override:
+        while True:
+            s = f.readline()
+            if len(s) == 0:
+                break
+            v = s.split('\t', maxsplit=4)
+            fargs = None
+            try:
+                fargs = v[2].rstrip().split(',')
+            except IndexError:
+                fargs = []
+            argvalue = None
+            if len(v) == 4:
+                argvalue = v[3]
+            try:
+                g.override_arg(v[0], v[1], fargs, argvalue=argvalue)
+            except KeyError:
+                logg.info('adding not previously registered key {} flags: {}'.format(v[0], ','.join(fargs)))
+                g.set_arg(v[0], v[1], fargs, argvalue=argvalue)
+        f.close()
+    return g
+
+
+def get_head(tool_name, source_dir):
+    header_file = os.path.join(source_dir, tool_name + '.head.groff') 
+    f = open(header_file, 'r')
+    head = f.read()
+    f.close()
+    return head
+
+
+def get_examples(tool_name, source_dir):
+    example_file = os.path.join(source_dir, tool_name + '.examples.groff')
+    f = None
+    try:
+        f = open(example_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no examples file found for {}'.format(tool_name))
+        return None
+    logg.info('examples file {} found for {}'.format(example_file, tool_name))
+    examples = f.read()
+    f.close()
+    return examples
+
+
+def get_custom(tool_name, source_dir):
+    custom_file = os.path.join(source_dir, tool_name + '.custom.groff')
+    f = None
+    try:
+        f = open(custom_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no custom file found for {}'.format(tool_name))
+        return None
+    logg.info('custom file {} found for {}'.format(custom_file, tool_name))
+    custom = f.read()
+    f.close()
+    return custom
+
+
+def get_seealso(tool_name, source_dir):
+    seealso_file = os.path.join(source_dir, tool_name + '.seealso.groff')
+    f = None
+    try:
+        f = open(seealso_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no seealso file found for {}'.format(tool_name))
+        return None
+    logg.info('seealso file {} found for {}'.format(seealso_file, tool_name))
+    seealso = f.read()
+    f.close()
+    return seealso
+
+
+g = apply_override(g, args.source_dir)
+
+ge = EnvDocGenerator(flags, override=args.overrides_env_dir)
+ge.process()
+
+head = get_head(toolname, args.source_dir)
+examples = get_examples(toolname, args.source_dir)
+custom = get_custom(toolname, args.source_dir)
+seealso = get_seealso(toolname, args.source_dir)
+
+if args.overrides_config_file != None:
+    f = open(args.overrides_config_file, 'r')
+    configuration_description = f.read()
+    f.close()
+
+(fd, fp) = tempfile.mkstemp()
+f = os.fdopen(fd, 'w')
+f.write(head)
+f.write(str(g))
+f.write(configuration_description)
+
+if custom != None:
+    f.write(custom)
+
+if examples != None:
+    f.write(".SH EXAMPLES\n\n")
+    f.write(examples)
+
+if seealso != None:
+    seealso_description = seealso
+
+if len(ge) > 0:
+    f.write(".SH ENVIRONMENT\n\n")
+    f.write(str(ge))
+
+f.write(legal_description)
+f.write(source_description)
+f.write(seealso_description)
+f.close()
+
+dest = os.path.join(args.d, toolname + '.1')
+shutil.copyfile(fp, dest)
+
+os.unlink(fp)

setup.cfg

@@ -1,10 +1,13 @@
 [metadata]
-name = chainlib
-version = 0.0.10a1
-description = Generic blockchain access library and tooling
-author = Louis Holbrook
-author_email = dev@holbrook.no
-url = https://gitlab.com/chaintools/chainlib
+name=chainlib
+license=WTFPL2
+author_email=dev@holbrook.no
+description=Generic blockchain access library and tooling
+version=0.1.1
+url=https://gitlab.com/chaintools/chainlib
+author=Louis Holbrook
+
+
 keywords =
 	dlt
 	blockchain
@@ -13,18 +16,8 @@ classifiers =
 	Programming Language :: Python :: 3
 	Operating System :: OS Independent
 	Development Status :: 3 - Alpha
+	Topic :: Software Development :: Libraries
 	Environment :: Console
 	Intended Audience :: Developers
 	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 	Topic :: Internet
-license = GPL3
-licence_files =
-	LICENSE.txt
-
-
-[options]
-python_requires = >= 3.6
-include_package_data = True
-packages = 
-	chainlib
-	chainlib.cli

setup.py

@@ -16,5 +16,15 @@ setup(
         install_requires=requirements,
         extras_require={
              'xdg': "pyxdg~=0.27",
-             }
+             },
+        license_files= ('LICENSE.txt',),
+        python_requires = '>=3.8',
+        include_package_data = True,
+        packages = [
+            'chainlib',
+            'chainlib.cli',
+            ],
+        scripts = [
+            'scripts/chainlib-man.py',
+            ],
     )

tests/test_chain.py

@@ -1,40 +1,81 @@
+# standard imports
 import unittest
+import logging
 
+# local imports
 from chainlib.chain import ChainSpec
 
+# test imports
 from tests.base import TestBase
 
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+logg.setLevel(logging.DEBUG)
+
 
 class TestChain(TestBase):
 
     def test_chain_spec_str(self):
-        s = ChainSpec('foo', 'bar', 3, 'baz')
-        self.assertEqual('foo:bar:3:baz', str(s))
-
         s = ChainSpec('foo', 'bar', 3)
         self.assertEqual('foo:bar:3', str(s))
 
-    def test_chain_spec(self):
+        s = ChainSpec('foo', 'bar', 3, 'baz')
+        self.assertEqual('foo:bar:3:baz', str(s))
 
+
+        s = ChainSpec('foo', 'bar', 3, 'baz', ['inky', 'pinky', 'blinky'])
+        self.assertEqual('foo:bar:3:baz:inky:pinky:blinky', str(s))
+
+
+    def test_chain_spec(self):
         s = ChainSpec.from_chain_str('foo:bar:3')
         s = ChainSpec.from_chain_str('foo:bar:3:baz')
+        s = ChainSpec.from_chain_str('foo:bar:3:baz:inky:pinky:blinky')
 
         with self.assertRaises(ValueError):
             s = ChainSpec.from_chain_str('foo:bar:a')
             s = ChainSpec.from_chain_str('foo:bar')
             s = ChainSpec.from_chain_str('foo')
+            s = ChainSpec.from_chain_str('foo1:bar:3')
+            s = ChainSpec.from_chain_str('foo:bar2:3')
 
 
     def test_chain_spec_dict(self):
-        s = 'foo:bar:3:baz'
-        c = ChainSpec.from_chain_str('foo:bar:3:baz')
+        ss = 'foo:bar:3:baz:inky:pinky:blinky'
+        c = ChainSpec.from_chain_str(ss)
         d = c.asdict()
         self.assertEqual(d['arch'], 'foo')
         self.assertEqual(d['fork'], 'bar')
         self.assertEqual(d['network_id'], 3)
         self.assertEqual(d['common_name'], 'baz')
+        self.assertEqual(d['custom'], ['inky', 'pinky', 'blinky'])
         cc = ChainSpec.from_dict(d)
-        self.assertEqual(s, str(cc))
+        self.assertEqual(ss, str(cc))
+
+        d = c.asdict(use_custom=False)
+        cc = ChainSpec.from_dict(d)
+        self.assertEqual(str(cc), 'foo:bar:3:baz')
+
+        d = c.asdict(use_common_name=False)
+        cc = ChainSpec.from_dict(d)
+        self.assertEqual(str(cc), 'foo:bar:3')
+
+    def test_chain_spec_compare(self):
+        a = 'foo:bar:42:baz'
+        b = 'foo:bar:42:barbar'
+        c = 'foo:bar:42:baz:inky:pinky:blinky'
+
+        ca = ChainSpec.from_chain_str(a)
+        cb = ChainSpec.from_chain_str(b)
+
+        self.assertTrue(ca.is_same_as(cb))
+        self.assertFalse(ca.is_same_as(cb, use_common_name=True))
+
+        cc = ChainSpec.from_chain_str(c)
+        logg.debug('chain_spec_cmp ' + str(cc.o))
+        self.assertTrue(ca.is_same_as(cc))
+        self.assertTrue(ca.is_same_as(cc, use_common_name=True))
+        self.assertFalse(ca.is_same_as(cc, use_common_name=True, use_custom=True))
 
 
 

tests/test_cli.py

@@ -1,6 +1,7 @@
 # standard imports
 import unittest
 import os
+import logging
 
 # local imports
 import chainlib.cli
@@ -10,6 +11,8 @@ script_dir = os.path.dirname(os.path.realpath(__file__))
 data_dir = os.path.join(script_dir, 'testdata')
 config_dir = os.path.join(data_dir, 'config')
 
+logging.basicConfig(level=logging.DEBUG)
+
 
 class TestCli(unittest.TestCase):