Bump vesrion

Reviewed-on: #2

CHANGELOG

@@ -1,16 +1,36 @@
+- 0.0.23
+	* Configuration variable descriptions
+	* Arg flags to names listing method
+	* New flags preset for reads without wallet
+	* Remove pesistent wallet arg flag bug
+	* Rename arg flag reset to flag_reset
+- 0.0.22
+	* Man page generator script
+- 0.0.21
+	* Log rpc reply before json parse
+- 0.0.20
+	* Add edit option for some long flag aliases of short flags: -a, -e, -s and -y
+- 0.0.19
+	* Passphrase file option to unlock keyfile for CLI tooling
+- 0.0.18
+	* Upgrade hexathon skipping buggy compact hex method
+- 0.0.17
+	* Add loglevel environment variable
+- 0.0.16
+	* Allow query string in query url
 - 0.0.14
 	* Add option to skip ssl verification on rpc
 - 0.0.5
 	* Move eth code to separate package
-- 0.0.4-unreleased
+- 0.0.4
 	* Add pack tx from already signed tx struct
 	* Add http auth handling for jsonrpc connections
 	* Add customizable jsonrpc id generator (to allow for buggy server id handling)
-- 0.0.3-unreleased
+- 0.0.3
 	* Remove erc20 module (to new external package)
-- 0.0.2-unreleased
+- 0.0.2
 	* 
-- 0.0.1-unreleased
+- 0.0.1
 	* Add eth tx decode
 	* Add eth balance query with erc20 option
 	* Add eth checksum address

MANIFEST.in

@@ -1 +1 @@
-include *requirements.txt LICENSE chainlib/data/config/*
+include *requirements.txt LICENSE chainlib/data/config/* chainlib/data/env/*

chainlib/cli/__init__.py

@@ -3,7 +3,9 @@ from .base import (
         argflag_std_read,
         argflag_std_write,
         argflag_std_base,
-        reset,
+        argflag_std_base_read,
+        flag_reset,
+        flag_set,
         )
 from .arg import ArgumentParser
 from .config import Config

chainlib/cli/arg.py

@@ -5,6 +5,7 @@ import enum
 import os
 import select
 import sys
+import re
 
 # local imports
 from .base import (
@@ -30,16 +31,35 @@ def stdin_arg():
         return v.rstrip()
     return None
 
+_default_long_args = {
+        '-a': '--recipient',
+        '-e': '--executable-address',
+        '-s': '--send',
+        '-y': '--key-file',
+                }
+
+_default_dest = {
+        '-a': 'recipient',
+        '-e': 'executable_address',
+        }
+
+
+_default_fmt = 'human'
+
 
 class ArgumentParser(argparse.ArgumentParser):
     """Extends the standard library argument parser to construct arguments based on configuration flags.
 
     The extended class is set up to facilitate piping of single positional arguments via stdin. For this reason, positional arguments should be added using the locally defined add_positional method instead of add_argument.
 
+    Long flag aliases for short flags are editable using the arg_long argument. Editing a non-existent short flag will produce no error and have no effect. Adding a long flag for a short flag that does not have an alias will also not have effect.
+
     Calls chainlib.cli.args.ArgumentParser.process_flags with arg_flags and env arguments, see the method's documentation for further details.
 
     :param arg_flags: Argument flag bit vector to generate configuration values for.
     :type arg_flags: chainlib.cli.Flag
+    :param arg_long: Change long flag alias for given short flags. Example value: {'-a': '--addr', '-e': '--contract'}
+    :type arg_long: dict
     :param env: Environment variables
     :type env: dict
     :param usage: Usage string, passed to parent
@@ -50,11 +70,24 @@ class ArgumentParser(argparse.ArgumentParser):
     :type epilog: str
     """
 
-    def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs):
+    def __init__(self, arg_flags=0x0f, arg_long={}, env=os.environ, usage=None, description=None, epilog=None, default_format=_default_fmt, *args, **kwargs):
         super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter, *args, **kwargs)
     
-        self.process_flags(arg_flags, env)
         self.pos_args = []
+        self.long_args = _default_long_args
+        self.arg_dest = _default_dest
+        self.default_format = default_format
+
+        re_long = r'^--[a-z\-]+$'
+        for k in arg_long.keys():
+            if re.match(re_long, arg_long[k]) == None:
+                raise ValueError('invalid long arg {}'.format(arg_long[k]))
+            self.long_args[k] = arg_long[k]
+            dest = arg_long[k][2:]
+            dest = dest.replace('-', '_')
+            self.arg_dest[k] = dest
+
+        self.process_flags(arg_flags, env)
 
 
     def add_positional(self, name, type=str, help=None, append=False, required=True):
@@ -96,7 +129,6 @@ class ArgumentParser(argparse.ArgumentParser):
             for arg in self.pos_args:
                 if arg[3]:
                     if arg[4]:
-                        logg.debug('argumen')
                         self.add_argument(arg[0], nargs='+', type=arg[1], help=arg[2])
                     else:
                         self.add_argument(arg[0], type=arg[1], help=arg[2])
@@ -139,6 +171,7 @@ class ArgumentParser(argparse.ArgumentParser):
         :type env: dict
         """
         if arg_flags & Flag.VERBOSE:
+            self.add_argument('--no-logs', dest='no_logs',action='store_true', help='Turn off all logging')
             self.add_argument('-v', action='store_true', help='Be verbose')
             self.add_argument('-vv', action='store_true', help='Be more verbose')
         if arg_flags & Flag.CONFIG:
@@ -153,7 +186,8 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & Flag.PROVIDER:
             self.add_argument('-p', '--rpc-provider', dest='p', type=str, help='RPC HTTP(S) provider url')
             self.add_argument('--rpc-dialect', dest='rpc_dialect', type=str, help='RPC HTTP(S) backend dialect')
-            self.add_argument('--height', default='latest', help='Block height to execute against')
+            if arg_flags & Flag.NO_TARGET == 0:
+                self.add_argument('--height', default='latest', help='Block height to execute against')
             if arg_flags & Flag.RPC_AUTH:
                 self.add_argument('--rpc-auth', dest='rpc_auth', type=str, help='RPC autentication scheme')
                 self.add_argument('--rpc-credentials', dest='rpc_credentials', type=str, help='RPC autentication credential values')
@@ -164,9 +198,10 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & Flag.SEQ:
             self.add_argument('--seq', action='store_true', help='Use sequential rpc ids')
         if arg_flags & Flag.KEY_FILE:
-            self.add_argument('-y', '--key-file', dest='y', type=str, help='Keystore file to use for signing or address')
+            self.add_argument('-y', self.long_args['-y'], dest='y', type=str, help='Keystore file to use for signing or address')
+            self.add_argument('--passphrase-file', dest='passphrase_file', type=str, help='File containing passphrase for keystore')
         if arg_flags & Flag.SEND:
-            self.add_argument('-s', '--send', dest='s', action='store_true', help='Send to network')
+            self.add_argument('-s', self.long_args['-s'], dest='s', action='store_true', help='Send to network')
         if arg_flags & Flag.RAW:
             self.add_argument('--raw', action='store_true', help='Do not decode output')
         if arg_flags & (Flag.SIGN | Flag.NONCE):
@@ -174,9 +209,19 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & (Flag.SIGN | Flag.FEE):
             self.add_argument('--fee-price', dest='fee_price', type=int, help='override fee price')
             self.add_argument('--fee-limit', dest='fee_limit', type=int, help='override fee limit')
-        if arg_flags & argflag_std_target == 0:
-            arg_flags |= Flag.WALLET
+        # wtf?
+        #if arg_flags & argflag_std_target == 0:
+        #    arg_flags |= Flag.WALLET
         if arg_flags & Flag.EXEC:
-            self.add_argument('-e', '--exectuable-address', dest='executable_address', type=str, help='contract address')
+            self.add_argument('-e', self.long_args['-e'], dest=self.arg_dest['-e'], type=str, help='contract address')
         if arg_flags & Flag.WALLET:
-            self.add_argument('-a', '--recipient', dest='recipient', type=str, help='recipient address')
+            self.add_argument('-a', self.long_args['-a'], dest=self.arg_dest['-a'], type=str, help='recipient address')
+        if arg_flags & (Flag.FMT_HUMAN | Flag.FMT_WIRE | Flag.FMT_RPC):
+            format_choices = []
+            if arg_flags & Flag.FMT_HUMAN:
+                format_choices.append('human')
+            if arg_flags & Flag.FMT_WIRE:
+                format_choices.append('bin')
+            if arg_flags & Flag.FMT_RPC:
+                format_choices.append('rpc')
+            self.add_argument('-f', '--format', type=str, choices=format_choices, help='output formatting (default: {})'.format(self.default_format))

chainlib/cli/base.py

@@ -33,14 +33,40 @@ class Flag(enum.IntEnum):
     SEND = 262144
     # rpc extras - nibble 6
     RPC_AUTH = 1048576
+    # formatting - nibble 7
+    FMT_HUMAN = 16777216
+    FMT_WIRE = 33554432
+    FMT_RPC = 67108864
+    # upper bound
+    MAX = 1048576
 
-argflag_std_read = 0x23ff 
-argflag_std_write = 0xff31ff
-argflag_std_base = 0x200f
-argflag_std_target = 0x00e000
-argflag_all = 0xffffff
+argflag_std_read = 0x000023ff
+argflag_std_write = 0x001731ff
+argflag_std_base = 0x0000200f
+argflag_std_base_read = 0x000000bf
+argflag_std_target = 0x0000e000
+argflag_all = 0x0317f7ff
 
-def reset(flags, v):
+
+def flag_reset(flags, v):
     mask = ~(argflag_all & v)
     r = flags & mask
     return r
+
+
+def flag_set(flags, v):
+    return flags | v
+
+
+def flag_names(flags):
+    flags_debug = []
+    i = Flag.MAX
+    while True:
+        if flags & i > 0:
+            v = Flag(i)
+            flags_debug.append(v.name)
+        i >>= 1
+        if i == 0:
+            break
+    flags_debug.reverse()
+    return flags_debug

chainlib/cli/config.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import sys
+import stat
 
 # external imports
 import confini
@@ -45,7 +46,7 @@ class Config(confini.Config):
 
 
     @classmethod
-    def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback, dump_writer=sys.stdout):
+    def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=None, dump_writer=sys.stdout):
         """Parses arguments in argparse.ArgumentParser instance, then match and override configuration values that match them.
 
         The method processes all known argument flags from chainlib.cli.Flag passed in the "args" argument. 
@@ -102,14 +103,51 @@ class Config(confini.Config):
         :rtype: confini.Config
         :return: Processed configuation
         """
+        env_prefix = getattr(args, 'env_prefix', None)
+        env_prefix_str = env_prefix
+        if env_prefix_str == None:
+            env_prefix_str = ''
+        else:
+            env_prefix_str += '_'
+
+        env_loglevel_key_str = env_prefix_str + 'LOGLEVEL'
+        env_loglevel = os.environ.get(env_loglevel_key_str)
+
         if logger == None:
             logger = logging.getLogger()
 
-        if arg_flags & Flag.CONFIG:
+        if env_loglevel != None:
+            env_loglevel = env_loglevel.lower()
+            if env_loglevel == '0' or env_loglevel == 'no' or env_loglevel == 'none' or env_loglevel == 'disable' or env_loglevel == 'disabled' or env_loglevel == 'off':
+                logging.disable()
+            elif env_loglevel == '1' or env_loglevel == 'err' or env_loglevel == 'error':
+                logger.setLevel(logging.ERROR)
+            elif env_loglevel == '2' or env_loglevel == 'warning' or env_loglevel == 'warn':
+                logger.setLevel(logging.WARNING)
+            elif env_loglevel == '3' or env_loglevel == 'info':
+                logger.setLevel(logging.INFO)
+            else:
+                valid_level = False
+                try:
+                    num_loglevel = int(env_loglevel)
+                    valid_level = True
+                except:
+                    if env_loglevel == 'debug':
+                        valid_level = True
+
+                if not valid_level:
+                    raise ValueError('unknown loglevel {} set in environment variable {}'.format(env_loglevel, env_loglevel_key_str))
+
+                logger.setLevel(logging.DEBUG)
+
+
+        if arg_flags & Flag.VERBOSE:
             if args.vv:
                 logger.setLevel(logging.DEBUG)
             elif args.v:
                 logger.setLevel(logging.INFO)
+            if args.no_logs:
+                logging.disable()
    
         override_config_dirs = []
         config_dir = [cls.default_base_config_dir]
@@ -160,7 +198,6 @@ class Config(confini.Config):
         #        default_config_dir = default_parent_config_dir
         #    config_dir = default_config_dir
         #    override_config_dirs = []
-        env_prefix = getattr(args, 'env_prefix', None)
 
         config = confini.Config(config_dir, env_prefix=env_prefix, override_dirs=override_config_dirs)
         config.process()
@@ -176,10 +213,18 @@ class Config(confini.Config):
             args_override['CHAIN_SPEC'] = getattr(args, 'i')
         if arg_flags & Flag.KEY_FILE:
             args_override['WALLET_KEY_FILE'] = getattr(args, 'y')
+            fp = getattr(args, 'passphrase_file')
+            if fp != None:
+                st = os.stat(fp)
+                if stat.S_IMODE(st.st_mode) & (stat.S_IRWXO | stat.S_IRWXG) > 0:
+                    logg.warning('others than owner have access on password file')
+                f = open(fp, 'r')
+                args_override['WALLET_PASSPHRASE'] = f.read()
+                f.close()
+            config.censor('PASSPHRASE', 'WALLET')
+        config.dict_override(args_override, 'cli args', allow_empty=True)
 
-        config.dict_override(args_override, 'cli args')
-
-        if arg_flags & Flag.PROVIDER:
+        if arg_flags & (Flag.PROVIDER | Flag.NO_TARGET) == Flag.PROVIDER:
             config.add(getattr(args, 'height'), '_HEIGHT')
         if arg_flags & Flag.UNSAFE:
             config.add(getattr(args, 'u'), '_UNSAFE')

chainlib/cli/man.py

@@ -0,0 +1,357 @@
+# standard imports
+import os
+
+# external imports
+import confini
+
+# local imports
+from .base import (
+        Flag,
+        argflag_std_target,
+        )
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+data_dir = os.path.join(script_dir, '..', 'data')
+
+
+def apply_groff(collection, v, arg=None, typ='arg'):
+    s = ''
+    for flag in collection:
+        if len(s) > 0:
+            s += ', '
+        s += format_groff(flag, v, arg=arg, typ=typ)
+    s = "\n.TP\n" + s + "\n" + v
+    return s
+
+
+def format_groff(k, v, arg=None, typ='arg'):
+    s = ''
+    if typ == 'env':
+        s += '\\fI'
+    else:
+        s += '\\fB'
+    s += k
+    if arg != None:
+        s += ' \\fI' + arg
+    s += '\\fP'
+    return s
+
+
+class DocEntry:
+
+    def __init__(self, *args, typ='arg'):
+        self.flags = args
+        self.render = self.get_empty
+        self.groff = None
+        self.typ = typ
+        self.v = None
+
+
+    def __check_line_default(self, m):
+        if self.render == self.get_empty:
+            self.render = m
+
+
+    def get_empty(self):
+        s = ''
+        for flag in self.flags:
+            if len(s) > 0:
+                s += ', '
+            s += flag
+   
+        s += '\n\t(undefined)\n'
+        return s
+
+
+    def set_groff(self, v):
+        self.__check_line_default(self.get_groff)
+        self.groff = v
+
+
+    def set_groff_argvalue(self, argvalue):
+        self.v = '\\fI' + argvalue + '\\fP'
+
+
+    def get_groff(self):
+        v = self.groff
+        if v == None:
+            v = self.plain
+        s = apply_groff(self.flags, v, arg=self.v, typ=self.typ) 
+        return s
+
+
+    def __str__(self):
+        return self.render()
+   
+
+class DocGenerator:
+
+#    def __init__(self, arg_flags, config):
+    def __init__(self, arg_flags):
+        #self.config = config
+        self.arg_flags = arg_flags
+        self.docs = {}
+        self.envs = {}
+#        self.envs = {}
+
+
+    def __str__(self):
+        s = ''
+        ks = list(self.docs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.docs[k]) 
+            env = self.envs.get(k)
+            if env != None:
+                s += ' Overrides the \\fI' + env + '\\fP configuration setting.'
+            s += "\n" 
+        return s
+
+
+    def get_args(self):
+        s = ''
+        ks = list(self.docs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.docs[k]) + "\n" 
+        return s
+
+
+    def set_arg(self, k, v, flags, argvalue=None):
+        o = DocEntry(*flags)
+        o.set_groff_argvalue(argvalue)
+        o.set_groff(v)
+        self.docs[k] = o
+
+
+    def override_arg(self, k, v, args, argvalue=None):
+        o = self.docs[k]
+        #g.docs[v[0]].groff = v[1].rstrip()
+        o.set_groff(v)
+        if argvalue != None:
+            o.set_groff_argvalue(argvalue)
+        l = len(args)
+        if l > 0:
+            o.flags = []
+        for i in range(l):
+            o.flags.append(args[i])
+
+#
+#    def process_env(self):
+#        for k in self.config.all():
+#            if k[0] == '_':
+#                continue
+#            self.envs[k] = None
+
+
+    def process_arg(self):
+
+        if self.arg_flags & Flag.VERBOSE:
+            o = DocEntry('--no-logs')
+            o.set_groff('Turn of logging completely. Negates \\fB-v\\fP and \\fB-vv\\fP')
+            self.docs['nologs'] = o
+
+            o = DocEntry('-v')
+            o.set_groff('Verbose. Show logs for important state changes.')
+            self.docs['v'] = o
+
+            o = DocEntry('-vv')
+            o.set_groff('Very verbose. Show logs with debugging information.')
+            self.docs['vv'] = o
+
+        if self.arg_flags & Flag.CONFIG:
+            o = DocEntry('-c', '--config')
+            o.set_groff('Load configuration files from given directory. All files with an .ini extension will be loaded, of which all must contain valid ini file data.')
+            o.set_groff_argvalue('config_dir')
+            self.docs['c'] = o
+
+            o = DocEntry('-n', '--namespace')
+            o.set_groff('Load given configuration namespace. Configuration will be loaded from the immediate configuration subdirectory with the same name.')
+            o.set_groff_argvalue('namespace')
+            self.docs['n'] = o
+           
+            o = DocEntry('--dumpconfig')
+            o.set_groff('Output configuration settings rendered from environment and inputs. Valid arguments are \\fIini\\fP for ini file output, and \\fIenv\\fP for environment variable output. See \\fBCONFIGURATION\\fP.')
+            o.set_groff_argvalue('format')
+            self.docs['dumpconfig'] = o
+
+
+        if self.arg_flags & Flag.WAIT:
+            o = DocEntry('-w')
+            o.set_groff('Wait for the last transaction to be confirmed on the network. Will generate an error if the EVM execution fails.')
+            self.docs['w'] = o
+
+            o = DocEntry('-ww')
+            o.set_groff('Wait for \\fIall\\fP transactions sequentially to be confirmed on the network. Will generate an error if EVM execution fails for any of the transactions.')
+            self.docs['ww'] = o
+
+
+        if self.arg_flags & Flag.ENV_PREFIX:
+            o = DocEntry('--env-prefix')
+            o.set_groff('Environment prefix for variables to overwrite configuration. Example: If \\fB--env-prefix\\fP is set to \\fBFOO\\fP then configuration variable \\fBBAR_BAZ\\fP would be set by environment variable \\fBFOO_BAZ_BAR\\fP. Also see \\fBENVIRONMENT\\fP.')
+            self.docs['envprefix'] = o
+
+        
+        if self.arg_flags & Flag.PROVIDER:
+            o = DocEntry('-p', '--rpc-provider')
+            o.set_groff('Fully-qualified URL of RPC provider.')
+            self.docs['p'] = o
+            self.envs['p'] = 'RPC_PROVIDER'
+
+            o = DocEntry('--rpc-dialect')
+            o.set_groff('RPC backend dialect. If specified it \\fImay\\fP help with encoding and decoding issues.')
+            self.docs['rpcdialect'] = o
+            self.envs['rpcdialect'] = 'RPC_DIALECT'
+
+            if self.arg_flags & Flag.NO_TARGET == 0:
+                o = DocEntry('--height')
+                o.set_groff('Block height at which to query state for. Does not apply to transactions.')
+                self.docs['height'] = o
+
+            if self.arg_flags & Flag.RPC_AUTH:
+                o = DocEntry('--rpc-auth')
+                o.set_groff('RPC endpoint authentication method, e.g. how to handle a HTTP WWW-Authenticate header.')
+                self.docs['rpcauth'] = o
+                self.envs['rpcauth'] = 'RPC_AUTH'
+
+                o = DocEntry('--rpc-credentials')
+                o.set_groff('RPC endpoint authentication data. Format depends on the authentication method defined in \\fB--rpc-auth\\fP.')
+                self.docs['rpccredentials'] = o
+                self.envs['rpccredentials'] = 'RPC_CREDENTIALS'
+
+
+        if self.arg_flags & Flag.CHAIN_SPEC:
+            o = DocEntry('-i', '--chain-spec')
+            o.set_groff('Chain specification string, in the format <engine>:<fork>:<chain_id>:<common_name>. Example: "evm:london:1:ethereum".')
+            o.set_groff_argvalue('chain_spec')
+            self.docs['i'] = o
+            self.envs['i'] = 'RPC_CREDENTIALS'
+
+
+        if self.arg_flags & Flag.UNSAFE:
+            o = DocEntry('-u', '--unsafe')
+            o.set_groff('Allow addresses that do not pass checksum.')
+            self.docs['u'] = o
+
+        
+        if self.arg_flags & Flag.SEQ:
+            o = DocEntry('--seq')
+            o.set_groff('Use numeric sequencial jsonrpc query ids. Useful for buggy server implementations who expects such.')
+            self.docs['seq'] = o
+
+
+        if self.arg_flags & Flag.KEY_FILE:
+            o = DocEntry('-y', '--key-path')
+            o.set_groff('Path to signing key.')
+            o.set_groff_argvalue('path')
+            self.docs['y'] = o
+            self.envs['y'] = 'WALLET_KEY_FILE'
+
+            o = DocEntry('--passphrase-file')
+            o.set_groff('Path to file containing password to unlock key file')
+            o.set_groff_argvalue('path')
+            self.docs['passphrasefile'] = o
+
+
+        if self.arg_flags & Flag.SEND:
+            o = DocEntry('-s')
+            o.set_groff('Send to network. If set, network state may change. This means tokens may be spent and so on. Use with care. Only applies to transactions.')
+            self.docs['s'] = o
+
+
+        if self.arg_flags & Flag.RAW:
+            o = DocEntry('--raw')
+            o.set_groff('Produce output most optimized for machines.')
+            self.docs['raw'] = o
+
+        if self.arg_flags & (Flag.SIGN | Flag.NONCE):
+            o = DocEntry('--nonce')
+            o.set_groff('Explicitly set nonce to use for transaction.')
+            self.docs['nonce'] = o
+
+
+        if self.arg_flags & (Flag.SIGN | Flag.FEE):
+            o = DocEntry('--fee-price')
+            o.set_groff('Set fee unit price to offer for the transaction. If used with \\fB-s\\fP this may incur actual network token cost.')
+            self.docs['feeprice'] = o
+
+            o = DocEntry('--fee-limit')
+            o.set_groff('Set the limit of execution units for the transaction. If used with \\fB-s\\fP this may incur actual network token cost. If \\fB--fee-price\\fP is not explicitly set, the price \\fImay\\fP be retrieved from the network, and multiplied with this value to define the cost.')
+            self.docs['feelimit'] = o
+
+
+#        # TODO: this manipulation should be DRYd
+#        if self.arg_flags & argflag_std_target == 0:
+#            self.arg_flags |= Flag.WALLET
+
+
+        if self.arg_flags & Flag.EXEC:
+            o = DocEntry('-e', '--executable-address')
+            o.set_groff('Address of an executable code point on the network.')
+            self.docs['e'] = o
+
+        
+        if self.arg_flags & Flag.WALLET:
+            o = DocEntry('-a', '--recipient-address')
+            o.set_groff('Network wallet address to operate on. For read calls, this will be the wallet address for which the query is anchored. For transaction calls, it will be the wallet address for which state will be changed.')
+            self.docs['a'] = o
+
+
+    def process(self):
+        self.process_arg()
+#        self.process_env()
+
+
+class EnvDocGenerator:
+
+    def __init__(self, arg_flags, override=None):
+        self.arg_flags = arg_flags
+        self.envs = {}
+        env_dir = os.path.join(data_dir, 'env')
+        self.config = confini.Config(env_dir, override_dirs=override)
+        self.config.process()
+
+
+    def __add(self, k):
+        #v = format_groff(k, self.config.get(k), None, typ='env')
+        v = apply_groff([k], self.config.get(k), None, typ='env')
+        self.envs[k] = v
+
+
+    def process(self):
+        ks = []
+        if self.arg_flags & Flag.PROVIDER:
+            ks += [
+                    'RPC_PROVIDER',
+                    'RPC_DIALECT',
+                    ]
+            if self.arg_flags & Flag.RPC_AUTH:
+                ks += [
+                        'RPC_AUTH',
+                        'RPC_CREDENTIALS',
+                        ]
+
+        if self.arg_flags & Flag.CHAIN_SPEC:
+            ks.append('CHAIN_SPEC')
+
+        if self.arg_flags & Flag.KEY_FILE:
+            ks += [
+                'WALLET_KEY_FILE',
+                'WALLET_PASSPHRASE',
+                    ]
+
+        for k in ks:
+            self.__add(k)
+
+
+    def __len__(self):
+        return len(self.envs)
+
+
+    def __str__(self):
+        s = ''
+        ks = list(self.envs.keys())
+        ks.sort()
+        for k in ks:
+            s += str(self.envs[k]) + "\n"  
+        return s

chainlib/connection.py

@@ -102,7 +102,8 @@ class RPCConnection:
         }
     __constructors_for_chains = {}
 
-    def __init__(self, url=None, chain_spec=None, auth=None, verify_identity=True):
+    def __init__(self, url=None, chain_spec=None, auth=None, verify_identity=True, timeout=1.0):
+        self.timeout = timeout
         self.chain_spec = chain_spec
         self.location = None
         self.basic = None
@@ -133,6 +134,9 @@ class RPCConnection:
         self.location = os.path.join('{}://'.format(url_parsed.scheme), location)
         self.location = urljoin(self.location, url_parsed.path)
 
+        if url_parsed.query != '':
+            self.location = urljoin(self.location, '?' + url_parsed.query)
+
         logg.debug('parsed url {} to location {}'.format(url, self.location))
 
 
@@ -320,17 +324,20 @@ class JSONRPCHTTPConnection(HTTPConnection):
             ho = build_opener(handler)
             install_opener(ho)
        
+        r = None
         try: 
             r = urlopen(
                 req,
                 data=data.encode('utf-8'),
                 context=ssl_ctx,
+                timeout=self.timeout,
                 )
         except URLError as e:
             raise RPCException(e)
 
-        result = json.load(r)
-        logg.debug('(HTTP) recv {}'.format(result))
+        resp = r.read()
+        logg.debug('(HTTP) recv {}'.format(resp.decode('utf-8')))
+        result = json.loads(resp)
         if o['id'] != result['id']:
             raise ValueError('RPC id mismatch; sent {} received {}'.format(o['id'], result['id']))
         return jsonrpc_result(result, error_parser)

chainlib/data/env/env.ini

@@ -0,0 +1,14 @@
+[rpc]
+provider = Fully-qualified URL to the RPC endpoint of the blockchain node.
+auth = Authentication method to use for the \fIRPC_PROVIDER\fP. Currently only \fIbasic\fP is supported.
+credentials = Authentication credentials to use for \fIRPC_AUTH\fP. For \fIbasic\fP authentication the value must be given as \fI<user>:<pass>\fP.
+dialect = Enables translations of EVM node specific formatting and response codes.
+scheme = (needs content)
+verify = (needs content)
+
+[chain]
+spec = String specifying the type of chain connected to, in the format \fI<engine>:<fork>:<network_id>:<common_name>\fP. For EVM nodes the \fIengine\fP value will always be \fIevm\fP.
+
+[wallet]
+key_file = The wallet key file containing private key to use for transaction signing. Overridden by \fB-y\fP. 
+passphrase = Passphrase to unlock wallet. \fBWARNING:\fP it is \fBunsafe\fP to pass the passphrase as an environment variable. If the key unlocks something of value, the passphrase should rather be in a configuration file, preferably as an encrypted entry. Alternatively, a passphrase can be read from file using the \fB--passphrase-file\fP option. Files containing passphrases should only be accessible by the owner.

chainlib/settings.py

@@ -0,0 +1,26 @@
+# local imports
+from .chain import ChainSpec
+
+
+class ChainSettings:
+
+    def __init__(self, include_sync=False, include_queue=False):
+        self.o = {}
+        self.get = self.o.get
+
+
+    def process_common(self, config):
+        self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+
+    def process(self, config):
+        self.process_common(config)
+
+
+    def __str__(self):
+        ks = list(self.o.keys())
+        ks.sort()
+        s = ''
+        for k in ks:
+            s += '{}: {}\n'.format(k, self.o.get(k))
+        return s

chainlib/tx.py

@@ -8,7 +8,7 @@ class Tx:
     """
 
     def __init__(self, src, block=None):
-        self.txs = []
         self.src = src
         self.block = block
         self.block_src = None
+        self.index = None

requirements.txt

@@ -1,3 +1,4 @@
-funga~=0.5.1
+funga~=0.5.2
 pysha3==1.0.2
-hexathon~=0.1.0
+hexathon~=0.1.5
+confini~=0.6.0

scripts/chainlib-man.py

@@ -0,0 +1,232 @@
+#!/usr/bin/python3
+
+import logging
+import os
+import sys
+import argparse
+import tempfile
+import shutil
+
+from hexathon import strip_0x, add_0x
+
+from chainlib.cli.man import (
+        EnvDocGenerator,
+        DocGenerator,
+        apply_groff,
+        )
+from chainlib.cli.base import (
+        argflag_std_base,
+        flag_names,
+        )
+from chainlib.cli.arg import ArgumentParser as ChainlibArgumentParser
+from chainlib.cli.config import Config
+        
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+configuration_description = """
+.SH CONFIGURATION
+
+All configuration settings may be overriden both by environment variables, or by overriding settings with the contents of ini-files in the directory defined by the \\fB-c\\fP option.
+
+The active configuration, with values assigned from environment and arguments, can be output using the \\fB--dumpconfig\\fP \\fIformat\\fP option. Note that entries having keys prefixed with underscore (e.g. _SEQ) are not actual configuration settings, and thus cannot be overridden with environment variables.
+
+To refer to a configuration setting by environment variables, the \\fIsection\\fP and \\fIkey\\fP are concatenated together with an underscore, and transformed to upper-case. For example, the configuration variable \\fIFOO_BAZ_BAR\\fP refers to an ini-file entry as follows:
+
+.EX
+[foo]
+bar_baz = xyzzy
+.EE
+
+In the \\fBENVIRONMENT\\fP section below, the relevant configuration settings for this tool is listed along with a short description of its meaning.
+
+Some configuration settings may also be overriden by command line options. Also note that the use of the \\fB-n\\fP and \\fB--env-prefix\\fP options affect how environment and configuration is read. The effects of options on how configuration settings are affective is described in the respective \\fBOPTIONS\\fP section.
+
+"""
+
+seealso_description = """
+.SH SEE ALSO
+
+.BP
+confini-dump(1), eth-keyfile(1)
+
+"""
+
+legal_description = """
+.SH LICENSE
+
+This documentation and its source is licensed under the Creative Commons Attribution-Sharealike 4.0 International license.
+
+The source code of the tool this documentation describes is licensed under the GNU General Public License 3.0.
+
+.SH COPYRIGHT
+
+Louis Holbrook <dev@holbrook.no> (https://holbrook.no)
+PGP: 59A844A484AC11253D3A3E9DCDCBD24DD1D0E001
+
+"""
+
+source_description = """
+
+.SH SOURCE CODE
+
+https://git.defalsify.org
+
+"""
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-b', default=add_0x(hex(argflag_std_base)), help='argument flag bitmask')
+argparser.add_argument('-c', help='config override directory')
+argparser.add_argument('-n', required=True, help='tool name to use for man filename')
+argparser.add_argument('-d', default='.', help='output directory')
+argparser.add_argument('-v', action='store_true', help='turn on debug logging')
+#argparser.add_argument('--overrides-dir', dest='overrides_dir', help='load options description override from file')
+argparser.add_argument('--overrides-env-dir', dest='overrides_env_dir', help='load envionment description override config from directory')
+argparser.add_argument('--overrides-config-file', dest='overrides_config_file', help='load configuration text from file')
+argparser.add_argument('source_dir', help='directory containing sources for the tool man page')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v:
+    logg.setLevel(logging.DEBUG)
+            
+b = bytes.fromhex(strip_0x(args.b))
+flags = int.from_bytes(b, byteorder='big')
+
+flags_debug= flag_names(flags)
+logg.debug('apply arg flags {}: {}'.format(flags, ', '.join(flags_debug)))
+
+g = DocGenerator(flags)
+
+toolname = args.n
+g.process()
+
+def apply_override(g, override_dir):
+    #if args.overrides_dir != None:
+    overrides_file = os.path.join(override_dir, toolname + '.overrides')
+    override = True
+    f = None
+    try:
+        f = open(overrides_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no overrides found for {}'.format(toolname))
+        override = False
+
+    if override:
+        while True:
+            s = f.readline()
+            if len(s) == 0:
+                break
+            v = s.split('\t', maxsplit=4)
+            fargs = None
+            try:
+                fargs = v[2].rstrip().split(',')
+            except IndexError:
+                fargs = []
+            argvalue = None
+            if len(v) == 4:
+                argvalue = v[3]
+            try:
+                g.override_arg(v[0], v[1], fargs, argvalue=argvalue)
+            except KeyError:
+                logg.info('adding not previously registered key {} flags: {}'.format(v[0], ','.join(fargs)))
+                g.set_arg(v[0], v[1], fargs, argvalue=argvalue)
+        f.close()
+    return g
+
+
+def get_head(tool_name, source_dir):
+    header_file = os.path.join(source_dir, tool_name + '.head.groff') 
+    f = open(header_file, 'r')
+    head = f.read()
+    f.close()
+    return head
+
+
+def get_examples(tool_name, source_dir):
+    example_file = os.path.join(source_dir, tool_name + '.examples.groff')
+    f = None
+    try:
+        f = open(example_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no examples file found for {}'.format(tool_name))
+        return None
+    logg.info('examples file {} found for {}'.format(example_file, tool_name))
+    examples = f.read()
+    f.close()
+    return examples
+
+
+def get_custom(tool_name, source_dir):
+    custom_file = os.path.join(source_dir, tool_name + '.custom.groff')
+    f = None
+    try:
+        f = open(custom_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no custom file found for {}'.format(tool_name))
+        return None
+    logg.info('custom file {} found for {}'.format(custom_file, tool_name))
+    custom = f.read()
+    f.close()
+    return custom
+
+
+def get_seealso(tool_name, source_dir):
+    seealso_file = os.path.join(source_dir, tool_name + '.seealso.groff')
+    f = None
+    try:
+        f = open(seealso_file, 'r')
+    except FileNotFoundError:
+        logg.debug('no seealso file found for {}'.format(tool_name))
+        return None
+    logg.info('seealso file {} found for {}'.format(seealso_file, tool_name))
+    seealso = f.read()
+    f.close()
+    return seealso
+
+
+g = apply_override(g, args.source_dir)
+
+ge = EnvDocGenerator(flags, override=args.overrides_env_dir)
+ge.process()
+
+head = get_head(toolname, args.source_dir)
+examples = get_examples(toolname, args.source_dir)
+custom = get_custom(toolname, args.source_dir)
+seealso = get_seealso(toolname, args.source_dir)
+
+if args.overrides_config_file != None:
+    f = open(args.overrides_config_file, 'r')
+    configuration_description = f.read()
+    f.close()
+
+(fd, fp) = tempfile.mkstemp()
+f = os.fdopen(fd, 'w')
+f.write(head)
+f.write(str(g))
+f.write(configuration_description)
+
+if custom != None:
+    f.write(custom)
+
+if examples != None:
+    f.write(".SH EXAMPLES\n\n")
+    f.write(examples)
+
+if seealso != None:
+    seealso_description = seealso
+
+if len(ge) > 0:
+    f.write(".SH ENVIRONMENT\n\n")
+    f.write(str(ge))
+
+f.write(legal_description)
+f.write(source_description)
+f.write(seealso_description)
+f.close()
+
+dest = os.path.join(args.d, toolname + '.1')
+shutil.copyfile(fp, dest)
+
+os.unlink(fp)

setup.cfg

@@ -1,12 +1,9 @@
-; Config::Simple 4.59
-; Mon Nov  8 05:19:17 2021
-
 [metadata]
 name=chainlib
 license=WTFPL2
 author_email=dev@holbrook.no
 description=Generic blockchain access library and tooling
-version=0.0.14
+version=0.1.1
 url=https://gitlab.com/chaintools/chainlib
 author=Louis Holbrook
 

setup.py

@@ -24,4 +24,7 @@ setup(
             'chainlib',
             'chainlib.cli',
             ],
+        scripts = [
+            'scripts/chainlib-man.py',
+            ],
     )