Merge pull request 'bug: Fix documentation compile and config dumps' (#2) from lash/docs-and-dumps into master

Reviewed-on: #2

CHANGELOG

@@ -1,4 +1,6 @@
-- 0.0.5-pending
+- 0.0.14
+	* Add option to skip ssl verification on rpc
+- 0.0.5
 	* Move eth code to separate package
 - 0.0.4-unreleased
 	* Add pack tx from already signed tx struct

chainlib/chain.py

@@ -1,5 +1,15 @@
 # standard imports
 import copy
+import re
+
+
+def is_valid_label(v, alpha_only=False):
+    re_m = None
+    if alpha_only:
+        re_m = r'^[a-zA-Z]+$'
+    else:
+        re_m = r'^[a-zA-Z0-9]+$'
+    return re.match(re_m, v)
 
 
 class ChainSpec:
@@ -16,13 +26,37 @@ class ChainSpec:
     :param tag: Descriptive tag
     :type tag: str
     """
-    def __init__(self, engine, common_name, network_id, tag=None):
+    def __init__(self, arch, fork, network_id, common_name=None, custom=[], safe=True):
+        if custom == None:
+            custom = []
+        elif not isinstance(custom, list):
+            raise ValueError('custom value must be list')
+
         self.o = {
-                'engine': engine,
-                'common_name': common_name,
-                'network_id': network_id,
-                'tag': tag,
-                }
+        'arch': arch,
+        'fork': fork,
+        'network_id': network_id,
+        'common_name': common_name,
+        'custom': custom,
+        }
+
+        if safe:
+            self.validate()
+
+
+    def validate(self):
+        self.o['network_id'] = int(self.o['network_id'])
+        if not is_valid_label(self.o['arch'], alpha_only=True):
+            raise ValueError('arch: ' + self.o['arch'])
+        if not is_valid_label(self.o['fork'], alpha_only=True):
+            raise ValueError('fork: ' + self.o['fork'])
+        if self.o.get('common_name') and not is_valid_label(self.o['common_name']):
+            raise ValueError('common_name: ' + self.o['common_name'])
+        if self.o.get('custom'):
+            for i, v in enumerate(self.o['custom']):
+                if not is_valid_label(v):
+                    raise ValueError('common_name {}: {}'.format(i, v))
+
 
     def network_id(self):
         """Returns the network id part of the spec.
@@ -43,12 +77,27 @@ class ChainSpec:
 
 
     def engine(self):
+        """Alias of self.arch()
+        """
+        return self.arch()
+
+
+    def arch(self):
         """Returns the chain architecture part of the spec
 
         :rtype: str
         :returns: engine
         """
-        return self.o['engine']
+        return self.o['arch']
+
+
+    def fork(self):
+        """Returns the fork part of the spec
+
+        :rtype: str
+        :returns: fork
+        """
+        return self.o['fork']
 
 
     def common_name(self):
@@ -60,6 +109,21 @@ class ChainSpec:
         return self.o['common_name']
 
 
+    def is_same_as(self, chain_spec_cmp, use_common_name=False, use_custom=False):
+        a = ['arch', 'fork', 'network_id']
+        if use_common_name:
+            a += ['common_name']
+        if use_custom:
+            a += ['custom']
+        try:
+            for k in a:
+                assert(chain_spec_cmp.o[k] == self.o[k])
+        except AssertionError:
+            return False
+        return True
+                
+
+
     @staticmethod
     def from_chain_str(chain_str):
         """Create a new ChainSpec object from a colon-separated string, as output by the string representation of the ChainSpec object.
@@ -78,10 +142,14 @@ class ChainSpec:
         o = chain_str.split(':')
         if len(o) < 3:
             raise ValueError('Chain string must have three sections, got {}'.format(len(o)))
-        tag = None
-        if len(o) == 4:
-            tag = o[3]
-        return ChainSpec(o[0], o[1], int(o[2]), tag)
+        common_name = None
+        if len(o) > 3:
+            common_name = o[3]
+        custom = []
+        if len(o) > 4:
+            for i in range(4, len(o)):
+                custom.append(o[i])
+        return ChainSpec(o[0], o[1], int(o[2]), common_name=common_name, custom=custom)
 
 
     @staticmethod
@@ -100,20 +168,35 @@ class ChainSpec:
         :rtype: chainlib.chain.ChainSpec
         :returns: Resulting chain spec
         """
-        return ChainSpec(o['engine'], o['common_name'], o['network_id'], tag=o['tag'])
+        return ChainSpec(o['arch'], o['fork'], o['network_id'], common_name=o.get('common_name'), custom=o.get('custom'))
 
   
-    def asdict(self):
+    def asdict(self, use_common_name=True, use_custom=True):
         """Create a dictionary representation of the chain spec.
 
         :rtype: dict
         :returns: Chain spec dictionary
         """
-        return copy.copy(self.o)
+        r = copy.copy(self.o)
+        if not use_common_name:
+            del r['common_name']
+            del r['custom']
+        if not use_custom:
+            del r['custom']
+        return r
+
+
+    def as_string(self, skip_optional=False):
+        s = '{}:{}:{}'.format(self.o['arch'], self.o['fork'], self.o['network_id'])
+        if skip_optional:
+            return s
+
+        if self.o.get('common_name'):
+            s += ':' + self.o['common_name']
+        if self.o.get('custom'):
+            s += ':' + ':'.join(self.o['custom'])
+        return s
 
 
     def __str__(self):
-        s = '{}:{}:{}'.format(self.o['engine'], self.o['common_name'], self.o['network_id'])
-        if self.o['tag'] != None:
-            s += ':' + self.o['tag']
-        return s
+        return self.as_string()        

chainlib/cli/arg.py

@@ -57,7 +57,7 @@ class ArgumentParser(argparse.ArgumentParser):
         self.pos_args = []
 
 
-    def add_positional(self, name, type=str, help=None, required=True):
+    def add_positional(self, name, type=str, help=None, append=False, required=True):
         """Add a positional argument.
 
         Stdin piping will only be possible in the event a single positional argument is defined.
@@ -73,7 +73,7 @@ class ArgumentParser(argparse.ArgumentParser):
         :param required: If true, argument will be set to required
         :type required: bool
         """
-        self.pos_args.append((name, type, help, required,))
+        self.pos_args.append((name, type, help, required, append,))
 
 
     def parse_args(self, argv=sys.argv[1:]):
@@ -88,16 +88,26 @@ class ArgumentParser(argparse.ArgumentParser):
         """
         if len(self.pos_args) == 1:
             arg = self.pos_args[0]
-            self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
+            if arg[4]:
+                self.add_argument(arg[0], nargs='*', type=arg[1], default=stdin_arg(), help=arg[2])
+            else:
+                self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
         else:
             for arg in self.pos_args:
                 if arg[3]:
-                    self.add_argument(arg[0], type=arg[1], help=arg[2])
+                    if arg[4]:
+                        logg.debug('argumen')
+                        self.add_argument(arg[0], nargs='+', type=arg[1], help=arg[2])
+                    else:
+                        self.add_argument(arg[0], type=arg[1], help=arg[2])
                 else:
-                    self.add_argument(arg[0], nargs='?', type=arg[1], help=arg[2])
+                    if arg[4]:
+                        self.add_argument(arg[0], nargs='*', type=arg[1], help=arg[2])
+                    else:
+                        self.add_argument(arg[0], type=arg[1], help=arg[2])
         args = super(ArgumentParser, self).parse_args(args=argv)
 
-        if args.dumpconfig:
+        if getattr(args, 'dumpconfig', None) != None:
             return args
 
         if len(self.pos_args) == 1:
@@ -134,7 +144,7 @@ class ArgumentParser(argparse.ArgumentParser):
         if arg_flags & Flag.CONFIG:
             self.add_argument('-c', '--config', type=str, default=env.get('CONFINI_DIR'), help='Configuration directory')
             self.add_argument('-n', '--namespace', type=str, help='Configuration namespace')
-            self.add_argument('--dumpconfig', action='store_true', help='Output configuration and quit. Use with --raw to omit values and output schema only.')
+            self.add_argument('--dumpconfig', type=str, choices=['env', 'ini'], help='Output configuration and quit. Use with --raw to omit values and output schema only.')
         if arg_flags & Flag.WAIT:
             self.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
             self.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')

chainlib/cli/config.py

@@ -170,7 +170,6 @@ class Config(confini.Config):
         args_override = {}
 
         if arg_flags & Flag.PROVIDER:
-            args_override['RPC_HTTP_PROVIDER'] = getattr(args, 'p')
             args_override['RPC_PROVIDER'] = getattr(args, 'p')
             args_override['RPC_DIALECT'] = getattr(args, 'rpc_dialect')
         if arg_flags & Flag.CHAIN_SPEC:
@@ -225,6 +224,7 @@ class Config(confini.Config):
             config.add(getattr(args, 'rpc_credentials'), 'RPC_CREDENTIALS')
 
         for k in extra_args.keys():
+            logg.debug('extra_agrs {}'.format(k))
             v = extra_args[k]
             if v == None:
                 v = '_' + k.upper()
@@ -236,20 +236,29 @@ class Config(confini.Config):
                 pass
             if existing_r == None or r != None:
                 config.add(r, v, exists_ok=True)
+                logg.debug('added {} to {}'.format(r, v))
 
-        if getattr(args, 'dumpconfig'):
-            config_keys = config.all()
-            with_values = not config.get('_RAW')
-            for k in config_keys:
-                if k[0] == '_':
-                    continue
-                s = k + '='
-                if with_values:
-                    v = config.get(k)
-                    if v != None:
-                        s += str(v)
-                s += '\n'
-                dump_writer.write(s)
+        if getattr(args, 'dumpconfig', None):
+            if args.dumpconfig == 'ini':
+                from confini.export import ConfigExporter
+                exporter = ConfigExporter(config, target=sys.stdout, doc=False)
+                exporter.export(exclude_sections=['config'])
+            elif args.dumpconfig == 'env':
+                from confini.env import export_env
+                export_env(config)
+
+#            config_keys = config.all()
+#            with_values = not config.get('_RAW')
+#            for k in config_keys:
+#                if k[0] == '_':
+#                    continue
+#                s = k + '='
+#                if with_values:
+#                    v = config.get(k)
+#                    if v != None:
+#                        s += str(v)
+#                s += '\n'
+#                dump_writer.write(s)
             sys.exit(0)
 
         if load_callback != None:

chainlib/cli/rpc.py

@@ -61,7 +61,7 @@ class Rpc:
             self.id_generator = IntSequenceGenerator()
 
         self.chain_spec = config.get('CHAIN_SPEC')
-        self.conn = self.constructor(url=config.get('RPC_PROVIDER'), chain_spec=self.chain_spec, auth=auth)
+        self.conn = self.constructor(url=config.get('RPC_PROVIDER'), chain_spec=self.chain_spec, auth=auth, verify_identity=config.true('RPC_VERIFY'))
 
         return self.conn
 

chainlib/cli/wallet.py

@@ -1,9 +1,6 @@
 # standard imports
 import logging
 
-# external imports
-from crypto_dev_signer.keystore.dict import DictKeystore
-
 logg = logging.getLogger(__name__)
 
 
@@ -19,7 +16,7 @@ class Wallet:
     :todo: sign_transaction_to_rlp from chainlib-eth must be renamed to sign_transaction_to_wire, and included as part of signer interface
     """
     
-    def __init__(self, signer_cls, keystore=DictKeystore(), checksummer=None):
+    def __init__(self, signer_cls, keystore=None, checksummer=None):
         self.signer_constructor = signer_cls
         self.keystore = keystore
         self.signer = None
@@ -30,6 +27,10 @@ class Wallet:
         self.use_checksum = False
 
 
+    def init(self):
+        self.signer = self.signer_constructor(self.keystore)
+
+
     def from_config(self, config):
         """Instantiates a signer from the registered signer class, using parameters from a processed configuration.
 

chainlib/connection.py

@@ -23,7 +23,10 @@ from .jsonrpc import (
         ErrorParser,
         )
 from .http import PreemptiveBasicAuthHandler
-from .error import JSONRPCException
+from .error import (
+        JSONRPCException,
+        RPCException,
+        )
 from .auth import Auth
 
 logg = logging.getLogger(__name__)
@@ -99,10 +102,13 @@ class RPCConnection:
         }
     __constructors_for_chains = {}
 
-    def __init__(self, url=None, chain_spec=None, auth=None):
+    def __init__(self, url=None, chain_spec=None, auth=None, verify_identity=True):
         self.chain_spec = chain_spec
         self.location = None
         self.basic = None
+        self.verify_identity = verify_identity
+        if not self.verify_identity:
+            logg.warning('RPC host identity verification is OFF. Beware, you will be easy to cheat')
         if url == None:
             return
         self.auth = auth
@@ -284,6 +290,11 @@ class JSONRPCHTTPConnection(HTTPConnection):
         :returns: Result value part of JSON RPC response
         :todo: Invalid response exception from invalid json response
         """
+        ssl_ctx = None
+        if not self.verify_identity:
+            import ssl
+            ssl_ctx = ssl.SSLContext()
+            ssl_ctx.verify_mode = ssl.CERT_NONE
         req = Request(
                 self.location,
                 method='POST',
@@ -308,8 +319,15 @@ class JSONRPCHTTPConnection(HTTPConnection):
                     )
             ho = build_opener(handler)
             install_opener(ho)
-       
-        r = urlopen(req, data=data.encode('utf-8'))
+
+        try: 
+            r = urlopen(
+                req,
+                data=data.encode('utf-8'),
+                context=ssl_ctx,
+                )
+        except URLError as e:
+            raise RPCException(e)
 
         result = json.load(r)
         logg.debug('(HTTP) recv {}'.format(result))

chainlib/data/config/config.ini

@@ -1,10 +1,10 @@
 [rpc]
-http_provider = 
 provider =
 auth = 
 credentials =
 dialect = default
 scheme = http
+verify = 1
 
 [chain]
 spec = 

chainlib/encode.py

@@ -30,9 +30,7 @@ class TxHexNormalizer:
 
 
     def __hex_normalize(self, data, context):
-        #r = add_0x(hex_uniform(strip_0x(data)))
         r = hex_uniform(strip_0x(data))
-        logg.debug('normalize {} {} -> {}'.format(context, data, r))
         return r
 
 

doc/texinfo/cli.texi

@@ -1,5 +1,3 @@
-@node chainlib-cli
-
 @section Command line interface provisions
 
 The base CLI provisions of @code{chainlib} simplifies the generation of a some base object instances by command line arguments, environment variables and configuration schemas.

doc/texinfo/code.texi

@@ -1,5 +1,3 @@
-@node chainlib-lib
-
 @section Base library contents
 
 

doc/texinfo/config.texi

@@ -1,5 +1,4 @@
-@node chainlib-config
-
+@anchor{chainlib-config}
 @section Rendering configurations
 
 Configurations in @code{chainlib} are processed, rendered and interfaced using the @code{confini} python package.

doc/texinfo/content.texi

@@ -1,5 +1,4 @@
-@top chainlib
-
+@node chainlib
 @chapter Chainlib
 
 @include intro.texi

doc/texinfo/intro.texi

@@ -1,4 +1,4 @@
-@node chainlib-intro
+@section Overview
 
 Chainlib is an attempt at employing a universal interface to manipulate and access blockchains regardless of underlying architecture.
 

requirements.txt

@@ -1,3 +1,3 @@
-crypto-dev-signer>=0.4.15rc2,<=0.4.15
+funga~=0.5.1
 pysha3==1.0.2
-hexathon~=0.0.1a8
+hexathon~=0.1.0

setup.cfg

@@ -1,10 +1,16 @@
+; Config::Simple 4.59
+; Mon Nov  8 05:19:17 2021
+
 [metadata]
-name = chainlib
-version = 0.0.9rc1
-description = Generic blockchain access library and tooling
-author = Louis Holbrook
-author_email = dev@holbrook.no
-url = https://gitlab.com/chaintools/chainlib
+name=chainlib
+license=WTFPL2
+author_email=dev@holbrook.no
+description=Generic blockchain access library and tooling
+version=0.0.14
+url=https://gitlab.com/chaintools/chainlib
+author=Louis Holbrook
+
+
 keywords =
 	dlt
 	blockchain
@@ -13,18 +19,8 @@ classifiers =
 	Programming Language :: Python :: 3
 	Operating System :: OS Independent
 	Development Status :: 3 - Alpha
+	Topic :: Software Development :: Libraries
 	Environment :: Console
 	Intended Audience :: Developers
 	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 	Topic :: Internet
-license = GPL3
-licence_files =
-	LICENSE.txt
-
-
-[options]
-python_requires = >= 3.6
-include_package_data = True
-packages = 
-	chainlib
-	chainlib.cli

setup.py

@@ -16,5 +16,12 @@ setup(
         install_requires=requirements,
         extras_require={
              'xdg': "pyxdg~=0.27",
-             }
+             },
+        license_files= ('LICENSE.txt',),
+        python_requires = '>=3.8',
+        include_package_data = True,
+        packages = [
+            'chainlib',
+            'chainlib.cli',
+            ],
     )

tests/test_chain.py

@@ -1,21 +1,82 @@
+# standard imports
 import unittest
+import logging
 
+# local imports
 from chainlib.chain import ChainSpec
 
+# test imports
 from tests.base import TestBase
 
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+logg.setLevel(logging.DEBUG)
+
 
 class TestChain(TestBase):
 
-    def test_chain_spec(self):
+    def test_chain_spec_str(self):
+        s = ChainSpec('foo', 'bar', 3)
+        self.assertEqual('foo:bar:3', str(s))
 
+        s = ChainSpec('foo', 'bar', 3, 'baz')
+        self.assertEqual('foo:bar:3:baz', str(s))
+
+
+        s = ChainSpec('foo', 'bar', 3, 'baz', ['inky', 'pinky', 'blinky'])
+        self.assertEqual('foo:bar:3:baz:inky:pinky:blinky', str(s))
+
+
+    def test_chain_spec(self):
         s = ChainSpec.from_chain_str('foo:bar:3')
         s = ChainSpec.from_chain_str('foo:bar:3:baz')
+        s = ChainSpec.from_chain_str('foo:bar:3:baz:inky:pinky:blinky')
 
         with self.assertRaises(ValueError):
             s = ChainSpec.from_chain_str('foo:bar:a')
             s = ChainSpec.from_chain_str('foo:bar')
             s = ChainSpec.from_chain_str('foo')
+            s = ChainSpec.from_chain_str('foo1:bar:3')
+            s = ChainSpec.from_chain_str('foo:bar2:3')
+
+
+    def test_chain_spec_dict(self):
+        ss = 'foo:bar:3:baz:inky:pinky:blinky'
+        c = ChainSpec.from_chain_str(ss)
+        d = c.asdict()
+        self.assertEqual(d['arch'], 'foo')
+        self.assertEqual(d['fork'], 'bar')
+        self.assertEqual(d['network_id'], 3)
+        self.assertEqual(d['common_name'], 'baz')
+        self.assertEqual(d['custom'], ['inky', 'pinky', 'blinky'])
+        cc = ChainSpec.from_dict(d)
+        self.assertEqual(ss, str(cc))
+
+        d = c.asdict(use_custom=False)
+        cc = ChainSpec.from_dict(d)
+        self.assertEqual(str(cc), 'foo:bar:3:baz')
+
+        d = c.asdict(use_common_name=False)
+        cc = ChainSpec.from_dict(d)
+        self.assertEqual(str(cc), 'foo:bar:3')
+
+    def test_chain_spec_compare(self):
+        a = 'foo:bar:42:baz'
+        b = 'foo:bar:42:barbar'
+        c = 'foo:bar:42:baz:inky:pinky:blinky'
+
+        ca = ChainSpec.from_chain_str(a)
+        cb = ChainSpec.from_chain_str(b)
+
+        self.assertTrue(ca.is_same_as(cb))
+        self.assertFalse(ca.is_same_as(cb, use_common_name=True))
+
+        cc = ChainSpec.from_chain_str(c)
+        logg.debug('chain_spec_cmp ' + str(cc.o))
+        self.assertTrue(ca.is_same_as(cc))
+        self.assertTrue(ca.is_same_as(cc, use_common_name=True))
+        self.assertFalse(ca.is_same_as(cc, use_common_name=True, use_custom=True))
+
 
 
 if __name__ == '__main__':

tests/test_cli.py

@@ -1,6 +1,7 @@
 # standard imports
 import unittest
 import os
+import logging
 
 # local imports
 import chainlib.cli
@@ -10,6 +11,8 @@ script_dir = os.path.dirname(os.path.realpath(__file__))
 data_dir = os.path.join(script_dir, 'testdata')
 config_dir = os.path.join(data_dir, 'config')
 
+logging.basicConfig(level=logging.DEBUG)
+
 
 class TestCli(unittest.TestCase):