Replace cic-base with chainlib cli utils

apps/cic-cache/.coveragerc

@@ -4,3 +4,4 @@ omit =
 	scripts/*
 	cic_cache/db/migrations/*
 	cic_cache/version.py
+	cic_cache/cli

apps/cic-cache/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt cic_cache/data/config/*

apps/cic-cache/cic_cache/cli/__init__.py

@@ -0,0 +1,15 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp
+from .registry import (
+        connect_registry,
+        connect_token_registry,
+        connect_declarator,
+        )

apps/cic-cache/cic_cache/cli/arg.py

@@ -0,0 +1,20 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+#    REDIS = 16
+#    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-cache/cic_cache/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-cache/cic_cache/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-cache/cic_cache/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+#        if local_arg_flags & CICFlag.REDIS:
+#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-cache/cic_cache/cli/registry.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# external imports
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
+from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
+
+logg = logging.getLogger()
+
+
+def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
+    logg.debug('using token registry address {}'.format(token_registry_address))
+    lookup = TokenIndexLookup(chain_spec, token_registry_address)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    logg.debug('using declarator address {}'.format(declarator_address))
+    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
+    CICRegistry.address = registry_address
+    registry = CICRegistry(chain_spec, conn)
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
+    return registry

apps/cic-cache/cic_cache/cli/rpc.py

@@ -0,0 +1,43 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return RPCConnection.connect(self.chain_spec, 'default')
+
+
+    @staticmethod
+    def from_config(config):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
+        if config.get('SIGNER_PROVIDER'):
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-eth
+debug = 0

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+trust_address =
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas

apps/cic-cache/cic_cache/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name = cic-cache
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-cache/cic_cache/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-cache/cic_cache/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -8,15 +8,7 @@ import sys
 import re
 
 # external imports
-import confini
-import celery
 import sqlalchemy
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import (
         dsn_from_config,
         add_tag,
@@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import (
         FaucetFilter,
         )
 
-script_dir = os.path.realpath(os.path.dirname(__file__))
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
 
-def add_block_args(argparser):
+# process args
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+arg_flags = cic_cache.cli.argflag_std_read
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+local_arg_flags = cic_cache.cli.argflag_local_sync
-    return argparser
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-logg = cic_base.log.create()
+# connect to database
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
 dsn = dsn_from_config(config)
-
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_cache.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+# set up chain provisions
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+registry = None
+try:
+    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def register_filter_tags(filters, session):
@@ -95,14 +92,12 @@ def main():
 
     syncers = []
 
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_start = config.get('SYNCER_OFFSET')
         initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
+        if config.get('SYNCER_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -112,10 +107,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
+        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
+    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/cic.ini

@@ -1,4 +1,3 @@
 [cic]
 registry_address = 
-chain_spec =
 trust_address = 

apps/cic-cache/config/docker/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 
-dir = /usr/local/share/bancor

apps/cic-cache/config/docker/cic.ini

@@ -1,4 +1,3 @@
 [cic]
-chain_spec =
 registry_address =  
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,3 +1,4 @@
 [syncer]
 loop_interval = 1
-history_start = 0
+offset = 0
+no_history = 0

apps/cic-cache/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 5
-history_start = 0

apps/cic-cache/requirements.txt

@@ -1,13 +1,14 @@
-cic-base~=0.2.0a4
 alembic==1.4.2
-confini>=0.3.6rc3,<0.5.0
+confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
-moolb~=0.1.0
+moolb~=0.1.1b2
-cic-eth-registry~=0.5.6a2
+cic-eth-registry~=0.5.8a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a5
+chainsyncer[sql]>=0.0.6a1,<0.1.0
-erc20-faucet~=0.2.2a2
+erc20-faucet~=0.2.4a1
+chainlib>=0.0.7a1,<0.1.0
+eth-address-index>=0.1.4a1,<0.2.0

apps/cic-cache/setup.cfg

@@ -23,11 +23,13 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
+include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
+	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.5a2
-sarafu-faucet~=0.0.4a1
+erc20-transfer-authorization>=0.3.4a1,<0.4.0

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp):
 
 
 def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CIC_CHAIN_SPEC')
+    chain_spec_str = config.get('CHAIN_SPEC')
     chain_spec = ChainSpec.from_chain_str(chain_spec_str)
     token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.2a3
+erc20-demurrage-token~=0.0.3a1
-cic-eth-registry~=0.5.6a1
+cic-eth-registry~=0.5.8a1
-chainlib~=0.0.5a1
+chainlib~=0.0.7a1
-cic_eth~=0.12.0a2
+cic_eth~=0.12.2a4

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a4
+version = 0.0.2a6
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -5,8 +5,7 @@ pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
 SQLAlchemy==1.3.20
-cic-eth~=0.12.0a1
 liveness~=0.0.1a7
-eth-accounts-index==0.0.12a1
+eth-accounts-index==0.1.1a1
-eth-contract-registry==0.5.6a1
+eth-contract-registry==0.5.8a1
-eth-address-index==0.1.2a1
+eth-address-index==0.2.1a1

apps/cic-eth/.coveragerc

@@ -6,4 +6,5 @@ omit =
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
 	cic_eth/queue/state.py
+	cic_eth/cli
 	*redis*.py

apps/cic-eth/MANIFEST.in

@@ -1,2 +1,2 @@
-include *requirements.txt config/test/*
+include *requirements.txt config/test/* cic_eth/data/config/*
 

apps/cic-eth/cic_eth/check/gas.py

@@ -21,7 +21,7 @@ def health(*args, **kwargs):
     session = SessionBase.create_session()
 
     config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
     logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
 
     try:

apps/cic-eth/cic_eth/check/signer.py

@@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__)
 def health(*args, **kwargs):
     blocked = True
     max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
+    conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer')
     for i in range(max_attempts):
         idx = i + 1
         logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))

apps/cic-eth/cic_eth/cli/__init__.py

@@ -0,0 +1,10 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp

apps/cic-eth/cic_eth/cli/arg.py

@@ -0,0 +1,31 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.REDIS:
+            self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+            self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+            self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+            self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
+
+
+

apps/cic-eth/cic_eth/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+    REDIS = 16
+    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-eth/cic_eth/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-eth/cic_eth/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+        if local_arg_flags & CICFlag.REDIS:
+            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-eth/cic_eth/cli/rpc.py

@@ -0,0 +1,86 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import (
+        EthUnixSignerConnection,
+        EthHTTPSignerConnection,
+        )
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return RPCConnection.connect(self.chain_spec, 'default')
+
+
+    @staticmethod
+    def from_config(config, use_signer=False):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
+        if use_signer:
+
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
+            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
+            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+# TOOD: re-implement file backend option from omittec code:
+#broker = config.get('CELERY_BROKER_URL')
+#if broker[:4] == 'file':
+#    bq = tempfile.mkdtemp()
+#    bp = tempfile.mkdtemp()
+#    conf_update = {
+#            'broker_url': broker,
+#            'broker_transport_options': {
+#                'data_folder_in': bq,
+#                'data_folder_out': bq,
+#                'data_folder_processed': bp,
+#            },
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
+#else:
+#    conf_update = {
+#            'broker_url': broker,
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#
+#result = config.get('CELERY_RESULT_URL')
+#if result[:4] == 'file':
+#    rq = tempfile.mkdtemp()
+#    current_app.conf.update({
+#        'result_backend': 'file://{}'.format(rq),
+#        })
+#    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
+#else:
+#    current_app.conf.update({
+#        'result_backend': result,
+#        })
+#

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-eth
+debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -1,8 +1,6 @@
 [cic]
 registry_address = 
-chain_spec = evm:bloxberg:8996
-tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
+default_token_symbol =
 health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/cic_eth/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name =
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -0,0 +1,2 @@
+[eth]
+gas_gifter_minimum_balance = 10000000000000000000

apps/cic-eth/cic_eth/data/config/redis.ini

@@ -1,4 +1,5 @@
 [redis]
 host = localhost
-port = 63379
+port = 6379
 db = 0
+timeout = 20.0

apps/cic-eth/cic_eth/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-eth/cic_eth/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-eth/cic_eth/runnable/create.py

@@ -7,54 +7,30 @@ import json
 import argparse
 
 # external imports
-import celery
-import confini
 import redis
 from xdg.BaseDirectory import xdg_config_home
+from chainlib.chain import ChainSpec
 
 # local imports
+import cic_eth.cli
 from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger('create_account_script')
+logg = logging.getLogger()
-logging.getLogger('confini').setLevel(logging.WARNING)
-logging.getLogger('gnupg').setLevel(logging.WARNING)
 
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+arg_flags = cic_eth.cli.argflag_std_base
-
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-argparser = argparse.ArgumentParser()
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
-argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
-argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
-argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
-argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
 args = argparser.parse_args()
 
-if args.vv:
+extra_args = {
-    logg.setLevel(logging.DEBUG)
+    'no_register': None,
-if args.v:
-    logg.setLevel(logging.INFO)
-
-config_dir = args.c
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'REDIS_HOST': getattr(args, 'redis_host'),
-        'REDIS_PORT': getattr(args, 'redis_port'),
-        'REDIS_DB': getattr(args, 'redis_db'),
         }
-config.dict_override(args_override, 'cli')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
 def main():
     redis_host = config.get('REDIS_HOST')
@@ -68,20 +44,20 @@ def main():
     ps.get_message()
 
     api = Api(
-            config.get('CIC_CHAIN_SPEC'),
+            config.get('CHAIN_SPEC'),
-            queue=args.q,
+            queue=config.get('CELERY_QUEUE'),
-            callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+            callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel),
             callback_task='cic_eth.callbacks.redis.redis',
-            callback_queue=args.q,
+            callback_queue=config.get('CELERY_QUEUE'),
             )
 
-    register = not args.no_register
+    register = not config.get('_NO_REGISTER')
     logg.debug('register {}'.format(register))
     t = api.create_account(register=register)
 
     ps.get_message()
     try:
-        o = ps.get_message(timeout=args.timeout)
+        o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
     except TimeoutError as e:
         sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e))
         sys.exit(1) 

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -12,64 +12,38 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.address import is_checksum_address
 
 # local imports
+import cic_eth.cli
 from cic_eth.api.admin import AdminApi
 from cic_eth.db.enum import LockEnum
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-default_format = 'terminal'
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
-argparser = argparse.ArgumentParser()
+arg_flags = cic_eth.cli.argflag_std_read
-argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+local_arg_flags = cic_eth.cli.argflag_local_task | cic_eth.cli.argflag_local_chain
-argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
 
 def process_lock_args(argparser):
     argparser.add_argument('flags', type=str, help='Flags to manipulate')
     argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,')
 
-sub = argparser.add_subparsers()
+sub = argparser.add_subparsers(help='')
 sub.dest = "command"
 sub_lock = sub.add_parser('lock', help='Set or reset locks')
 sub_unlock = sub.add_parser('unlock', help='Set or reset locks')
 process_lock_args(sub_lock) 
 process_lock_args(sub_unlock) 
-
 args = argparser.parse_args()
 
-if args.v == True:
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
 
-config_dir = os.path.join(args.c)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-# override args
-config.dict_override(args_override, 'cli')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-queue = args.q
-
-chain_spec = None
-if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)
 
 
@@ -100,7 +74,7 @@ def main():
                 args.address,
                 flags,
                 ],
-            queue=queue,
+            queue=config.get('CELERY_QUEUE'),
             )
         t = s.apply_async()
         logg.debug('unlock {} on {} task {}'.format(flags, args.address, t))
@@ -119,7 +93,7 @@ def main():
                 args.address,
                 flags,
                 ],
-            queue=queue,
+            queue=config.get('CELERY_QUEUE'),
             )
         t = s.apply_async()
         logg.debug('lock {} on {} task {}'.format(flags, args.address, t))

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -8,8 +8,7 @@ import sys
 import re
 import datetime
 
-# third-party imports
+# external imports
-import confini
 import celery
 from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
@@ -24,7 +23,7 @@ from chainqueue.error import NotLocalTxError
 from chainqueue.sql.state import set_reserved
 
 # local imports
-import cic_eth
+import cic_eth.cli
 from cic_eth.db import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.db import dsn_from_config
@@ -39,51 +38,30 @@ from cic_eth.error import (
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+arg_flags = cic_eth.cli.argflag_std_read 
+local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
-config_dir = os.path.join('/usr/local/etc/cic-eth')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+# connect to celery
-argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider')
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args(sys.argv[1:])
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-# override args
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'ETH_PROVIDER': getattr(args, 'p'),
-        }
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-
-queue = args.q
 
+# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
 run = True
 
+
 class DispatchSyncer:
 
     yield_delay = 0.0005

apps/cic-eth/cic_eth/runnable/daemons/retry.py

@@ -6,7 +6,6 @@ import argparse
 import re
 
 # external imports
-import confini
 import celery
 from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
@@ -14,6 +13,7 @@ from chainlib.connection import RPCConnection
 from chainsyncer.filter import SyncFilter
 
 # local imports
+import cic_eth.cli
 from cic_eth.db import dsn_from_config
 from cic_eth.db import SessionBase
 from cic_eth.admin.ctrl import lock_send
@@ -25,66 +25,41 @@ from cic_eth.stat import init_chain_stat
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-config_dir = os.path.join('/usr/local/etc/cic-eth')
+arg_flags = cic_eth.cli.argflag_std_read
-
+local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--batch-size', dest='batch_size', type=int, default=50, help='max amount of txs to resend per iteration')
-argparser.add_argument('--retry-delay', dest='retry_delay', type=int, help='seconds to wait for retrying a transaction that is marked as sent')
+argparser.add_argument('--retry-delay', dest='retry_delay', type=int, default=20, help='seconds to wait for retrying a transaction that is marked as sent')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+args = argparser.parse_args()
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args(sys.argv[1:])
 
-
+extra_args = {
-if args.v == True:
+    'retry_delay': 'RETRY_DELAY',
-    logging.getLogger().setLevel(logging.INFO)
+    'batch_size': 'RETRY_BATCH_SIZE',
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-# override args
-args_override = {
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'),
         }
-config.dict_override(args_override, 'cli flag')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-config.add(args.batch_size, '_BATCH_SIZE', True)
-
-app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-
-queue = args.q
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
 
+# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config)
+conn = rpc.get_default()
+
 
 def main(): 
-    conn = RPCConnection.connect(chain_spec, 'default')
+    straggler_delay = int(config.get('RETRY_DELAY'))
-
-    straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
     loop_interval = config.get('SYNCER_LOOP_INTERVAL')
     if loop_interval == None:
         stat = init_chain_stat(conn)
         loop_interval = stat.block_average()
 
-    syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE'))
+    syncer = RetrySyncer(conn, chain_spec, cic_eth.cli.chain_interface, straggler_delay, batch_size=config.get('RETRY_BATCH_SIZE'))
     syncer.backend.set(0, 0)
-    fltr = StragglerFilter(chain_spec, queue=queue)
+    fltr = StragglerFilter(chain_spec, queue=config.get('CELERY_QUEUE'))
     syncer.add_filter(fltr)
     syncer.loop(int(loop_interval), conn)
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -21,14 +21,17 @@ from chainlib.eth.connection import (
         EthUnixSignerConnection,
         EthHTTPSignerConnection,
         )
+from chainlib.eth.address import to_checksum_address
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
 from cic_eth_registry.erc20 import ERC20Token
+from hexathon import add_0x
 import liveness.linux
 
 
 # local imports
+import cic_eth.cli
 from cic_eth.eth import (
         erc20,
         tx,
@@ -70,114 +73,53 @@ from cic_eth.task import BaseTask
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-script_dir = os.path.dirname(os.path.realpath(__file__))
+arg_flags = cic_eth.cli.argflag_std_read
-
+local_arg_flags = cic_eth.cli.argflag_local_task
-config_dir = os.path.join('/usr/local/etc/cic-eth')
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-
+argparser.process_local_flags(local_arg_flags)
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-r', type=str, help='CIC registry address')
 argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
 argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-if args.vv:
+# process config
-    logging.getLogger().setLevel(logging.DEBUG)
+extra_args = {
-elif args.v:
+    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
-    logging.getLogger().setLevel(logging.INFO)
+    'aux_all': None,
-
+    'aux': None,
-config = confini.Config(args.c, args.env_prefix)
+    'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
-config.process()
-# override args
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
-config.add(args.q, '_CELERY_QUEUE', True)
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-health_modules = config.get('CIC_HEALTH_MODULES', [])
+# connect to celery
-if len(health_modules) != 0:
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-    health_modules = health_modules.split(',')
-logg.debug('health mods {}'.format(health_modules))
 
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config, use_signer=True)
+conn = rpc.get_default()
 
 
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-
-
-# set up celery
-current_app = celery.Celery(__name__)
-
-broker = config.get('CELERY_BROKER_URL')
-if broker[:4] == 'file':
-    bq = tempfile.mkdtemp()
-    bp = tempfile.mkdtemp()
-    conf_update = {
-            'broker_url': broker,
-            'broker_transport_options': {
-                'data_folder_in': bq,
-                'data_folder_out': bq,
-                'data_folder_processed': bp,
-            },
-            }
-    if config.true('CELERY_DEBUG'):
-        conf_update['result_extended'] = True
-    current_app.conf.update(conf_update)
-    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
-else:
-    conf_update = {
-            'broker_url': broker,
-            }
-    if config.true('CELERY_DEBUG'):
-        conf_update['result_extended'] = True
-    current_app.conf.update(conf_update)
-
-result = config.get('CELERY_RESULT_URL')
-if result[:4] == 'file':
-    rq = tempfile.mkdtemp()
-    current_app.conf.update({
-        'result_backend': 'file://{}'.format(rq),
-        })
-    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
-else:
-    current_app.conf.update({
-        'result_backend': result,
-        })
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
-RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
-RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
-
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
-#import cic_eth.checks.gas
+
-#if not cic_eth.checks.gas.health(config=config):
+# execute health checks
-#    raise RuntimeError()
+# TODO: health should be separate service with endpoint that can be queried
+health_modules = config.get('CIC_HEALTH_MODULES', [])
+if len(health_modules) != 0:
+    health_modules = health_modules.split(',')
+logg.debug('health mods {}'.format(health_modules))
 liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
-rpc = RPCConnection.connect(chain_spec, 'default')
+
+# set up chain provisions
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+registry = None
 try:
-    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 except UnknownContractError as e:
     logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
     sys.exit(1)
@@ -188,15 +130,15 @@ if trusted_addresses_src == None:
     logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
     sys.exit(1)
 trusted_addresses = trusted_addresses_src.split(',')
-for address in trusted_addresses:
+for i, address in enumerate(trusted_addresses):
+    if config.get('_UNSAFE'):
+        trusted_addresses[i] = to_checksum_address(address)
     logg.info('using trusted address {}'.format(address))
+connect_declarator(conn, chain_spec, trusted_addresses)
+connect_token_registry(conn, chain_spec)
 
-connect_declarator(rpc, chain_spec, trusted_addresses)
+# detect auxiliary task modules (plugins)
-connect_token_registry(rpc, chain_spec)
-
-# detect aux 
 # TODO: move to separate file
-#aux_dir = os.path.join(script_dir, '..', '..', 'aux')
 aux = []
 if args.aux_all:
     if len(args.aux) > 0:
@@ -249,36 +191,24 @@ elif len(args.aux) > 0:
 for v in aux:
     mname = 'cic_eth_aux.' + v
     mod = importlib.import_module(mname)
-    mod.aux_setup(rpc, config)
+    mod.aux_setup(conn, config)
     logg.info('loaded aux module {}'.format(mname))
 
 
 def main():
     argv = ['worker']
-    if args.vv:
+    log_level = logg.getEffectiveLevel()
-        argv.append('--loglevel=DEBUG')
+    log_level_name = logging.getLevelName(log_level)
-    elif args.v:
+    argv.append('--loglevel=' + log_level_name)
-        argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
     argv.append('-n')
-    argv.append(args.q)
+    argv.append(config.get('CELERY_QUEUE'))
-
-#    if config.true('SSL_ENABLE_CLIENT'):
-#        Callback.ssl = True
-#        Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
-#        Callback.ssl_key_file = config.get('SSL_KEY_FILE')
-#        Callback.ssl_password = config.get('SSL_PASSWORD')
-#
-#    if config.get('SSL_CA_FILE') != '':
-#        Callback.ssl_ca_file = config.get('SSL_CA_FILE')
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
+    default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address)
-    default_token.load(rpc)
+    default_token.load(conn)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
 
@@ -286,13 +216,13 @@ def main():
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    
     liveness.linux.set(rundir=config.get('CIC_RUN_DIR'))
-    current_app.worker_main(argv)
+    celery_app.worker_main(argv)
     liveness.linux.reset(rundir=config.get('CIC_RUN_DIR'))
 
 
 @celery.signals.eventlet_pool_postshutdown.connect
 def shutdown(sender=None, headers=None, body=None, **kwargs):
-    logg.warning('in shudown event hook')
+    logg.warning('in shutdown event hook')
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -8,14 +8,6 @@ import sys
 import re
 
 # external imports
-import confini
-import celery
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -30,8 +22,13 @@ from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver.head import HeadSyncer
 from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        )
 
 # local imports
+import cic_eth.cli
 from cic_eth.db import dsn_from_config
 from cic_eth.runnable.daemons.filters import (
         CallbackFilter,
@@ -47,61 +44,50 @@ from cic_eth.registry import (
         connect_token_registry,
         )
 
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
 
-script_dir = os.path.realpath(os.path.dirname(__file__))
+arg_flags = cic_eth.cli.argflag_std_read
+local_arg_flags = cic_eth.cli.argflag_local_sync
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
-def add_block_args(argparser):
+# process config
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
 
+# connect to celery
+cic_eth.cli.CeleryApp.from_config(config)
 
-logg = cic_base.log.create()
+# set up database
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.y, '_KEYSTORE_FILE', True)
-config.add(args.q, '_CELERY_QUEUE', True)
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
 dsn = dsn_from_config(config)
-
 SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+# set up chain provisions
-
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-rpc = RPCConnection.connect(chain_spec, 'default')
 registry = None
 try:
-    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 except UnknownContractError as e:
     logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
     sys.exit(1)
 logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
-
 def main():
-    # connect to celery
-    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
     # Connect to blockchain with chainlib
 
     o = block_latest()
-    r = rpc.do(o)
+    r = conn.do(o)
     block_current = int(r, 16)
     block_offset = block_current + 1
 
     loop_interval = config.get('SYNCER_LOOP_INTERVAL')
     if loop_interval == None:
-        stat = init_chain_stat(rpc, block_start=block_current)
+        stat = init_chain_stat(conn, block_start=block_current)
         loop_interval = stat.block_average()
 
     logg.debug('current block height {}'.format(block_offset))
@@ -113,9 +99,9 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_start = config.get('SYNCER_OFFSET')
         initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
+        if config.true('SYNCER_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -128,40 +114,45 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend, chain_interface))
+            syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend, chain_interface))
+            syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface))
 
-    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
         sys.exit(1)
     trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
+    for i, address in enumerate(trusted_addresses):
+        if not config.get('_UNSAFE'):
+            if not is_checksum_address(address):
+                raise ValueError('address {} is not a valid checksum address'.format(address))
+        else:
+            trusted_addresses[i] = to_checksum_address(address)
         logg.info('using trusted address {}'.format(address))
-    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_declarator(conn, chain_spec, trusted_addresses)
-    connect_token_registry(rpc, chain_spec)
+    connect_token_registry(conn, chain_spec)
     CallbackFilter.trusted_addresses = trusted_addresses
 
     callback_filters = []
     for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
         task_split = cb.split(':')
-        task_queue = config.get('_CELERY_QUEUE')
+        task_queue = config.get('CELERY_QUEUE')
         if len(task_split) > 1:
             task_queue = task_split[0]
         callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
         callback_filters.append(callback_filter)
 
-    tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE'))
 
     account_registry_address = registry.by_name('AccountRegistry')
-    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
+    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE'))
 
-    gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
 
     #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
 
@@ -176,7 +167,7 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        r = syncer.loop(int(loop_interval), rpc)
+        r = syncer.loop(int(loop_interval), conn)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 
         i += 1

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,50 +12,27 @@ import confini
 import celery
 
 # local imports
+import cic_eth.cli
 from cic_eth.api import Api
 from cic_eth.api.admin import AdminApi
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-default_format = 'terminal'
+arg_flags = cic_eth.cli.argflag_std_base
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-
+argparser.process_local_flags(local_arg_flags)
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
 args = argparser.parse_args()
 
-if args.v == True:
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
 
-config_dir = os.path.join(args.c)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli args')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-
+api = Api(config.get('CHAIN_SPEC'), queue=config.get('CELERY_QUEUE'))
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-queue = args.q
-
-api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
 admin_api = AdminApi(None)
 
+
 def main():
     t = admin_api.registry()
     registry_address = t.get()

apps/cic-eth/cic_eth/runnable/resend.py

@@ -5,65 +5,38 @@ import re
 import os
 
 # third-party imports
-import celery
-import confini
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
 
 # local imports
+import cic_eth.cli
 from cic_eth.api.admin import AdminApi
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-logging.getLogger('web3').setLevel(logging.WARNING)
+arg_flags = cic_eth.cli.argflag_std_base
-logging.getLogger('urllib3').setLevel(logging.WARNING)
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+argparser.add_argument('--unlock', action='store_true', help='Unlock account after resend')
-
+argparser.add_positional('tx_hash', type=str, help='Transaction hash')
-
+argparser.process_local_flags(local_arg_flags)
-argparser = argparse.ArgumentParser()
+extra_args = {
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+    'unlock': None,
-argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+    'tx_hash': None,
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
+    }
-argparser.add_argument('--unlock', action='store_true', help='Append task to unlock account')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-argparser.add_argument('tx_hash', type=str, help='Transaction hash')
 args = argparser.parse_args()
 
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
 
-if args.vv:
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-    logg.setLevel(logging.DEBUG)
-elif args.v:
-    logg.setLevel(logging.INFO)
 
-config_dir = os.path.join(args.c)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
 
-# override args
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-config.add(args.tx_hash, '_TX_HASH', True)
-config.add(args.unlock, '_UNLOCK', True)
-
-chain_spec = ChainSpec.from_chain_str(args.i)
-
-rpc = EthHTTPConnection(config.get('ETH_PROVIDER'))
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 def main():
-    api = AdminApi(rpc)
+    api = AdminApi(None)
-    tx_details = api.tx(chain_spec, args.tx_hash)
+    tx_details = api.tx(chain_spec, config.get('_TX_HASH'))
     t = api.resend(args.tx_hash, chain_spec, unlock=config.get('_UNLOCK'))
     print(t.get_leaf())
 

apps/cic-eth/cic_eth/runnable/tag.py

@@ -6,8 +6,7 @@ import argparse
 import re
 
 # external imports
-import celery
+import cic_eth.cli
-import confini
 from chainlib.chain import ChainSpec
 from xdg.BaseDirectory import xdg_config_home
 
@@ -19,43 +18,28 @@ from cic_eth.db.models.base import SessionBase
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+arg_flags = cic_eth.cli.argflag_std_base
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser.add_positional('tag', type=str, help='address tag')
+argparser.add_positional('address', type=str, help='address')
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-argparser.add_argument('tag', type=str, help='address tag')
-argparser.add_argument('address', type=str, help='address')
-args = argparser.parse_args(sys.argv[1:])
 
-if args.v == True:
+admin_api = AdminApi(None)
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
 
-config = confini.Config(args.c)
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-config.process()
-args_override = {
-        'ETH_PROVIDER': getattr(args, 'p'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}\n{}'.format(args.c, config))
 
-chain_spec = ChainSpec.from_chain_str(args.i)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-
+api = AdminApi(None)
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 
 def main():
-    api = AdminApi(None)
+    admin_api.tag_account(args.tag, args.address, chain_spec)
-    api.tag_account(args.tag, args.address, chain_spec)
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -7,88 +7,58 @@ import json
 import argparse
 
 # external imports
-import celery
-import confini
 import redis
 from xdg.BaseDirectory import xdg_config_home
 from chainlib.eth.address import to_checksum_address
 
 # local imports
+import cic_eth.cli
 from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger('create_account_script')
-logging.getLogger('confini').setLevel(logging.WARNING)
-logging.getLogger('gnupg').setLevel(logging.WARNING)
 
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+arg_flags = cic_eth.cli.argflag_std_base
-
+local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-argparser = argparse.ArgumentParser()
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
+argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_positional('sender', type=str, help='Token transaction sender')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_positional('recipient', type=str, help='Token transaction recipient')
-argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Symbol of token to transfer')
+argparser.add_positional('value', type=int, help='Token transaction value')
-argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
-argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
-argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
-argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-argparser.add_argument('sender', type=str, help='Transaction sender')
-argparser.add_argument('recipient', type=str, help='Transaction recipient')
-argparser.add_argument('value', type=int, help='Transaction value with decimals')
 args = argparser.parse_args()
 
-if args.vv:
+extra_args = {
-    logg.setLevel(logging.DEBUG)
+    'token_symbol': None,
-if args.v:
+    'sender': None,
-    logg.setLevel(logging.INFO)
+    'recipient': None,
-
+    'value': None,
-config_dir = args.c
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'REDIS_HOST': getattr(args, 'redis_host'),
-        'REDIS_PORT': getattr(args, 'redis_port'),
-        'REDIS_DB': getattr(args, 'redis_db'),
         }
-config.dict_override(args_override, 'cli')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
-config.add(to_checksum_address(args.sender), '_SENDER', True)
+
-config.add(to_checksum_address(args.recipient), '_RECIPIENT', True)
+celery_app = cic_eth.cli.CeleryApp.from_config(config)
-config.add(args.value, '_VALUE', True)
-config.add(args.token_symbol, '_SYMBOL', True)
-if config.get('_SYMBOL') == None:
-    raise ValueError('gas transfers not yet supported; token symbol required')
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 
 def main():
-    redis_host = config.get('REDIS_HOST')
-    redis_port = config.get('REDIS_PORT')
-    redis_db = config.get('REDIS_DB')
     redis_channel = str(uuid.uuid4())
-    r = redis.Redis(redis_host, redis_port, redis_db)
+    r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB'))
 
     ps = r.pubsub()
     ps.subscribe(redis_channel)
     ps.get_message()
 
     api = Api(
-            config.get('CIC_CHAIN_SPEC'),
+            config.get('CHAIN_SPEC'),
-            queue=args.q,
+            queue=config.get('CELERY_QUEUE'),
-            callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+            callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel),
             callback_task='cic_eth.callbacks.redis.redis',
-            callback_queue=args.q,
+            callback_queue=config.get('CELERY_QUEUE')
             )
 
-    t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL'))
+    t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_TOKEN_SYMBOL'))
 
     ps.get_message()
-    o = ps.get_message(timeout=args.timeout)
+    o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
     m = json.loads(o['data'])
     print(m['result'])
 

apps/cic-eth/cic_eth/sync/retry.py

@@ -3,7 +3,7 @@ import logging
 import datetime
 
 # external imports
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainsyncer.backend.memory import MemBackend
 from chainsyncer.error import NoBlockForYou
 from chainlib.eth.block import (
@@ -39,9 +39,9 @@ class DbSessionMemBackend(MemBackend):
 
 class RetrySyncer(HeadSyncer):
 
-    def __init__(self, conn, chain_spec, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None):
+    def __init__(self, conn, chain_spec, chain_interface, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None):
         backend = DbSessionMemBackend(chain_spec, None)
-        super(RetrySyncer, self).__init__(backend)
+        super(RetrySyncer, self).__init__(backend, chain_interface)
         self.chain_spec = chain_spec
         if failed_grace_seconds == None:
             failed_grace_seconds = stalled_grace_seconds

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         2,
-        'alpha.3',
+        'alpha.4',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = /usr/local/share/cic/bancor

apps/cic-eth/config/celery.ini

@@ -1,4 +0,0 @@
-[celery]
-broker_url = redis://
-result_url = redis://
-debug = 0

apps/cic-eth/config/custody.ini

@@ -1,2 +0,0 @@
-[custody]
-account_index_address =

apps/cic-eth/config/database.ini

@@ -1,9 +1,9 @@
 [database]
-NAME=cic-eth
+NAME=cic_eth
 USER=postgres
-PASSWORD=
+PASSWORD=tralala
 HOST=localhost
-PORT=5432
+PORT=63432
 ENGINE=postgresql
 DRIVER=psycopg2
 POOL_SIZE=50

apps/cic-eth/config/dispatcer.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 0.9

apps/cic-eth/config/docker/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = /usr/local/share/cic/bancor

apps/cic-eth/config/docker/celery.ini

@@ -1,4 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379
-debug = 0

apps/cic-eth/config/docker/cic.ini

@@ -1,8 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = evm:bloxberg:8996
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-tx_retry_delay = 20
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/custody.ini

@@ -1,2 +0,0 @@
-[custody]
-account_index_address =

apps/cic-eth/config/docker/dispatcer.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 0.9

apps/cic-eth/config/docker/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-provider = http://localhost:63545
-gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/config/docker/signer.ini

@@ -1,5 +0,0 @@
-[signer]
-socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
-secret = deedbeef
-database_name = signer_test
-dev_keys_path = 

apps/cic-eth/config/docker/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = false
-cert_file = 
-key_file =
-password = 
-ca_file =

apps/cic-eth/config/docker/syncer.ini

@@ -1,3 +0,0 @@
-[SYNCER]
-loop_interval = 
-history_start = 0

apps/cic-eth/config/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-provider = http://localhost:8545
-gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/config/lash/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549
-dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6

apps/cic-eth/config/lash/database.ini

@@ -1,8 +0,0 @@
-[database]
-NAME=cic-eth
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite

apps/cic-eth/config/lash/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-provider = http://localhost:8545

apps/cic-eth/config/lash/redis.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://
-result_url = file://

apps/cic-eth/config/lash/signer.ini

@@ -1,2 +0,0 @@
-[signer]
-socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/config/lash/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = true
-cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt
-key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key
-password = test
-ca_file =  /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt

apps/cic-eth/config/lash/syncer.ini

@@ -1,2 +0,0 @@
-[SYNCER]
-loop_interval = 1

apps/cic-eth/config/redis.ini

@@ -1,4 +0,0 @@
-[redis]
-host =
-port =
-db = 

apps/cic-eth/config/signer.ini

@@ -1,5 +0,0 @@
-[signer]
-socket_path = /run/crypto-dev-signer/jsonrpc.ipc
-secret = deedbeef
-database_name = signer_test
-dev_keys_path = 

apps/cic-eth/config/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = false
-cert_file = 
-key_file =
-password = 
-ca_file =

apps/cic-eth/config/syncer.ini

@@ -1,3 +0,0 @@
-[SYNCER]
-loop_interval = 
-history_start = 0

apps/cic-eth/config/tasks.ini

@@ -1,3 +0,0 @@
-[tasks]
-transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
-trace_queue_status = 1

apps/cic-eth/config/test/chain.ini

@@ -0,0 +1,2 @@
+[chain]
+spec =

apps/cic-eth/docker/Dockerfile

@@ -27,6 +27,12 @@ RUN python setup.py install
 
 ENV PYTHONPATH .
 
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
+
 COPY docker/entrypoints/* ./ 
 RUN chmod 755 *.sh
 

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.6a1,<0.1.0
+chainlib-eth>=0.0.7a1,<0.1.0
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,15 @@
 chainqueue>=0.0.3a2,<0.1.0
-chainsyncer[sql]>=0.0.5a1,<0.1.0
+chainsyncer[sql]>=0.0.6a1,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.1.3a1,<0.2.0
+eth-address-index>=0.1.4a1,<0.2.0
-eth-accounts-index>=0.0.13a1,<0.1.0
+eth-accounts-index>=0.0.14a1,<0.1.0
-cic-eth-registry>=0.5.7a1,<0.6.0
+cic-eth-registry>=0.5.8a1,<0.6.0
-erc20-faucet>=0.2.3a1,<0.3.0
+erc20-faucet>=0.2.4a1,<0.3.0
-erc20-transfer-authorization>=0.3.3a1,<0.4.0
+erc20-transfer-authorization>=0.3.4a1,<0.4.0
-sarafu-faucet>=0.0.4a5,<0.1.0
+sarafu-faucet>=0.0.5a2,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/setup.cfg

@@ -24,8 +24,10 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
+include_package_data = True
 packages = 
 	cic_eth
+	cic_eth.cli
 	cic_eth.admin
 	cic_eth.eth
 	cic_eth.api

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.0.11a1
+eth-erc20~=0.0.12a1

apps/cic-eth/tests/check/test_check_gas.py

@@ -11,7 +11,7 @@ def test_check_gas(
     whoever,
         ):
 
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True)
     config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True)
     assert health(config=config)
 

apps/cic-eth/tests/check/test_check_signer.py

@@ -9,5 +9,5 @@ def test_check_signer(
         eth_rpc,
         ):
 
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
+    config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True)
     assert health(config=config)

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,7 @@
 crypto-dev-signer>=0.4.14b7,<=0.4.14
-chainqueue~=0.0.2b6
+chainqueue>=0.0.3a1,<0.1.0
 confini>=0.3.6rc4,<0.5.0
-cic-eth-registry>=0.5.7a1,<0.6.0
+cic-eth-registry>=0.5.8a1,<0.6.0
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1

apps/cic-ussd/requirements.txt

@@ -13,5 +13,11 @@ redis==3.5.3
 semver==2.13.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
+phonenumbers==8.12.12
+redis==3.5.3
+celery==4.4.7
+python-i18n[YAML]==0.3.9
+pyxdg==0.27
+bcrypt==3.2.0
+uWSGI==2.0.19.1
 transitions==0.8.4
-uWSGI==2.0.19.1