Replace cic-base in tracker

apps/cic-eth/cic_eth/cli/__init__.py

@@ -14,6 +14,12 @@ from chainlib.eth.cli import (
         ArgumentParser as BaseArgumentParser,
         Flag,
     )
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.chain import ChainSpec
+from chainlib.eth.connection import EthUnixSignerConnection
 
 logg = logging.getLogger(__name__)
 
@@ -24,14 +30,22 @@ class CICFlag(enum.IntEnum):
    
     # celery - nibble 1
     CELERY = 1
-    CELERY_QUEUE = 2
 
     # redis - nibble 2
     REDIS = 16
     REDIS_CALLBACK = 32
- 
+
-argflag_local_task = CICFlag.CELERY
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
 argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
    
 
 class Config(BaseConfig):
@@ -54,6 +68,10 @@ class Config(BaseConfig):
             local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
             local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
             local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
         if local_arg_flags & CICFlag.CELERY:
             local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
         config.dict_override(local_args_override, 'local cli args')
@@ -65,6 +83,8 @@ class Config(BaseConfig):
         if local_arg_flags & CICFlag.CELERY:
             config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
 
+
+
         logg.debug('config loaded:\n{}'.format(config))
 
         return config
@@ -83,6 +103,11 @@ class ArgumentParser(BaseArgumentParser):
             self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
         if local_arg_flags & CICFlag.CELERY:
             self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
 
 
 class CeleryApp:
@@ -100,3 +125,31 @@ class CeleryApp:
             logg.info('creating celery app without results backend on {}'.format(broker_url))
 
         return celery_app
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return RPCConnection.connect(self.chain_spec, 'default')
+
+
+    @staticmethod
+    def from_config(config):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
+        if config.get('SIGNER_PROVIDER'):
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -1,2 +1,3 @@
 [cic]
 registry_address = 
+trust_address =

apps/cic-eth/cic_eth/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name =
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-eth/cic_eth/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-eth/cic_eth/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+block_offset = 
+no_history = 0

apps/cic-eth/cic_eth/runnable/create.py

@@ -17,7 +17,7 @@ import cic_eth.cli
 from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger('create_account_script')
+logg = logging.getLogger()
 logging.getLogger('confini').setLevel(logging.WARNING)
 logging.getLogger('gnupg').setLevel(logging.WARNING)
 

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -8,13 +8,6 @@ import sys
 import re
 
 # external imports
-import confini
-import celery
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
 from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -30,8 +23,13 @@ from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver.head import HeadSyncer
 from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
+from chainlib.eth.address import (
+        is_checksum_address,
+        to_checksum_address,
+        )
 
 # local imports
+import cic_eth.cli
 from cic_eth.db import dsn_from_config
 from cic_eth.runnable.daemons.filters import (
         CallbackFilter,
@@ -47,61 +45,52 @@ from cic_eth.registry import (
         connect_token_registry,
         )
 
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+logging.getLogger('confini').setLevel(logging.WARNING)
+logging.getLogger('gnupg').setLevel(logging.WARNING)
 
-script_dir = os.path.realpath(os.path.dirname(__file__))
+arg_flags = cic_eth.cli.argflag_std_read
+local_arg_flags = cic_eth.cli.argflag_local_sync
+argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
-def add_block_args(argparser):
+# process config
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
 
+# connect to celery
+cic_eth.cli.CeleryApp.from_config(config)
 
-logg = cic_base.log.create()
+# set up database
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.y, '_KEYSTORE_FILE', True)
-config.add(args.q, '_CELERY_QUEUE', True)
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
 dsn = dsn_from_config(config)
-
 SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_eth.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+# set up chain provisions
-
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-rpc = RPCConnection.connect(chain_spec, 'default')
 registry = None
 try:
-    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 except UnknownContractError as e:
     logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
     sys.exit(1)
 logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
-
 def main():
-    # connect to celery
-    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
     # Connect to blockchain with chainlib
 
     o = block_latest()
-    r = rpc.do(o)
+    r = conn.do(o)
     block_current = int(r, 16)
     block_offset = block_current + 1
 
     loop_interval = config.get('SYNCER_LOOP_INTERVAL')
     if loop_interval == None:
-        stat = init_chain_stat(rpc, block_start=block_current)
+        stat = init_chain_stat(conn, block_start=block_current)
         loop_interval = stat.block_average()
 
     logg.debug('current block height {}'.format(block_offset))
@@ -134,34 +123,39 @@ def main():
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
             syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
-    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
         sys.exit(1)
     trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
+    for i, address in enumerate(trusted_addresses):
+        if not config.get('_UNSAFE'):
+            if not is_checksum_address(address):
+                raise ValueError('address {} is not a valid checksum address'.format(address))
+        else:
+            trusted_addresses[i] = to_checksum_address(address)
         logg.info('using trusted address {}'.format(address))
-    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_declarator(conn, chain_spec, trusted_addresses)
-    connect_token_registry(rpc, chain_spec)
+    connect_token_registry(conn, chain_spec)
     CallbackFilter.trusted_addresses = trusted_addresses
 
     callback_filters = []
     for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
         task_split = cb.split(':')
-        task_queue = config.get('_CELERY_QUEUE')
+        task_queue = config.get('CELERY_QUEUE')
         if len(task_split) > 1:
             task_queue = task_split[0]
         callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
         callback_filters.append(callback_filter)
 
-    tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE'))
 
     account_registry_address = registry.by_name('AccountRegistry')
-    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
+    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE'))
 
-    gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
+    gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
 
     #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
 
@@ -176,7 +170,7 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        r = syncer.loop(int(loop_interval), rpc)
+        r = syncer.loop(int(loop_interval), conn)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 
         i += 1

apps/cic-eth/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = /usr/local/share/cic/bancor

apps/cic-eth/config/celery.ini

@@ -1,4 +0,0 @@
-[celery]
-broker_url = redis://
-result_url = redis://
-debug = 0

apps/cic-eth/config/cic.ini

@@ -1,8 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = evm:bloxberg:8996
-tx_retry_delay = 
-trust_address =
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/custody.ini

@@ -1,2 +0,0 @@
-[custody]
-account_index_address =

apps/cic-eth/config/database.ini

@@ -1,10 +0,0 @@
-[database]
-NAME=cic-eth
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-POOL_SIZE=50
-DEBUG=0

apps/cic-eth/config/dispatcer.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 0.9

apps/cic-eth/config/docker/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir = /usr/local/share/cic/bancor

apps/cic-eth/config/docker/celery.ini

@@ -1,4 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379
-debug = 0

apps/cic-eth/config/docker/cic.ini

@@ -1,8 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = evm:bloxberg:8996
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-tx_retry_delay = 20
-default_token_symbol = GFT
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
-run_dir = /run

apps/cic-eth/config/docker/custody.ini

@@ -1,2 +0,0 @@
-[custody]
-account_index_address =

apps/cic-eth/config/docker/dispatcer.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 0.9

apps/cic-eth/config/docker/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-provider = http://localhost:63545
-gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/config/docker/redis.ini

@@ -1,4 +0,0 @@
-[redis]
-host = localhost
-port = 63379
-db = 0

apps/cic-eth/config/docker/signer.ini

@@ -1,5 +0,0 @@
-[signer]
-socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
-secret = deedbeef
-database_name = signer_test
-dev_keys_path = 

apps/cic-eth/config/docker/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = false
-cert_file = 
-key_file =
-password = 
-ca_file =

apps/cic-eth/config/docker/syncer.ini

@@ -1,3 +0,0 @@
-[SYNCER]
-loop_interval = 
-history_start = 0

apps/cic-eth/config/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-provider = http://localhost:8545
-gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/config/lash/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549
-dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6

apps/cic-eth/config/lash/database.ini

@@ -1,8 +0,0 @@
-[database]
-NAME=cic-eth
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite

apps/cic-eth/config/lash/eth.ini

@@ -1,3 +0,0 @@
-[eth]
-gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-provider = http://localhost:8545

apps/cic-eth/config/lash/redis.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://
-result_url = file://

apps/cic-eth/config/lash/signer.ini

@@ -1,2 +0,0 @@
-[signer]
-socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc

apps/cic-eth/config/lash/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = true
-cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt
-key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key
-password = test
-ca_file =  /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt

apps/cic-eth/config/lash/syncer.ini

@@ -1,2 +0,0 @@
-[SYNCER]
-loop_interval = 1

apps/cic-eth/config/redis.ini

@@ -1,4 +0,0 @@
-[redis]
-host =
-port =
-db = 

apps/cic-eth/config/signer.ini

@@ -1,5 +0,0 @@
-[signer]
-socket_path = /run/crypto-dev-signer/jsonrpc.ipc
-secret = deedbeef
-database_name = signer_test
-dev_keys_path = 

apps/cic-eth/config/ssl.ini

@@ -1,6 +0,0 @@
-[SSL]
-enable_client = false
-cert_file = 
-key_file =
-password = 
-ca_file =

apps/cic-eth/config/syncer.ini

@@ -1,3 +0,0 @@
-[SYNCER]
-loop_interval = 
-history_start = 0

apps/cic-eth/config/tasks.ini

@@ -1,3 +0,0 @@
-[tasks]
-transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
-trace_queue_status = 1