Adds README to describe integration testing.

.gitignore

@@ -11,6 +11,3 @@ build/
 **/*sqlite
 **/.nyc_output
 **/coverage
-**/.venv
-.idea
-**/.vim

apps/cic-cache/.coveragerc

@@ -2,6 +2,4 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/migrations/*
-	cic_cache/version.py
-	cic_cache/cli
+	cic_cache/db/postgres/*

apps/cic-cache/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_cache_variables:
     variables:
         APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_cache_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-cache:
     extends:
+        - .cic_cache_changes_target
         - .py_build_merge_request
         - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
 
 build-push-cic-cache:
     extends:
         - .py_build_push
         - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
         
 

apps/cic-cache/MANIFEST.in

@@ -1 +0,0 @@
-include *requirements.txt cic_cache/data/config/*

apps/cic-cache/cic_cache/cache.py

@@ -15,8 +15,6 @@ from cic_cache.db.list import (
 logg = logging.getLogger()
 
 
-DEFAULT_FILTER_SIZE = 8192 * 8
-
 class Cache:
 
     def __init__(self, session):
@@ -27,7 +25,7 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = DEFAULT_FILTER_SIZE
+        n = 8192 * 8
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 

apps/cic-cache/cic_cache/cli/__init__.py

@@ -1,15 +0,0 @@
-# local imports
-from .base import *
-from .chain import (
-        EthChainInterface,
-        chain_interface,
-        )
-from .rpc import RPC
-from .arg import ArgumentParser
-from .config import Config
-from .celery import CeleryApp
-from .registry import (
-        connect_registry,
-        connect_token_registry,
-        connect_declarator,
-        )

apps/cic-cache/cic_cache/cli/arg.py

@@ -1,20 +0,0 @@
-# external imports
-from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
-
-# local imports
-from .base import (
-        CICFlag,
-        Flag,
-        )
-
-
-class ArgumentParser(BaseArgumentParser):
-
-    def process_local_flags(self, local_arg_flags):
-        if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
-        if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
-            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-        if local_arg_flags & CICFlag.CHAIN:
-            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/base.py

@@ -1,31 +0,0 @@
-# standard imports
-import enum
-
-# external imports
-from chainlib.eth.cli import (
-    argflag_std_read,
-    argflag_std_write,
-    argflag_std_base,
-    Flag,
-    )
-
-class CICFlag(enum.IntEnum):
-   
-    # celery - nibble 1
-    CELERY = 1
-
-    # redis - nibble 2
-#    REDIS = 16
-#    REDIS_CALLBACK = 32
-
-    # chain - nibble 3
-    CHAIN = 256
-
-    # sync - nibble 4
-    SYNCER = 4096
-
-
-argflag_local_task = CICFlag.CELERY 
-#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
-argflag_local_chain = CICFlag.CHAIN
-argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-cache/cic_cache/cli/celery.py

@@ -1,24 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-logg = logging.getLogger(__name__)
-
-
-class CeleryApp:
-   
-    @classmethod
-    def from_config(cls, config):
-        backend_url = config.get('CELERY_RESULT_URL')
-        broker_url = config.get('CELERY_BROKER_URL')
-        celery_app = None
-        if backend_url != None:
-            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
-            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
-        else:
-            celery_app = celery.Celery(broker=broker_url)
-            logg.info('creating celery app without results backend on {}'.format(broker_url))
-
-        return celery_app

apps/cic-cache/cic_cache/cli/chain.py

@@ -1,21 +0,0 @@
-# external imports
-from chainlib.eth.block import (
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        Tx,
-        )
-from chainlib.interface import ChainInterface
-
-
-class EthChainInterface(ChainInterface):
-    
-    def __init__(self):
-        self._tx_receipt = receipt
-        self._block_by_number = block_by_number
-        self._block_from_src = Block.from_src
-        self._src_normalize = Tx.src_normalize
-
-chain_interface = EthChainInterface()

apps/cic-cache/cic_cache/cli/config.py

@@ -1,63 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.cli import (
-        Config as BaseConfig,
-        Flag,
-        )
-
-# local imports
-from .base import CICFlag
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-
-logg = logging.getLogger(__name__)
-
-
-class Config(BaseConfig):
-
-    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
-
-    @classmethod
-    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
-        expanded_base_config_dir = [cls.local_base_config_dir]
-        if base_config_dir != None:
-            if isinstance(base_config_dir, str):
-                base_config_dir = [base_config_dir]
-            for d in base_config_dir:
-                expanded_base_config_dir.append(d)
-        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
-
-        local_args_override = {}
-#        if local_arg_flags & CICFlag.REDIS:
-#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
-#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
-#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
-#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
-
-        if local_arg_flags & CICFlag.CHAIN:
-            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
-
-        if local_arg_flags & CICFlag.CELERY:
-            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
-
-        if local_arg_flags & CICFlag.SYNCER:
-            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
-            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
-
-        config.dict_override(local_args_override, 'local cli args')
-
-#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
-        if local_arg_flags & CICFlag.CELERY:
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        logg.debug('config loaded:\n{}'.format(config))
-
-        return config
-
-

apps/cic-cache/cic_cache/cli/registry.py

@@ -1,33 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
-from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
-from chainlib.eth.constant import ZERO_ADDRESS
-
-logg = logging.getLogger()
-
-
-def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
-    logg.debug('using token registry address {}'.format(token_registry_address))
-    lookup = TokenIndexLookup(chain_spec, token_registry_address)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    logg.debug('using declarator address {}'.format(declarator_address))
-    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
-    CICRegistry.address = registry_address
-    registry = CICRegistry(chain_spec, conn)
-    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
-    return registry

apps/cic-cache/cic_cache/cli/rpc.py

@@ -1,43 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import EthUnixSignerConnection
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-
-class RPC:
-
-    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
-        self.chain_spec = chain_spec
-        self.rpc_provider = rpc_provider
-        self.signer_provider = signer_provider
-
-
-    def get_default(self):
-        return RPCConnection.connect(self.chain_spec, 'default')
-
-
-    @staticmethod
-    def from_config(config):
-        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
-        if config.get('SIGNER_PROVIDER'):
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
-        logg.info('set up rpc: {}'.format(rpc))
-        return rpc
-
-
-    def __str__(self):
-        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
-
-
-

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -1,5 +0,0 @@
-[celery]
-broker_url = redis://localhost:6379
-result_url = 
-queue = cic-eth
-debug = 0

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-trust_address =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas

apps/cic-cache/cic_cache/data/config/database.ini

@@ -1,10 +0,0 @@
-[database]
-engine =
-driver =
-host =
-port =
-name = cic-cache
-user =
-password =
-debug = 0
-pool_size = 0

apps/cic-cache/cic_cache/data/config/signer.ini

@@ -1,2 +0,0 @@
-[signer]
-provider =

apps/cic-cache/cic_cache/data/config/syncer.ini

@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.default.export import (
+from chainsyncer.db.migrations.sqlalchemy import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,4 +100,3 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,9 +4,6 @@ import json
 import re
 import base64
 
-# external imports
-from hexathon import add_0x
-
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -14,11 +11,10 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
-#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
+re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -30,13 +26,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = add_0x(address)
-    offset = 0
+        address = '0x' + address
+    offset = DEFAULT_LIMIT
     if r.lastindex > 2:
-        offset = r[4]
-    limit = DEFAULT_LIMIT
-    if r.lastindex > 4:
-        limit = r[6]
+        offset = r[3]
+    limit = 0
+    if r.lastindex > 3:
+        limit = r[4]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -8,7 +8,14 @@ import sys
 import re
 
 # external imports
+import confini
+import celery
 import sqlalchemy
+import rlp
+import cic_base.config
+import cic_base.log
+import cic_base.argparse
+import cic_base.rpc
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -21,12 +28,13 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
-import cic_cache.cli
 from cic_cache.db import (
         dsn_from_config,
         add_tag,
@@ -36,36 +44,32 @@ from cic_cache.runnable.daemons.filters import (
         FaucetFilter,
         )
 
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
+script_dir = os.path.realpath(os.path.dirname(__file__))
 
-# process args
-arg_flags = cic_cache.cli.argflag_std_read
-local_arg_flags = cic_cache.cli.argflag_local_sync
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
 
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-# connect to database
+logg = cic_base.log.create()
+argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
+args = cic_base.argparse.parse(argparser, logg)
+config = cic_base.config.create(args.c, args, args.env_prefix)
+
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
+
+cic_base.config.log(config)
+
 dsn = dsn_from_config(config)
+
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-# set up rpc
-rpc = cic_cache.cli.RPC.from_config(config)
-conn = rpc.get_default()
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-# set up chain provisions
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-registry = None
-try:
-    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
+cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
 def register_filter_tags(filters, session):
@@ -92,12 +96,14 @@ def main():
 
     syncers = []
 
+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
+        initial_block_start = config.get('SYNCER_HISTORY_START')
         initial_block_offset = block_offset
-        if config.get('SYNCER_NO_HISTORY'):
+        if config.get('_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -107,10 +113,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
+        syncers.append(HistorySyncer(syncer_backend))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
+    syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/config/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir =

apps/cic-cache/config/cic.ini

@@ -1,3 +1,4 @@
 [cic]
 registry_address = 
+chain_spec =
 trust_address = 

apps/cic-cache/config/docker/bancor.ini

@@ -0,0 +1,3 @@
+[bancor]
+registry_address = 
+dir = /usr/local/share/bancor

apps/cic-cache/config/docker/cic.ini

@@ -1,3 +1,4 @@
 [cic]
+chain_spec =
 registry_address =  
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/eth.ini

@@ -0,0 +1,2 @@
+[eth]
+provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,4 +1,3 @@
 [syncer]
 loop_interval = 1
-offset = 0
-no_history = 0
+history_start = 0

apps/cic-cache/config/eth.ini

@@ -0,0 +1,2 @@
+[eth]
+provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -0,0 +1,3 @@
+[syncer]
+loop_interval = 5
+history_start = 0

apps/cic-cache/docker/Dockerfile

@@ -1,38 +1,52 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+FROM python:3.8.6-slim-buster
 
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
+WORKDIR /usr/src/cic-cache
 
-COPY . . 
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG root_requirement_file='requirements.txt'
 
-RUN python setup.py install
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
+
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file ${root_requirement_file}"
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY cic-cache/requirements.txt ./
+COPY cic-cache/setup.cfg \
+	 cic-cache/setup.py \
+ 	 ./
+COPY cic-cache/cic_cache/ ./cic_cache/
+COPY cic-cache/scripts/ ./scripts/
+COPY cic-cache/test_requirements.txt ./
+RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+RUN pip install $pip_extra_index_url_flag .
+RUN pip install .[server]
+
+COPY cic-cache/tests/ ./tests/
+#COPY db/ cic-cache/db
+#RUN apk add postgresql-client
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
+COPY cic-cache/config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
+COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
+COPY cic-cache/docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -1,37 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/requirements.txt

@@ -1,14 +1,12 @@
+cic-base~=0.1.2b10
 alembic==1.4.2
-confini>=0.3.6rc4,<0.5.0
+confini~=0.3.6rc3
 uwsgi==2.0.19.1
-moolb~=0.1.1b2
-cic-eth-registry~=0.5.8a1
+moolb~=0.1.0
+cic-eth-registry~=0.5.5a4
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a1,<0.1.0
-erc20-faucet~=0.2.4a1
-chainlib>=0.0.7a1,<0.1.0
-eth-address-index>=0.1.4a1,<0.2.0
+chainsyncer[sql]~=0.0.2a4

apps/cic-cache/scripts/migrate.py

@@ -2,7 +2,6 @@
 import os
 import argparse
 import logging
-import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -24,8 +23,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -56,10 +53,4 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/setup.cfg

@@ -23,13 +23,11 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
-include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
-	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,6 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.5a2
-erc20-transfer-authorization>=0.3.4a1,<0.4.0
+web3==5.12.2
+cic-eth-registry~=0.5.5a3
+cic-base[full]==0.1.2b8

apps/cic-cache/tests/conftest.py

@@ -5,12 +5,9 @@ import datetime
 
 # external imports
 import pytest
-import moolb
 
 # local imports
 from cic_cache import db
-from cic_cache import BloomCache
-from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -104,7 +101,3 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
-
-@pytest.fixture(scope='session')
-def zero_filter():
-    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,7 +10,6 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
-from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -19,21 +18,10 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
-def test_base_filter_str(
-        init_database,
-        ):
-        f = TagSyncFilter('foo')
-        assert 'foo' == str(f)
-        f = TagSyncFilter('foo', domain='bar')
-        assert 'bar.foo' == str(f)
-
-
-
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -79,95 +67,3 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
-
-
-def test_erc20_filter_nocontract(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': os.urandom(20).hex(),
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-
-@pytest.mark.parametrize(
-        'contract_method,contract_input,expected_exception',
-        [
-            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
-            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
-            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
-            ],
-        )
-def test_erc20_filter_bogus(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        contract_method,
-        contract_input,
-        expected_exception,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = contract_method
-    data += contract_input 
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_query.py

@@ -1,230 +0,0 @@
-# standard imports
-import logging
-import json
-import base64
-import copy
-import re
-
-# external imports
-import pytest
-from hexathon import strip_0x
-
-# local imports
-from cic_cache.runnable.daemons.query import *
-
-logg = logging.getLogger()
-
-
-
-@pytest.mark.parametrize(
-        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
-        [
-            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
-            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
-            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
-            ],
-        )
-def test_query_regex(
-        list_actors,
-        query_path_prefix,
-        query_role,
-        query_address_index,
-        query_offset,
-        query_offset_index,
-        query_limit,
-        query_limit_index,
-        match_re,
-        ):
-
-        paths = []
-        path = query_path_prefix
-        query_address = None
-        if query_role != None:
-            query_address = strip_0x(list_actors[query_role])
-            paths.append(path + '0x' + query_address)
-            paths.append(path + query_address)
-        if query_offset != None:
-            if query_limit != None:
-                for i in range(len(paths)-1):
-                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
-            else:
-                for i in range(len(paths)-1):
-                    paths[i] += '/' + str(query_offset)
-
-        for i in range(len(paths)):
-            paths.append(paths[i] + '/')
-
-        for p in paths:
-            logg.debug('testing path {} against {}'.format(p, match_re))
-            m = re.match(match_re, p)
-            l = len(m.groups())
-            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
-            for i in range(l+1):
-                logg.debug('group {} {}'.format(i, m[i]))
-            if m.lastindex >= query_offset_index:
-                assert query_offset == int(m[query_offset_index + 1])
-            if m.lastindex >= query_limit_index:
-                assert query_limit == int(m[query_limit_index + 1])
-            if query_address_index != None:
-                match_address = strip_0x(m[query_address_index + 1])
-                assert query_address == match_address
-
-
-
-@pytest.mark.parametrize(
-        'role_name, query_offset, query_limit, query_match',
-        [ 
-            ('alice', None, None, [(420000, 13), (419999, 42)]),
-            ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
-            ('alice', 2, None, []), # 420000 == list_defaults['block']
-            ],
-        )
-def test_query_process_txs_account(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        role_name,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    actor = None
-    try:
-        actor = list_actors[role_name]
-    except KeyError:
-        actor = os.urandom(20).hex()
-    path_info = '/tx/user/0x' + strip_0x(actor)
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_account_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_offset, query_limit, query_match',
-        [ 
-            (None, 2, [(420000, 13), (419999, 42)]),
-            (0, 1, [(420000, 13)]),
-            (1, 1, [(419999, 42)]),
-            (2, 0, []),
-            ],
-        )
-def test_query_process_txs_bloom(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    path_info = '/tx'
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_block_start, query_block_end, query_match_count',
-        [ 
-            (None, 42, 0),
-            (420000, 420001, 1),
-            (419999, 419999, 1), # matches are inclusive
-            (419999, 420000, 2),
-            (419999, 420001, 2),
-            ],
-        )
-def test_query_process_txs_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_block_start,
-        query_block_end,
-        query_match_count,
-        ):
-
-    path_info = '/txa'
-    if query_block_start != None:
-        path_info += '/' + str(query_block_start)
-    if query_block_end != None:
-        if query_block_start == None:
-            path_info += '/0'
-        path_info += '/' + str(query_block_end)
-    env = {
-            'PATH_INFO': path_info,
-            'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_data(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    assert len(o['data']) == query_match_count

apps/cic-eth-aux/erc20-demurrage-token/MANIFEST.in

@@ -1 +0,0 @@
-include *requirements.txt

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -1,53 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from erc20_demurrage_token.demurrage import DemurrageCalculator
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from chainlib.eth.constant import ZERO_ADDRESS
-from cic_eth_registry import CICRegistry
-
-logg = logging.getLogger(__name__)
-
-celery_app = celery.current_app
-
-
-class NoopCalculator:
-
-    def amount_since(self, amount, timestamp):
-        logg.debug('noopcalculator amount {} timestamp {}'.format(amount, timestamp))
-        return amount
-
-
-class DemurrageCalculationTask(celery.Task):
-
-    demurrage_token_calcs = {}
-
-    @classmethod
-    def register_token(cls, rpc, chain_spec, token_symbol, sender_address=ZERO_ADDRESS):
-        registry = CICRegistry(chain_spec, rpc)
-        token_address = registry.by_name(token_symbol, sender_address=sender_address)
-        try:
-            c = DemurrageCalculator.from_contract(rpc, chain_spec, token_address, sender_address=sender_address)
-            logg.info('found demurrage calculator for ERC20 {} @ {}'.format(token_symbol, token_address))
-        except:
-            logg.warning('Token {} at address {} does not appear to be a demurrage contract. Calls to balance adjust for this token will always return the same amount'.format(token_symbol, token_address))
-            c = NoopCalculator()
-
-        cls.demurrage_token_calcs[token_symbol] = c
-
-
-@celery_app.task(bind=True, base=DemurrageCalculationTask)
-def get_adjusted_balance(self, token_symbol, amount, timestamp):
-    c = self.demurrage_token_calcs[token_symbol]
-    return c.amount_since(amount, timestamp)
-
-
-def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CHAIN_SPEC')
-    chain_spec = ChainSpec.from_chain_str(chain_spec_str)
-    token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    
-    DemurrageCalculationTask.register_token(rpc, chain_spec, token_symbol, sender_address=sender_address)

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/api.py

@@ -1,30 +0,0 @@
-# standard imports
-import logging
-
-# external imports 
-import celery
-from cic_eth.api.base import ApiBase
-
-app = celery.current_app
-
-logg = logging.getLogger(__name__)
-
-
-class Api(ApiBase):
-   
-    def get_adjusted_balance(self, token_symbol, balance, timestamp):
-        s = celery.signature(
-                'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
-                [
-                    token_symbol,
-                    balance,
-                    timestamp,
-                    ],
-                queue=None,
-                )
-        if self.callback_param != None:
-            s.link(self.callback_success)
-            s.link.on_error(self.callback_error)
-
-        t = s.apply_async(queue=self.queue)
-        return t

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +0,0 @@
-celery==4.4.7
-erc20-demurrage-token~=0.0.3a1
-cic-eth-registry~=0.5.8a1
-chainlib~=0.0.7a1
-cic_eth~=0.12.2a4

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,30 +0,0 @@
-[metadata]
-name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a6
-description = cic-eth tasks supporting erc20 demurrage token
-author = Louis Holbrook
-author_email = dev@holbrook.no
-url = https://gitlab.com/ccicnet/erc20-demurrage-token
-keywords =
-	ethereum
-	blockchain
-	cryptocurrency
-	erc20
-classifiers =
-	Programming Language :: Python :: 3
-	Operating System :: OS Independent
-	Development Status :: 3 - Alpha
-	Environment :: No Input/Output (Daemon)
-	Intended Audience :: Developers
-	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-	Topic :: Internet
-	#Topic :: Blockchain :: EVM
-license = GPL3
-licence_files =
-	LICENSE
-
-[options]
-include_package_data = True
-python_requires = >= 3.6
-packages =
-	cic_eth_aux.erc20_demurrage_token

apps/cic-eth-aux/erc20-demurrage-token/setup.py

@@ -1,25 +0,0 @@
-from setuptools import setup
-
-requirements = []
-f = open('requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    requirements.append(l.rstrip())
-f.close()
-
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
-setup(
-        install_requires=requirements,
-        tests_require=test_requirements,
-        )

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -1,11 +0,0 @@
-pytest==6.0.1
-pytest-celery==0.0.0a1
-pytest-mock==3.3.1
-pytest-cov==2.10.1
-eth-tester==0.5.0b3
-py-evm==0.3.0a20
-SQLAlchemy==1.3.20
-liveness~=0.0.1a7
-eth-accounts-index==0.1.1a1
-eth-contract-registry==0.5.8a1
-eth-address-index==0.2.1a1

apps/cic-eth-aux/erc20-demurrage-token/tests/conftest.py

@@ -1,88 +0,0 @@
-# external imports
-import celery
-from chainlib.eth.pytest.fixtures_chain import *
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from cic_eth_registry.pytest.fixtures_tokens import *
-from erc20_demurrage_token.unittest.base import TestTokenDeploy
-from erc20_demurrage_token.token import DemurrageToken
-from eth_token_index.index import TokenUniqueSymbolIndex
-from eth_address_declarator.declarator import AddressDeclarator
-
-# cic-eth imports
-from cic_eth.pytest.fixtures_celery import *
-from cic_eth.pytest.fixtures_token import *
-from cic_eth.pytest.fixtures_config import *
-
-
-@pytest.fixture(scope='function')
-def demurrage_token(
-        default_chain_spec,
-        eth_rpc,
-        token_registry,
-        contract_roles,
-        eth_signer,
-        ):
-    d = TestTokenDeploy(eth_rpc, token_symbol='BAR', token_name='Bar Token')
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    c = DemurrageToken(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    token_address =  d.deploy(eth_rpc, contract_roles['CONTRACT_DEPLOYER'], c, 'SingleNocap')
-    logg.debug('demurrage token contract "BAR" deployed to {}'.format(token_address))
-
-    return token_address
-    
-
-@pytest.fixture(scope='function')
-def demurrage_token_symbol(
-        default_chain_spec,
-        eth_rpc,
-        demurrage_token,
-        contract_roles,
-        ):
-
-    c = DemurrageToken(default_chain_spec)
-    o = c.symbol(demurrage_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
-@pytest.fixture(scope='function')
-def demurrage_token_declaration(
-        foo_token_declaration,
-        ):
-    return foo_token_declaration
-
-
-@pytest.fixture(scope='function')
-def register_demurrage_token(
-        default_chain_spec,
-        token_registry,
-        eth_rpc,
-        eth_signer,
-        register_lookups,
-        contract_roles,
-        demurrage_token_declaration,
-        demurrage_token,
-        address_declarator,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
-
-    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], demurrage_token)
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    nonce_oracle = RPCNonceOracle(contract_roles['TRUSTED_DECLARATOR'], eth_rpc)
-    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['TRUSTED_DECLARATOR'], demurrage_token, demurrage_token_declaration)
-
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    return token_registry
-

apps/cic-eth-aux/erc20-demurrage-token/tests/test_demurrage_task.py

@@ -1,69 +0,0 @@
-# standard imports
-import logging
-import copy
-import datetime
-
-# external imports
-import celery
-
-# cic-eth imports
-from cic_eth_aux.erc20_demurrage_token import (
-        DemurrageCalculationTask,
-        aux_setup,
-        )
-from cic_eth_aux.erc20_demurrage_token.api import Api as AuxApi
-
-logg = logging.getLogger()
-
-
-def test_demurrage_calulate_task(
-        default_chain_spec,
-        eth_rpc,
-        cic_registry,
-        celery_session_worker,
-        register_demurrage_token,
-        demurrage_token_symbol,
-        contract_roles,
-        load_config,
-        ):
- 
-    config = copy.copy(load_config)
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    config.add(demurrage_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
-    aux_setup(eth_rpc, load_config, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
-    s = celery.signature(
-            'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
-            [
-                demurrage_token_symbol,
-                1000,
-                since.timestamp(),
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    assert r == 980
-
-
-
-def test_demurrage_calculate_api(
-        default_chain_spec,
-        eth_rpc,
-        cic_registry,
-        celery_session_worker,
-        register_demurrage_token,
-        demurrage_token_symbol,
-        contract_roles,
-        load_config,
-        ):
-
-        api = AuxApi(str(default_chain_spec), queue=None)
-        since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
-        t = api.get_adjusted_balance(demurrage_token_symbol, 1000, since.timestamp())
-        r = t.get_leaf()
-        assert t.successful()
-        assert r == 980
-

apps/cic-eth/.coveragerc

@@ -6,5 +6,4 @@ omit =
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
 	cic_eth/queue/state.py
-	cic_eth/cli
 	*redis*.py

apps/cic-eth/.dockerignore

@@ -1,6 +0,0 @@
-.git
-.cache
-.dot
-**/doc
-**/.venv
-**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,33 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_eth_changes_target:
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      #changes:
+          #- $CONTEXT/$APP_NAME/**/*
+      when: always
 
 build-mr-cic-eth:
     extends:
         - .cic_eth_variables
-        - .py_build_target_dev
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
+        - .cic_eth_changes_target
+        - .py_build_target_test
 
 test-mr-cic-eth:
-    stage: test
     extends:
         - .cic_eth_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
+        - .cic_eth_changes_target
+    stage: test
+    image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
     script:
         - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
     needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always

apps/cic-eth/MANIFEST.in

@@ -1,2 +0,0 @@
-include *requirements.txt config/test/* cic_eth/data/config/*
-

apps/cic-eth/admin_requirements.txt

@@ -1,5 +0,0 @@
-SQLAlchemy==1.3.20
-cic-eth-registry>=0.5.6a2,<0.6.0
-hexathon~=0.0.1a7
-chainqueue>=0.0.3a1,<0.1.0
-eth-erc20>=0.0.10a3,<0.1.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -6,11 +6,6 @@ import logging
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from hexathon import (
-        add_0x,
-        strip_0x,
-        uniform as hex_uniform,
-        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -24,12 +19,6 @@ from cic_eth.error import LockedError
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
@@ -43,7 +32,6 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -65,7 +53,6 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -85,7 +72,6 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -103,7 +89,6 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -121,7 +106,6 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -139,7 +123,6 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -148,7 +131,6 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))

apps/cic-eth/cic_eth/admin/nonce.py

@@ -14,11 +14,7 @@ from chainqueue.sql.query import get_tx
 from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import (
-        strip_0x,
-        add_0x,
-        uniform as hex_uniform,
-        )
+from hexathon import strip_0x
 from potaahto.symbols import snake_and_camel
 
 # local imports
@@ -73,17 +69,15 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==query_address)
+    q = q.filter(TxCache.sender==address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
-    gas_total = 0
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
@@ -95,10 +89,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         tx_new['gas_price'] += 1
         tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
-        gas_total += tx_new['gas_price'] * tx_new['gas']
 
         logg.debug('tx_new {}'.format(tx_new))
-        logg.debug('gas running total {}'.format(gas_total))
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
@@ -130,10 +122,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s = create_check_gas_task(
          txs, 
          chain_spec,
-         #tx_new['from'],
-         address,
-         #gas=tx_new['gas'],
-         gas=gas_total,
+         tx_new['from'],
+         gas=tx_new['gas'],
          tx_hashes_hex=tx_hashes, 
          queue=queue,
         )
@@ -142,8 +132,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_send',
         [
             chain_spec.asdict(),
-            address,
-            #tx_new['from'],
+            tx_new['from'],
             ],
         queue=queue,
         )
@@ -151,8 +140,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_queue',
         [
             chain_spec.asdict(),
-            address,
-            #tx_new['from'],
+            tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/api/__init__.py

@@ -5,3 +5,4 @@
 """
 
 from .api_task import Api
+from .api_admin import AdminApi

apps/cic-eth/cic_eth/api/api_admin.py

@@ -21,7 +21,6 @@ from chainlib.hash import keccak256_hex_to_hex
 from hexathon import (
         strip_0x,
         add_0x,
-        uniform as hex_uniform,
         )
 from chainlib.eth.gas import balance
 from chainqueue.db.enum import (
@@ -308,8 +307,6 @@ class AdminApi:
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
         """
-
-        address = add_0x(hex_uniform(strip_0x(address)))
         last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',
@@ -565,13 +562,13 @@ class AdminApi:
             tx['source_token_symbol'] = source_token.symbol
             o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance(r)
+            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
 
         if destination_token != None:
             tx['destination_token_symbol'] = destination_token.symbol
             o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
+            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
             #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which

apps/cic-eth/cic_eth/api/api_task.py

@@ -8,19 +8,59 @@ import logging
 
 # external imports 
 import celery
+from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 
 # local imports
-from cic_eth.api.base import ApiBase
-from cic_eth.enum import LockEnum
+from cic_eth.db.enum import LockEnum
 
 app = celery.current_app
 
 logg = logging.getLogger(__name__)
 
 
-class Api(ApiBase):
-    
+class Api:
+    """Creates task chains to perform well-known CIC operations.
+
+    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
+
+    :param callback_param: Static value to pass to callback
+    :type callback_param: str
+    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
+    :type callback_task: string
+    :param queue: Name of worker queue to submit tasks to
+    :type queue: str
+    """
+    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
+        self.chain_str = chain_str
+        self.chain_spec = ChainSpec.from_chain_str(chain_str)
+        self.callback_param = callback_param
+        self.callback_task = callback_task
+        self.queue = queue
+        logg.debug('api using queue {}'.format(self.queue))
+        self.callback_success = None
+        self.callback_error = None
+        if callback_queue == None:
+            callback_queue=self.queue
+
+        if callback_param != None:
+            self.callback_success = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        0,
+                        ],
+                    queue=callback_queue,
+                    )
+            self.callback_error = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        1,
+                        ],
+                    queue=callback_queue,
+                    )       
+
 
     def default_token(self):
         s_token = celery.signature(
@@ -164,82 +204,6 @@ class Api(ApiBase):
 #        return t
 
 
-    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
-        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
-
-        :param from_address: Ethereum address of sender
-        :type from_address: str, 0x-hex
-        :param to_address: Ethereum address of recipient
-        :type to_address: str, 0x-hex
-        :param value: Estimated return from conversion
-        :type  value: int
-        :param token_symbol: ERC20 token symbol of token to send
-        :type token_symbol: str
-        :param spender_address: Ethereum address of recipient
-        :type spender_address: str, 0x-hex
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        s_check = celery.signature(
-                'cic_eth.admin.ctrl.check_lock',
-                [
-                    [token_symbol],
-                    self.chain_spec.asdict(),
-                    LockEnum.QUEUE,
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_nonce = celery.signature(
-                'cic_eth.eth.nonce.reserve_nonce',
-                [
-                    self.chain_spec.asdict(),
-                    from_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    self.chain_spec.asdict(),
-                    ],
-                queue=self.queue,
-                )
-        s_allow = celery.signature(
-                'cic_eth.eth.erc20.check_allowance',
-                [
-                    from_address,
-                    value,
-                    self.chain_spec.asdict(),
-                    spender_address,
-                    ],
-                queue=self.queue,
-                )
-        s_transfer = celery.signature(
-                'cic_eth.eth.erc20.transfer_from',
-                [
-                    from_address,
-                    to_address,
-                    value,
-                    self.chain_spec.asdict(),
-                    spender_address,
-                    ],
-                queue=self.queue,
-                )
-        s_tokens.link(s_allow)
-        s_nonce.link(s_tokens)
-        s_check.link(s_nonce)
-        if self.callback_param != None:
-            s_transfer.link(self.callback_success)
-            s_allow.link(s_transfer).on_error(self.callback_error)
-        else:
-            s_allow.link(s_transfer)
-
-        t = s_check.apply_async(queue=self.queue)
-        return t
-
-
-
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/api/base.py

@@ -1,52 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-class ApiBase:
-    """Creates task chains to perform well-known CIC operations.
-
-    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
-
-    :param callback_param: Static value to pass to callback
-    :type callback_param: str
-    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
-    :type callback_task: string
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
-        self.chain_str = chain_str
-        self.chain_spec = ChainSpec.from_chain_str(chain_str)
-        self.callback_param = callback_param
-        self.callback_task = callback_task
-        self.queue = queue
-        logg.debug('api using queue {}'.format(self.queue))
-        self.callback_success = None
-        self.callback_error = None
-        if callback_queue == None:
-            callback_queue=self.queue
-
-        if callback_param != None:
-            self.callback_success = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        0,
-                        ],
-                    queue=callback_queue,
-                    )
-            self.callback_error = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        1,
-                        ],
-                    queue=callback_queue,
-                    )       
-
-

apps/cic-eth/cic_eth/check/gas.py

@@ -21,7 +21,7 @@ def health(*args, **kwargs):
     session = SessionBase.create_session()
 
     config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
     logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
 
     try:

apps/cic-eth/cic_eth/check/signer.py

@@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__)
 def health(*args, **kwargs):
     blocked = True
     max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer')
+    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
     for i in range(max_attempts):
         idx = i + 1
         logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))

apps/cic-eth/cic_eth/cli/__init__.py

@@ -1,10 +0,0 @@
-# local imports
-from .base import *
-from .chain import (
-        EthChainInterface,
-        chain_interface,
-        )
-from .rpc import RPC
-from .arg import ArgumentParser
-from .config import Config
-from .celery import CeleryApp

apps/cic-eth/cic_eth/cli/arg.py

@@ -1,31 +0,0 @@
-# external imports
-from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
-
-# local imports
-from .base import (
-        CICFlag,
-        Flag,
-        )
-
-
-class ArgumentParser(BaseArgumentParser):
-
-    def process_local_flags(self, local_arg_flags):
-        if local_arg_flags & CICFlag.REDIS:
-            self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
-            self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
-            self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
-            self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
-        if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
-        if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
-            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-        if local_arg_flags & CICFlag.CHAIN:
-            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
-
-
-

apps/cic-eth/cic_eth/cli/base.py

@@ -1,31 +0,0 @@
-# standard imports
-import enum
-
-# external imports
-from chainlib.eth.cli import (
-    argflag_std_read,
-    argflag_std_write,
-    argflag_std_base,
-    Flag,
-    )
-
-class CICFlag(enum.IntEnum):
-   
-    # celery - nibble 1
-    CELERY = 1
-
-    # redis - nibble 2
-    REDIS = 16
-    REDIS_CALLBACK = 32
-
-    # chain - nibble 3
-    CHAIN = 256
-
-    # sync - nibble 4
-    SYNCER = 4096
-
-
-argflag_local_task = CICFlag.CELERY 
-argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
-argflag_local_chain = CICFlag.CHAIN
-argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/celery.py

@@ -1,24 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-logg = logging.getLogger(__name__)
-
-
-class CeleryApp:
-   
-    @classmethod
-    def from_config(cls, config):
-        backend_url = config.get('CELERY_RESULT_URL')
-        broker_url = config.get('CELERY_BROKER_URL')
-        celery_app = None
-        if backend_url != None:
-            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
-            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
-        else:
-            celery_app = celery.Celery(broker=broker_url)
-            logg.info('creating celery app without results backend on {}'.format(broker_url))
-
-        return celery_app

apps/cic-eth/cic_eth/cli/chain.py

@@ -1,21 +0,0 @@
-# external imports
-from chainlib.eth.block import (
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        Tx,
-        )
-from chainlib.interface import ChainInterface
-
-
-class EthChainInterface(ChainInterface):
-    
-    def __init__(self):
-        self._tx_receipt = receipt
-        self._block_by_number = block_by_number
-        self._block_from_src = Block.from_src
-        self._src_normalize = Tx.src_normalize
-
-chain_interface = EthChainInterface()

apps/cic-eth/cic_eth/cli/config.py

@@ -1,63 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.cli import (
-        Config as BaseConfig,
-        Flag,
-        )
-
-# local imports
-from .base import CICFlag
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-
-logg = logging.getLogger(__name__)
-
-
-class Config(BaseConfig):
-
-    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
-
-    @classmethod
-    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
-        expanded_base_config_dir = [cls.local_base_config_dir]
-        if base_config_dir != None:
-            if isinstance(base_config_dir, str):
-                base_config_dir = [base_config_dir]
-            for d in base_config_dir:
-                expanded_base_config_dir.append(d)
-        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
-
-        local_args_override = {}
-        if local_arg_flags & CICFlag.REDIS:
-            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
-            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
-            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
-            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
-
-        if local_arg_flags & CICFlag.CHAIN:
-            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
-
-        if local_arg_flags & CICFlag.CELERY:
-            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
-
-        if local_arg_flags & CICFlag.SYNCER:
-            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
-            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
-
-        config.dict_override(local_args_override, 'local cli args')
-
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
-        if local_arg_flags & CICFlag.CELERY:
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        logg.debug('config loaded:\n{}'.format(config))
-
-        return config
-
-

apps/cic-eth/cic_eth/cli/rpc.py

@@ -1,86 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import (
-        EthUnixSignerConnection,
-        EthHTTPSignerConnection,
-        )
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-
-class RPC:
-
-    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
-        self.chain_spec = chain_spec
-        self.rpc_provider = rpc_provider
-        self.signer_provider = signer_provider
-
-
-    def get_default(self):
-        return RPCConnection.connect(self.chain_spec, 'default')
-
-
-    @staticmethod
-    def from_config(config, use_signer=False):
-        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
-        if use_signer:
-
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
-        logg.info('set up rpc: {}'.format(rpc))
-        return rpc
-
-
-    def __str__(self):
-        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
-
-
-# TOOD: re-implement file backend option from omittec code:
-#broker = config.get('CELERY_BROKER_URL')
-#if broker[:4] == 'file':
-#    bq = tempfile.mkdtemp()
-#    bp = tempfile.mkdtemp()
-#    conf_update = {
-#            'broker_url': broker,
-#            'broker_transport_options': {
-#                'data_folder_in': bq,
-#                'data_folder_out': bq,
-#                'data_folder_processed': bp,
-#            },
-#            }
-#    if config.true('CELERY_DEBUG'):
-#        conf_update['result_extended'] = True
-#    current_app.conf.update(conf_update)
-#    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
-#else:
-#    conf_update = {
-#            'broker_url': broker,
-#            }
-#    if config.true('CELERY_DEBUG'):
-#        conf_update['result_extended'] = True
-#    current_app.conf.update(conf_update)
-#
-#result = config.get('CELERY_RESULT_URL')
-#if result[:4] == 'file':
-#    rq = tempfile.mkdtemp()
-#    current_app.conf.update({
-#        'result_backend': 'file://{}'.format(rq),
-#        })
-#    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
-#else:
-#    current_app.conf.update({
-#        'result_backend': result,
-#        })
-#

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -1,5 +0,0 @@
-[celery]
-broker_url = redis://localhost:6379
-result_url = 
-queue = cic-eth
-debug = 0

apps/cic-eth/cic_eth/data/config/database.ini

@@ -1,10 +0,0 @@
-[database]
-engine =
-driver =
-host =
-port =
-name =
-user =
-password =
-debug = 0
-pool_size = 0

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-gas_gifter_minimum_balance = 10000000000000000000

apps/cic-eth/cic_eth/data/config/signer.ini

@@ -1,2 +0,0 @@
-[signer]
-provider =

apps/cic-eth/cic_eth/data/config/syncer.ini

@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0

apps/cic-eth/cic_eth/db/enum.py

@@ -1 +1,158 @@
-from cic_eth.enum import *
+# standard imports
+import enum
+
+
+@enum.unique
+class StatusBits(enum.IntEnum):
+    """Individual bit flags that are combined to define the state and legacy of a queued transaction
+
+    """
+    QUEUED = 0x01 # transaction should be sent to network
+    IN_NETWORK = 0x08 # transaction is in network
+   
+    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
+    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
+
+    LOCAL_ERROR = 0x100 # errors that originate internally from the component
+    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
+    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
+    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
+
+    FINAL = 0x1000 # transaction processing has completed
+    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
+    MANUAL = 0x8000 # transaction processing has been manually overridden
+
+
+@enum.unique
+class StatusEnum(enum.IntEnum):
+    """
+
+    - Inactive, not finalized. (<0)
+        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
+        * SENDFAIL: The transaction was not received by the node.
+        * RETRY: The transaction is queued for a new send attempt after previously failing.
+        * READYSEND: The transaction is queued for its first send attempt
+        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
+        * WAITFORGAS: The transaction is on hold pending gas funding.
+    - Active state: (==0)
+        * SENT: The transaction has been sent to the mempool.
+    - Inactive, finalized. (>0)
+        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
+        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
+        * OVERRIDDEN: Transaction has been manually overriden.
+        * REJECTED: The transaction was rejected by the node.
+        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
+        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
+
+    """
+    PENDING = 0
+
+    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
+    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
+    READYSEND = StatusBits.QUEUED
+
+    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
+
+    WAITFORGAS = StatusBits.GAS_ISSUES
+
+    SENT = StatusBits.IN_NETWORK
+    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
+    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
+    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
+
+    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
+    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
+    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
+
+
+@enum.unique
+class LockEnum(enum.IntEnum):
+    """
+    STICKY: When set, reset is not possible
+    CREATE: Disable creation of accounts
+    SEND: Disable sending to network
+    QUEUE: Disable queueing new or modified transactions
+    """
+    STICKY=1
+    INIT=2
+    CREATE=4
+    SEND=8
+    QUEUE=16
+    QUERY=32
+    ALL=int(0xfffffffffffffffe)
+
+
+def status_str(v, bits_only=False):
+    """Render a human-readable string describing the status
+
+    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
+
+    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
+
+    :param v: Status bit field
+    :type v: number
+    :param bits_only: Only render individual bit labels.
+    :type bits_only: bool
+    :returns: Status string
+    :rtype: str
+    """
+    s = ''
+    if not bits_only:
+        try:
+            s = StatusEnum(v).name
+            return s
+        except ValueError:
+            pass
+
+    if v == 0:
+        return 'NONE'
+
+    for i in range(16):
+        b = (1 << i)
+        if (b & 0xffff) & v:
+            n = StatusBits(b).name
+            if len(s) > 0:
+                s += ','
+            s += n
+    if not bits_only:
+        s += '*'
+    return s
+
+
+def all_errors():
+    """Bit mask of all error states
+
+    :returns: Error flags
+    :rtype: number
+    """
+    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
+
+
+def is_error_status(v):
+    """Check if value is an error state
+
+    :param v: Status bit field
+    :type v: number
+    :returns: True if error
+    :rtype: bool
+    """
+    return bool(v & all_errors())
+
+
+def dead():
+    """Bit mask defining whether a transaction is still likely to be processed on the network.
+
+    :returns: Bit mask
+    :rtype: number
+    """
+    return StatusBits.FINAL | StatusBits.OBSOLETE
+
+
+def is_alive(v):
+    """Check if transaction is still likely to be processed on the network.
+
+    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
+
+    :returns: 
+    """
+    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py

@@ -8,8 +8,7 @@ Create Date: 2021-04-02 18:30:55.398388
 from alembic import op
 import sqlalchemy as sa
 
-#from chainqueue.db.migrations.sqlalchemy import (
-from chainqueue.db.migrations.default.export import (
+from chainqueue.db.migrations.sqlalchemy import (
         chainqueue_upgrade,
         chainqueue_downgrade,
         )

apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py

@@ -8,8 +8,7 @@ Create Date: 2021-04-02 18:36:44.459603
 from alembic import op
 import sqlalchemy as sa
 
-#from chainsyncer.db.migrations.sqlalchemy import (
-from chainsyncer.db.migrations.default.export import (
+from chainsyncer.db.migrations.sqlalchemy import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-eth/cic_eth/db/models/base.py

@@ -126,4 +126,3 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/enum.py

@@ -1,158 +0,0 @@
-# standard imports
-import enum
-
-
-@enum.unique
-class StatusBits(enum.IntEnum):
-    """Individual bit flags that are combined to define the state and legacy of a queued transaction
-
-    """
-    QUEUED = 0x01 # transaction should be sent to network
-    IN_NETWORK = 0x08 # transaction is in network
-   
-    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
-    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
-
-    LOCAL_ERROR = 0x100 # errors that originate internally from the component
-    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
-    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
-    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
-
-    FINAL = 0x1000 # transaction processing has completed
-    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
-    MANUAL = 0x8000 # transaction processing has been manually overridden
-
-
-@enum.unique
-class StatusEnum(enum.IntEnum):
-    """
-
-    - Inactive, not finalized. (<0)
-        * PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
-        * SENDFAIL: The transaction was not received by the node.
-        * RETRY: The transaction is queued for a new send attempt after previously failing.
-        * READYSEND: The transaction is queued for its first send attempt
-        * OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
-        * WAITFORGAS: The transaction is on hold pending gas funding.
-    - Active state: (==0)
-        * SENT: The transaction has been sent to the mempool.
-    - Inactive, finalized. (>0)
-        * FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
-        * CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
-        * OVERRIDDEN: Transaction has been manually overriden.
-        * REJECTED: The transaction was rejected by the node.
-        * REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
-        * SUCCESS: THe transaction was successfully mined. (Block number will be set)
-
-    """
-    PENDING = 0
-
-    SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
-    RETRY = StatusBits.QUEUED | StatusBits.DEFERRED 
-    READYSEND = StatusBits.QUEUED
-
-    OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
-
-    WAITFORGAS = StatusBits.GAS_ISSUES
-
-    SENT = StatusBits.IN_NETWORK
-    FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
-    CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
-    OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
-
-    REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
-    REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
-    SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL 
-
-
-@enum.unique
-class LockEnum(enum.IntEnum):
-    """
-    STICKY: When set, reset is not possible
-    CREATE: Disable creation of accounts
-    SEND: Disable sending to network
-    QUEUE: Disable queueing new or modified transactions
-    """
-    STICKY=1
-    INIT=2
-    CREATE=4
-    SEND=8
-    QUEUE=16
-    QUERY=32
-    ALL=int(0xfffffffffffffffe)
-
-
-def status_str(v, bits_only=False):
-    """Render a human-readable string describing the status
-
-    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
-
-    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
-
-    :param v: Status bit field
-    :type v: number
-    :param bits_only: Only render individual bit labels.
-    :type bits_only: bool
-    :returns: Status string
-    :rtype: str
-    """
-    s = ''
-    if not bits_only:
-        try:
-            s = StatusEnum(v).name
-            return s
-        except ValueError:
-            pass
-
-    if v == 0:
-        return 'NONE'
-
-    for i in range(16):
-        b = (1 << i)
-        if (b & 0xffff) & v:
-            n = StatusBits(b).name
-            if len(s) > 0:
-                s += ','
-            s += n
-    if not bits_only:
-        s += '*'
-    return s
-
-
-def all_errors():
-    """Bit mask of all error states
-
-    :returns: Error flags
-    :rtype: number
-    """
-    return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
-
-
-def is_error_status(v):
-    """Check if value is an error state
-
-    :param v: Status bit field
-    :type v: number
-    :returns: True if error
-    :rtype: bool
-    """
-    return bool(v & all_errors())
-
-
-def dead():
-    """Bit mask defining whether a transaction is still likely to be processed on the network.
-
-    :returns: Bit mask
-    :rtype: number
-    """
-    return StatusBits.FINAL | StatusBits.OBSOLETE
-
-
-def is_alive(v):
-    """Check if transaction is still likely to be processed on the network.
-
-    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
-
-    :returns: 
-    """
-    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/error.py

@@ -80,8 +80,3 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
-
-
-class YouAreBrokeError(Exception):
-    """Exception raised when a value transfer is attempted without access to sufficient funds
-    """

apps/cic-eth/cic_eth/eth/account.py

@@ -23,7 +23,7 @@ from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
 from eth_accounts_index import AccountsIndex 
 from sarafu_faucet import MinterFaucet
-from chainqueue.sql.tx import cache_tx_dict
+from chainqueue.db.models.tx import TxCache
 
 # local import
 from cic_eth_registry import CICRegistry
@@ -300,17 +300,20 @@ def cache_gift_data(
 
     session = self.create_session()
 
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
-            'from_value': 0,
-            'to_value': 0,
-            }
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['to'],
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
+        0,
+        0,
+        session=session,
+            )
 
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -339,15 +342,18 @@ def cache_account_data(
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
-            'from_value': 0,
-            'to_value': 0,
-            }
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['to'],
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
+        0,
+        0,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/bancor.py.bak

@@ -0,0 +1,385 @@
+# standard imports
+import os
+import logging
+
+# third-party imports
+import celery
+import web3
+from cic_registry import CICRegistry
+from cic_registry.chain import ChainSpec
+
+# local imports
+from cic_eth.db import SessionBase
+from cic_eth.db.models.convert import TxConvertTransfer
+from cic_eth.db.models.otx import Otx
+from cic_eth.db.models.tx import TxCache
+from cic_eth.eth.task import sign_and_register_tx
+from cic_eth.eth.task import create_check_gas_and_send_task
+from cic_eth.eth.token import TokenTxFactory
+from cic_eth.eth.factory import TxFactory
+from cic_eth.eth.util import unpack_signed_raw_tx
+from cic_eth.eth.rpc import RpcClient
+
+celery_app = celery.current_app 
+#logg = celery_app.log.get_default_logger()
+logg = logging.getLogger()
+
+contract_function_signatures = {
+        'convert': 'f3898a97',
+        'convert2': '569706eb',
+        }
+
+
+class BancorTxFactory(TxFactory):
+
+    """Factory for creating Bancor network transactions.
+    """
+    def convert(
+            self,
+            source_token_address,
+            destination_token_address,
+            reserve_address,
+            source_amount,
+            minimum_return,
+            chain_spec,
+            fee_beneficiary='0x0000000000000000000000000000000000000000',
+            fee_ppm=0,
+            ):
+        """Create a BancorNetwork "convert" transaction.
+
+        :param source_token_address: ERC20 contract address for token to convert from
+        :type source_token_address: str, 0x-hex
+        :param destination_token_address: ERC20 contract address for token to convert to
+        :type destination_token_address: str, 0x-hex
+        :param reserve_address: ERC20 contract address of Common reserve token
+        :type reserve_address: str, 0x-hex
+        :param source_amount: Amount of source tokens to convert
+        :type source_amount: int
+        :param minimum_return: Minimum amount of destination tokens to accept as result for conversion
+        :type source_amount: int
+        :return: Unsigned "convert" transaction in standard Ethereum format
+        :rtype: dict
+        """
+        network_contract = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
+        network_gas = network_contract.gas('convert')
+        tx_convert_buildable = network_contract.contract.functions.convert2(
+            [
+                source_token_address,
+                source_token_address,
+                reserve_address,
+                destination_token_address,
+                destination_token_address,
+            ],
+            source_amount,
+            minimum_return,
+            fee_beneficiary,
+            fee_ppm,
+            )
+        tx_convert = tx_convert_buildable.buildTransaction({
+                'from': self.address,
+                'gas': network_gas,
+                'gasPrice': self.gas_price,
+                'chainId': chain_spec.chain_id(),
+                'nonce': self.next_nonce(),
+                })
+        return tx_convert
+
+
+def unpack_convert(data):
+    f = data[2:10]
+    if f != contract_function_signatures['convert2']:
+        raise ValueError('Invalid convert data ({})'.format(f))
+
+    d = data[10:]
+    path = d[384:]
+    source = path[64-40:64]
+    destination = path[-40:]
+
+    amount = int(d[64:128], 16)
+    min_return = int(d[128:192], 16)
+    fee_recipient = d[192:256]
+    fee = int(d[256:320], 16)
+    return {
+        'amount': amount,
+        'min_return': min_return,
+        'source_token': web3.Web3.toChecksumAddress('0x' + source),
+        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
+        'fee_recipient': fee_recipient,
+        'fee': fee,
+        }
+
+
+
+# Kept for historical reference, it unpacks a convert call without fee parameters
+#def _unpack_convert_mint(data):
+#    f = data[2:10]
+#    if f != contract_function_signatures['convert2']:
+#        raise ValueError('Invalid convert data ({})'.format(f))
+#
+#    d = data[10:]
+#    path = d[256:]
+#    source = path[64-40:64]
+#    destination = path[-40:]
+#
+#    amount = int(d[64:128], 16)
+#    min_return = int(d[128:192], 16)
+#    return {
+#        'amount': amount,
+#        'min_return': min_return,
+#        'source_token': web3.Web3.toChecksumAddress('0x' + source),
+#        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
+#        }
+
+
+@celery_app.task(bind=True)
+def convert_with_default_reserve(self, tokens, from_address, source_amount, minimum_return, to_address, chain_str):
+    """Performs a conversion between two liquid tokens using Bancor network.
+
+    :param tokens: Token pair, source and destination respectively
+    :type tokens: list of str, 0x-hex
+    :param from_address: Ethereum address of sender
+    :type from_address: str, 0x-hex
+    :param source_amount: Amount of source tokens to convert
+    :type source_amount: int
+    :param minimum_return: Minimum about of destination tokens to receive
+    :type minimum_return: int
+    """
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    queue = self.request.delivery_info['routing_key']
+
+    c = RpcClient(chain_spec, holder_address=from_address)
+
+    cr = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
+    source_token = CICRegistry.get_address(chain_spec, tokens[0]['address'])
+    reserve_address = CICRegistry.get_contract(chain_spec, 'BNTToken', 'ERC20').address()
+
+    tx_factory = TokenTxFactory(from_address, c)
+   
+    tx_approve_zero = tx_factory.approve(source_token.address(), cr.address(), 0, chain_spec)
+    (tx_approve_zero_hash_hex, tx_approve_zero_signed_hex) = sign_and_register_tx(tx_approve_zero, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
+
+    tx_approve = tx_factory.approve(source_token.address(), cr.address(), source_amount, chain_spec)
+    (tx_approve_hash_hex, tx_approve_signed_hex) = sign_and_register_tx(tx_approve, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
+
+    tx_factory = BancorTxFactory(from_address, c)
+    tx_convert = tx_factory.convert(
+            tokens[0]['address'],
+            tokens[1]['address'],
+            reserve_address,
+            source_amount,
+            minimum_return,
+            chain_spec,
+            )
+    (tx_convert_hash_hex, tx_convert_signed_hex) = sign_and_register_tx(tx_convert, chain_str, queue, 'cic_eth.eth.bancor.otx_cache_convert')
+
+    # TODO: consider moving save recipient to async task / chain it before the tx send
+    if to_address != None:
+        save_convert_recipient(tx_convert_hash_hex, to_address, chain_str)
+
+    s = create_check_gas_and_send_task(
+            [tx_approve_zero_signed_hex, tx_approve_signed_hex, tx_convert_signed_hex],
+            chain_str,
+            from_address,
+            tx_approve_zero['gasPrice'] * tx_approve_zero['gas'],
+            tx_hashes_hex=[tx_approve_hash_hex],
+            queue=queue,
+            )
+    s.apply_async()
+    return tx_convert_hash_hex
+
+
+#@celery_app.task()
+#def process_approval(tx_hash_hex):
+#    t = session.query(TxConvertTransfer).query(TxConvertTransfer.approve_tx_hash==tx_hash_hex).first()
+#    c = session.query(Otx).query(Otx.tx_hash==t.convert_tx_hash)
+#    gas_limit = 8000000
+#    gas_price = GasOracle.gas_price()
+#
+#    # TODO: use celery group instead
+#    s_queue = celery.signature(
+#            'cic_eth.queue.tx.create',
+#            [
+#                nonce,
+#                c['address'], # TODO: check that this is in fact sender address
+#                c['tx_hash'],
+#                c['signed_tx'],
+#                ]
+#            )
+#    s_queue.apply_async()
+#
+#    s_check_gas = celery.signature(
+#            'cic_eth.eth.gas.check_gas',
+#            [
+#                c['address'],
+#                [c['signed_tx']],
+#                gas_limit * gas_price,
+#                ]
+#            )
+#    s_send = celery.signature(
+#            'cic_eth.eth.tx.send',
+#            [],
+#            )
+#
+#    s_set_sent = celery.signature(
+#            'cic_eth.queue.state.set_sent',
+#            [False],
+#            )
+#    s_send.link(s_set_sent)
+#    s_check_gas.link(s_send)
+#    s_check_gas.apply_async()
+#    return tx_hash_hex
+
+
+
+@celery_app.task()
+def save_convert_recipient(convert_hash, recipient_address, chain_str):
+    """Registers the recipient target for a convert-and-transfer operation.
+
+    :param convert_hash: Transaction hash of convert operation
+    :type convert_hash: str, 0x-hex
+    :param recipient_address: Address of consequtive transfer recipient
+    :type recipient_address: str, 0x-hex
+    """
+    session = SessionBase.create_session()
+    t = TxConvertTransfer(convert_hash, recipient_address, chain_str)
+    session.add(t)
+    session.commit()
+    session.close()
+
+
+@celery_app.task()
+def save_convert_transfer(convert_hash, transfer_hash):
+    """Registers that the transfer part of a convert-and-transfer operation has been executed.
+
+    :param convert_hash: Transaction hash of convert operation
+    :type convert_hash: str, 0x-hex
+    :param convert_hash: Transaction hash of transfer operation
+    :type convert_hash: str, 0x-hex
+    :returns: transfer_hash,
+    :rtype: list, single str, 0x-hex
+    """
+    session = SessionBase.create_session()
+    t = TxConvertTransfer.get(convert_hash)
+    t.transfer(transfer_hash)
+    session.add(t)
+    session.commit()
+    session.close()
+    return [transfer_hash]
+
+
+# TODO: seems unused, consider removing
+@celery_app.task()
+def resolve_converters_by_tokens(tokens, chain_str):
+    """Return converters for a list of tokens.
+
+    :param tokens: Token addresses to look up
+    :type tokens: list of str, 0x-hex
+    :return: Addresses of matching converters
+    :rtype: list of str, 0x-hex
+    """
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    for t in tokens:
+        c = CICRegistry.get_contract(chain_spec, 'ConverterRegistry')
+        fn = c.function('getConvertersByAnchors')
+        try:
+            converters = fn([t['address']]).call()
+        except Exception as e:
+            raise e
+        t['converters'] = converters
+
+    return tokens
+
+
+@celery_app.task(bind=True)
+def transfer_converted(self, tokens, holder_address, receiver_address, value, tx_convert_hash_hex, chain_str):
+    """Execute the ERC20 transfer of a convert-and-transfer operation.
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param holder_address: Token receiver address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    if len(tokens) != 1:
+        raise TokenCountError
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+
+    queue = self.request.delivery_info['routing_key']
+
+    c = RpcClient(chain_spec, holder_address=holder_address)
+
+    # get transaction parameters
+    gas_price = c.gas_price()
+    tx_factory = TokenTxFactory(holder_address, c)
+
+    token_address = tokens[0]['address']
+    tx_transfer = tx_factory.transfer(
+        token_address,
+        receiver_address,
+        value,
+        chain_spec,
+            )
+    (tx_transfer_hash_hex, tx_transfer_signed_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.token.otx_cache_transfer')
+
+    # send transaction
+    logg.info('transfer converted token {} from {} to {} value {} {}'.format(token_address, holder_address, receiver_address, value, tx_transfer_signed_hex))
+    s = create_check_gas_and_send_task(
+            [tx_transfer_signed_hex],
+            chain_str,
+            holder_address,
+            tx_transfer['gasPrice'] * tx_transfer['gas'],
+            None,
+            queue,
+            )
+    s_save = celery.signature(
+            'cic_eth.eth.bancor.save_convert_transfer',
+            [
+                tx_convert_hash_hex,
+                tx_transfer_hash_hex,
+                ],
+            queue=queue,
+            )
+    s_save.link(s)
+    s_save.apply_async()
+    return tx_transfer_hash_hex
+
+
+@celery_app.task()
+def otx_cache_convert(
+        tx_hash_hex,
+        tx_signed_raw_hex,
+        chain_str,
+        ):
+
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx_data = unpack_convert(tx['data'])
+    logg.debug('tx data {}'.format(tx_data))
+
+    session = TxCache.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['from'],
+        tx_data['source_token'],
+        tx_data['destination_token'],
+        tx_data['amount'],
+        tx_data['amount'],
+            )
+    session.add(tx_cache)
+    session.commit()
+    session.close()
+    return tx_hash_hex
+

apps/cic-eth/cic_eth/eth/erc20.py

@@ -13,9 +13,9 @@ from chainlib.eth.tx import (
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
+from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
-from chainqueue.sql.tx import cache_tx_dict
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -24,7 +24,6 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
-        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -72,117 +71,6 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
-@celery_app.task(bind=True)
-def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
-    """Best-effort verification that the allowance for a transfer from spend is sufficient.
-
-    :raises YouAreBrokeError: If allowance is insufficient
-    
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :return: Token list as passed to task
-    :rtype: dict
-    """
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    caller_address = ERC20Token.caller_address 
-    c = ERC20(chain_spec)
-    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
-    r = rpc.do(o)
-    allowance = c.parse_allowance(r)
-    if allowance < value:
-        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
-        logg.error(errstr)
-        raise YouAreBrokeError(errstr)
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
-def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
-    """Transfer ERC20 tokens between addresses
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Token receiver address
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param spender_address: Address of account spending on behalf of holder
-    :type spender_address: str, 0x-hex
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    :return: Transaction hash for tranfer operation
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    logg.debug('tokens {}'.format(tokens))
-    if len(tokens) != 1:
-        raise TokenCountError
-    t = tokens[0]
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    queue = self.request.delivery_info.get('routing_key')
-
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
-
-    session = self.create_session()
-    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
-    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
-
-    rpc_signer.disconnect()
-    rpc.disconnect()
-
-    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
-
-    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
-    session.commit()
-    session.close()
-    
-    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
-    gas_budget = gas_pair[0] * gas_pair[1]
-    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
-
-    s = create_check_gas_task(
-             [tx_signed_raw_hex],
-             chain_spec,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return tx_hash_hex
-
-
-
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -344,7 +232,6 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
-            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -375,55 +262,19 @@ def cache_transfer_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
-            'from_value': token_value,
-            'to_value': token_value,
-            }
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
-    session.close()
-    return (tx_hash_hex, cache_id)
-
-
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def cache_transfer_from_data(
-    tx_hash_hex,
-    tx_signed_raw_hex,
-    chain_spec_dict,
-        ):
-    """Helper function for otx_cache_transfer_from
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
-
-    tx_data = ERC20.parse_transfer_from_request(tx['data'])
-    spender_address = tx_data[0]
-    recipient_address = tx_data[1]
-    token_value = tx_data[2]
-
-    session = SessionBase.create_session()
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
-            'from_value': token_value,
-            'to_value': token_value,
-            }
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -452,16 +303,19 @@ def cache_approve_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': recipient_address,
-            'source_token': tx['to'],
-            'destination_token': tx['to'],
-            'from_value': token_value,
-            'to_value': token_value,
-            }
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -9,7 +9,6 @@ from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
-from chainqueue.sql.tx import cache_tx_dict
 from chainlib.eth.gas import (
         balance,
         price,
@@ -134,17 +133,20 @@ def cache_gas_data(
 
     session = SessionBase.create_session()
 
-    tx_dict = {
-            'hash': tx_hash_hex,
-            'from': tx['from'],
-            'to': tx['to'],
-            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS,
-            'from_value': tx['value'],
-            'to_value': tx['value'],
-            }
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        tx['to'],
+        ZERO_ADDRESS,
+        ZERO_ADDRESS,
+        tx['value'],
+        tx['value'],
+        session=session,
+            )
 
-    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/tx.py

@@ -18,6 +18,7 @@ from hexathon import (
         strip_0x,
         )
 from chainqueue.db.models.tx import Otx
+from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
 from potaahto.symbols import snake_and_camel

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,11 +14,9 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
-from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
@@ -116,6 +114,9 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
+                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token_symbol = token.symbol
+                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -125,6 +126,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
+                            'source_token_symbol': token_symbol,
+                            'destination_token_symbol': token_symbol,
+                            'source_token_decimals': token_decimals,
+                            'destination_token_decimals': token_decimals,
+                            'source_token_chain': chain_str,
+                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -139,8 +146,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task(bind=True, base=BaseTask)
-def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
+@celery_app.task()
+def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -189,32 +196,6 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
     if newest_first:
         ks.reverse()
     for k in ks:
-        tx = txs_by_block[k]
-        if verify_contracts:
-            try:
-                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
-            except UnknownContractError:
-                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
-                continue
-        txs.append(tx)
+        txs.append(txs_by_block[k])
 
     return txs
-
-
-def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    registry = CICRegistry(chain_spec, rpc)
-
-    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['source_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['source_token'])
-        tx['source_token_symbol'] = token.symbol
-        tx['source_token_decimals'] = token.decimals
-
-    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
-        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
-        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
-        tx['destination_token_symbol'] = token.symbol
-        tx['destination_token_decimals'] = token.decimals
-
-    return tx

apps/cic-eth/cic_eth/pytest/fixtures_token.py

@@ -1,19 +0,0 @@
-# external imports
-import pytest
-from eth_erc20 import ERC20
-
-# TODO: missing dep fixture includes
-
-
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)

apps/cic-eth/cic_eth/queue/query.py

@@ -100,7 +100,6 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     q_outer = q_outer.join(Lock, isouter=True)
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
-
     if not is_alive(status):
         SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))

apps/cic-eth/cic_eth/runnable/create.py

@@ -7,30 +7,54 @@ import json
 import argparse
 
 # external imports
+import celery
+import confini
 import redis
 from xdg.BaseDirectory import xdg_config_home
-from chainlib.chain import ChainSpec
 
 # local imports
-import cic_eth.cli
 from cic_eth.api import Api
 
 logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
+logg = logging.getLogger('create_account_script')
+logging.getLogger('confini').setLevel(logging.WARNING)
+logging.getLogger('gnupg').setLevel(logging.WARNING)
 
-arg_flags = cic_eth.cli.argflag_std_base
-local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser()
 argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
-argparser.process_local_flags(local_arg_flags)
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
 args = argparser.parse_args()
 
-extra_args = {
-    'no_register': None,
-        }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
+if args.vv:
+    logg.setLevel(logging.DEBUG)
+if args.v:
+    logg.setLevel(logging.INFO)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        }
+config.dict_override(args_override, 'cli')
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
 
 def main():
     redis_host = config.get('REDIS_HOST')
@@ -44,24 +68,19 @@ def main():
     ps.get_message()
 
     api = Api(
-            config.get('CHAIN_SPEC'),
-            queue=config.get('CELERY_QUEUE'),
-            callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel),
+            config.get('CIC_CHAIN_SPEC'),
+            queue=args.q,
+            callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
             callback_task='cic_eth.callbacks.redis.redis',
-            callback_queue=config.get('CELERY_QUEUE'),
+            callback_queue=args.q,
             )
 
-    register = not config.get('_NO_REGISTER')
+    register = not args.no_register
     logg.debug('register {}'.format(register))
     t = api.create_account(register=register)
 
     ps.get_message()
-    try:
-        o = ps.get_message(timeout=config.get('REDIS_TIMEOUT'))
-    except TimeoutError as e:
-        sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e))
-        sys.exit(1) 
-    ps.unsubscribe()
+    o = ps.get_message(timeout=args.timeout)
     m = json.loads(o['data'])
     print(m['result'])
 

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -12,38 +12,64 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.address import is_checksum_address
 
 # local imports
-import cic_eth.cli
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import AdminApi
 from cic_eth.db.enum import LockEnum
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
-arg_flags = cic_eth.cli.argflag_std_read
-local_arg_flags = cic_eth.cli.argflag_local_task | cic_eth.cli.argflag_local_chain
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
-argparser.process_local_flags(local_arg_flags)
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
 
 def process_lock_args(argparser):
     argparser.add_argument('flags', type=str, help='Flags to manipulate')
     argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,')
 
-sub = argparser.add_subparsers(help='')
+sub = argparser.add_subparsers()
 sub.dest = "command"
 sub_lock = sub.add_parser('lock', help='Set or reset locks')
 sub_unlock = sub.add_parser('unlock', help='Set or reset locks')
 process_lock_args(sub_lock) 
 process_lock_args(sub_unlock) 
+
 args = argparser.parse_args()
 
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
 
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+args_override = {
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+# override args
+config.dict_override(args_override, 'cli')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
+queue = args.q
+
+chain_spec = None
+if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)
 
 
@@ -74,7 +100,7 @@ def main():
                 args.address,
                 flags,
                 ],
-            queue=config.get('CELERY_QUEUE'),
+            queue=queue,
             )
         t = s.apply_async()
         logg.debug('unlock {} on {} task {}'.format(flags, args.address, t))
@@ -93,7 +119,7 @@ def main():
                 args.address,
                 flags,
                 ],
-            queue=config.get('CELERY_QUEUE'),
+            queue=queue,
             )
         t = s.apply_async()
         logg.debug('lock {} on {} task {}'.format(flags, args.address, t))

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -8,7 +8,8 @@ import sys
 import re
 import datetime
 
-# external imports
+# third-party imports
+import confini
 import celery
 from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
@@ -23,7 +24,7 @@ from chainqueue.error import NotLocalTxError
 from chainqueue.sql.state import set_reserved
 
 # local imports
-import cic_eth.cli
+import cic_eth
 from cic_eth.db import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.db import dsn_from_config
@@ -38,37 +39,57 @@ from cic_eth.error import (
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_read 
-local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
 
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-eth')
 
-# connect to celery
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+queue = args.q
 
-# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-# set up rpc
-rpc = cic_eth.cli.RPC.from_config(config)
-conn = rpc.get_default()
+RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
 
 run = True
 
-
 class DispatchSyncer:
 
     yield_delay = 0.0005
 
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
-        self.session = None
 
 
     def chain(self):
@@ -79,18 +100,16 @@ class DispatchSyncer:
         c = len(txs.keys())
         logg.debug('processing {} txs {}'.format(c, list(txs.keys())))
         chain_str = str(self.chain_spec)
-        self.session = SessionBase.create_session()
+        session = SessionBase.create_session()
         for k in txs.keys():
             tx_raw = txs[k]
             tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
             tx = unpack(tx_raw_bytes, self.chain_spec)
             
             try:
-                set_reserved(self.chain_spec, tx['hash'], session=self.session)
-                self.session.commit()
+                set_reserved(self.chain_spec, tx['hash'], session=session)
             except NotLocalTxError as e:
                 logg.warning('dispatcher was triggered with non-local tx {}'.format(tx['hash']))
-                self.session.rollback()
                 continue
 
             s_check = celery.signature(
@@ -113,25 +132,16 @@ class DispatchSyncer:
             s_check.link(s_send)
             t = s_check.apply_async()
             logg.info('processed {}'.format(k))
-        self.session.close()
-        self.session = None
 
 
-    def loop(self, interval):
+    def loop(self, w3, interval):
         while run:
             txs = {}
             typ = StatusBits.QUEUED
             utxs = get_upcoming_tx(self.chain_spec, typ)
             for k in utxs.keys():
                 txs[k] = utxs[k]
-            try:
-                conn = RPCConnection.connect(self.chain_spec, 'default')
-                self.process(conn, txs)
-            except ConnectionError as e:
-                if self.session != None:
-                    self.session.close()
-                    self.session = None
-                logg.error('connection to node failed: {}'.format(e))
+            self.process(w3, txs)
 
             if len(utxs) > 0:
                 time.sleep(self.yield_delay)
@@ -141,7 +151,8 @@ class DispatchSyncer:
 
 def main(): 
     syncer = DispatchSyncer(chain_spec)
-    syncer.loop(float(config.get('DISPATCHER_LOOP_INTERVAL')))
+    conn = RPCConnection.connect(chain_spec, 'default')
+    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -11,7 +11,6 @@ from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
-from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -48,13 +47,12 @@ class GasFilter(SyncFilter):
 
             SessionBase.release_session(session)
 
-            address = to_checksum_address(r[0])
             logg.info('resuming gas-in-waiting txs for {}'.format(r[0]))
             if len(txs) > 0:
                 s = create_check_gas_task(
                         list(txs.values()),
                         self.chain_spec,
-                        address,
+                        r[0],
                         0,
                         tx_hashes_hex=list(txs.keys()),
                         queue=self.queue,

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -12,24 +12,20 @@ from hexathon import (
 # local imports
 from .base import SyncFilter
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger().getChild(__name__)
 
 account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430'
 
 
 class RegistrationFilter(SyncFilter):
 
-    def __init__(self, chain_spec, contract_address, queue=None):
+    def __init__(self, chain_spec, queue):
         self.chain_spec = chain_spec
         self.queue = queue
-        self.contract_address = contract_address
 
 
     def filter(self, conn, block, tx, db_session=None): 
-        if self.contract_address != tx.inputs[0]:
-            logg.debug('not an account registry tx; {} != {}'.format(self.contract_address, tx.inputs[0]))
-            return None
-
+        registered_address = None
         for l in tx.logs:
             event_topic_hex = l['topics'][0]
             if event_topic_hex == account_registry_add_log_hash:

apps/cic-eth/cic_eth/runnable/daemons/retry.py

@@ -6,6 +6,7 @@ import argparse
 import re
 
 # external imports
+import confini
 import celery
 from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
@@ -13,7 +14,6 @@ from chainlib.connection import RPCConnection
 from chainsyncer.filter import SyncFilter
 
 # local imports
-import cic_eth.cli
 from cic_eth.db import dsn_from_config
 from cic_eth.db import SessionBase
 from cic_eth.admin.ctrl import lock_send
@@ -25,41 +25,66 @@ from cic_eth.stat import init_chain_stat
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_read
-local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--batch-size', dest='batch_size', type=int, default=50, help='max amount of txs to resend per iteration')
-argparser.add_argument('--retry-delay', dest='retry_delay', type=int, default=20, help='seconds to wait for retrying a transaction that is marked as sent')
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
+argparser.add_argument('--retry-delay', dest='retry_delay', type=int, help='seconds to wait for retrying a transaction that is marked as sent')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+args = argparser.parse_args(sys.argv[1:])
 
-extra_args = {
-    'retry_delay': 'RETRY_DELAY',
-    'batch_size': 'RETRY_BATCH_SIZE',
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'),
         }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+config.add(args.batch_size, '_BATCH_SIZE', True)
+
+app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+queue = args.q
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
 
-# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-# set up rpc
-rpc = cic_eth.cli.RPC.from_config(config)
-conn = rpc.get_default()
-
 
 def main(): 
-    straggler_delay = int(config.get('RETRY_DELAY'))
+    conn = RPCConnection.connect(chain_spec, 'default')
+
+    straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
     loop_interval = config.get('SYNCER_LOOP_INTERVAL')
     if loop_interval == None:
         stat = init_chain_stat(conn)
         loop_interval = stat.block_average()
 
-    syncer = RetrySyncer(conn, chain_spec, cic_eth.cli.chain_interface, straggler_delay, batch_size=config.get('RETRY_BATCH_SIZE'))
+    syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE'))
     syncer.backend.set(0, 0)
-    fltr = StragglerFilter(chain_spec, queue=config.get('CELERY_QUEUE'))
+    fltr = StragglerFilter(chain_spec, queue=queue)
     syncer.add_filter(fltr)
     syncer.loop(int(loop_interval), conn)
 

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -0,0 +1,136 @@
+# standard imports
+import os
+import re
+import logging
+import argparse
+import json
+
+# third-party imports
+import web3
+import confini
+import celery
+from json.decoder import JSONDecodeError
+from cic_registry.chain import ChainSpec
+
+# local imports
+from cic_eth.db import dsn_from_config
+from cic_eth.db.models.base import SessionBase
+from cic_eth.eth.util import unpack_signed_raw_tx
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_eth', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
+
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+args = argparser.parse_args()
+
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn)
+
+celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+queue = args.q
+
+re_something = r'^/something/?'
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+
+def process_something(session, env):
+    r = re.match(re_something, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    #if env.get('CONTENT_TYPE') != 'application/json':
+    #    raise AttributeError('content type')
+
+    #if env.get('REQUEST_METHOD') != 'POST':
+    #    raise AttributeError('method')
+
+    #post_data = json.load(env.get('wsgi.input'))
+    
+    #return ('text/plain', 'foo'.encode('utf-8'),)
+
+
+# uwsgi application
+def application(env, start_response):
+    
+    for k in env.keys():
+        logg.debug('env {} {}'.format(k, env[k]))
+
+    headers = []
+    content = b''
+    err = None
+
+    session = SessionBase.create_session()
+    for handler in [
+            process_something,
+            ]:
+        try:
+            r = handler(session, env)
+        except AttributeError as e:
+            logg.error('handler fail attribute {}'.format(e))
+            err = '400 Impertinent request'
+            break
+        except JSONDecodeError as e:
+            logg.error('handler fail json {}'.format(e))
+            err = '400 Invalid data format'
+            break
+        except KeyError as e:
+            logg.error('handler fail key {}'.format(e))
+            err = '400 Invalid JSON'
+            break
+        except ValueError as e:
+            logg.error('handler fail value {}'.format(e))
+            err = '400 Invalid data'
+            break
+        except RuntimeError as e:
+            logg.error('task fail value {}'.format(e))
+            err = '500 Task failed, sorry I cannot tell you more'
+            break
+        if r != None:
+            (mime_type, content) = r
+            break
+    session.close()
+
+    if err != None:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response(err, headers)
+        session.close()
+        return [content]
+
+    headers.append(('Content-Length', str(len(content))),)
+    headers.append(('Access-Control-Allow-Origin', '*',));
+
+    if len(content) == 0:
+        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
+        start_response('404 Looked everywhere, sorry', headers)
+    else:
+        headers.append(('Content-Type', mime_type,))
+        start_response('200 OK', headers)
+
+    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -7,8 +7,6 @@ import tempfile
 import re
 import urllib
 import websocket
-import stat
-import importlib
 
 # external imports
 import celery
@@ -21,17 +19,14 @@ from chainlib.eth.connection import (
         EthUnixSignerConnection,
         EthHTTPSignerConnection,
         )
-from chainlib.eth.address import to_checksum_address
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import add_0x
 import liveness.linux
 
 
 # local imports
-import cic_eth.cli
 from cic_eth.eth import (
         erc20,
         tx,
@@ -73,142 +68,148 @@ from cic_eth.task import BaseTask
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_read
-local_arg_flags = cic_eth.cli.argflag_local_task
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
+argparser.add_argument('-r', type=str, help='CIC registry address')
 argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
-argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
-argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-# process config
-extra_args = {
-    'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
-    'aux_all': None,
-    'aux': None,
-    'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+config.add(args.q, '_CELERY_QUEUE', True)
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-# connect to celery
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
-
-# set up rpc
-rpc = cic_eth.cli.RPC.from_config(config, use_signer=True)
-conn = rpc.get_default()
-
-
-# connect to database
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
-
-
-# execute health checks
-# TODO: health should be separate service with endpoint that can be queried
 health_modules = config.get('CIC_HEALTH_MODULES', [])
 if len(health_modules) != 0:
     health_modules = health_modules.split(',')
 logg.debug('health mods {}'.format(health_modules))
+
+# connect to database
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
+
+
+# set up celery
+current_app = celery.Celery(__name__)
+
+broker = config.get('CELERY_BROKER_URL')
+if broker[:4] == 'file':
+    bq = tempfile.mkdtemp()
+    bp = tempfile.mkdtemp()
+    conf_update = {
+            'broker_url': broker,
+            'broker_transport_options': {
+                'data_folder_in': bq,
+                'data_folder_out': bq,
+                'data_folder_processed': bp,
+            },
+            }
+    if config.true('CELERY_DEBUG'):
+        conf_update['result_extended'] = True
+    current_app.conf.update(conf_update)
+    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
+else:
+    conf_update = {
+            'broker_url': broker,
+            }
+    if config.true('CELERY_DEBUG'):
+        conf_update['result_extended'] = True
+    current_app.conf.update(conf_update)
+
+result = config.get('CELERY_RESULT_URL')
+if result[:4] == 'file':
+    rq = tempfile.mkdtemp()
+    current_app.conf.update({
+        'result_backend': 'file://{}'.format(rq),
+        })
+    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
+else:
+    current_app.conf.update({
+        'result_backend': result,
+        })
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
+RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
+RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
+RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
+
+Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
+
+#import cic_eth.checks.gas
+#if not cic_eth.checks.gas.health(config=config):
+#    raise RuntimeError()
 liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
-
-# set up chain provisions
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-registry = None
-try:
-    registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
-
-trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-if trusted_addresses_src == None:
-    logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-    sys.exit(1)
-trusted_addresses = trusted_addresses_src.split(',')
-for i, address in enumerate(trusted_addresses):
-    if config.get('_UNSAFE'):
-        trusted_addresses[i] = to_checksum_address(address)
-    logg.info('using trusted address {}'.format(address))
-connect_declarator(conn, chain_spec, trusted_addresses)
-connect_token_registry(conn, chain_spec)
-
-# detect auxiliary task modules (plugins)
-# TODO: move to separate file
-aux = []
-if args.aux_all:
-    if len(args.aux) > 0:
-        logg.warning('--aux-all is set so --aux will have no effect')
-    for p in sys.path:
-        logg.debug('checking for aux modules in {}'.format(p))
-        aux_dir = os.path.join(p, 'cic_eth_aux')
-        try:
-            d = os.listdir(aux_dir)
-        except FileNotFoundError:
-            logg.debug('no aux module found in {}'.format(aux_dir))
-            continue
-        for v in d:
-            if v[:1] == '.':
-                logg.debug('dotfile, skip {}'.format(v))
-                continue
-            aux_mod_path = os.path.join(aux_dir, v)
-            st = os.stat(aux_mod_path)
-            if not stat.S_ISDIR(st.st_mode):
-                logg.debug('not a dir, skip {}'.format(v))
-                continue
-            aux_mod_file = os.path.join(aux_dir, v,'__init__.py')
-            try:
-                st = os.stat(aux_mod_file)
-            except FileNotFoundError:
-                logg.debug('__init__.py not found, skip {}'.format(v))
-                continue 
-            aux.append(v)
-            logg.debug('found module {} in {}'.format(v, aux_dir))
-
-elif len(args.aux) > 0:
-    for p in sys.path:
-        v_found = None
-        for v in args.aux:
-            aux_dir = os.path.join(p, 'cic_eth_aux')
-            aux_mod_file = os.path.join(aux_dir, v, '__init__.py')
-            try:
-                st = os.stat(aux_mod_file)
-                v_found = v
-            except FileNotFoundError:
-                logg.debug('cannot find explicity requested aux module {} in path {}'.format(v, aux_dir))
-                continue
-        if v_found == None:
-            logg.critical('excplicity requested aux module {} not found in any path'.format(v))
-            sys.exit(1)
-
-        logg.info('aux module {} found in path {}'.format(v, aux_dir))
-        aux.append(v)
-
-for v in aux:
-    mname = 'cic_eth_aux.' + v
-    mod = importlib.import_module(mname)
-    mod.aux_setup(conn, config)
-    logg.info('loaded aux module {}'.format(mname))
-
-
 def main():
     argv = ['worker']
-    log_level = logg.getEffectiveLevel()
-    log_level_name = logging.getLevelName(log_level)
-    argv.append('--loglevel=' + log_level_name)
+    if args.vv:
+        argv.append('--loglevel=DEBUG')
+    elif args.v:
+        argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
     argv.append('-n')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
+
+#    if config.true('SSL_ENABLE_CLIENT'):
+#        Callback.ssl = True
+#        Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
+#        Callback.ssl_key_file = config.get('SSL_KEY_FILE')
+#        Callback.ssl_password = config.get('SSL_PASSWORD')
+#
+#    if config.get('SSL_CA_FILE') != '':
+#        Callback.ssl_ca_file = config.get('SSL_CA_FILE')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    try:
+        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    except UnknownContractError as e:
+        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+        sys.exit(1)
+
+    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+    if trusted_addresses_src == None:
+        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+        sys.exit(1)
+    trusted_addresses = trusted_addresses_src.split(',')
+    for address in trusted_addresses:
+        logg.info('using trusted address {}'.format(address))
+
+    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_token_registry(rpc, chain_spec)
 
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address)
-    default_token.load(conn)
+    default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
+    default_token.load(rpc)
     BaseTask.default_token_decimals = default_token.decimals
     BaseTask.default_token_name = default_token.name
 
@@ -216,13 +217,13 @@ def main():
     logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    
     liveness.linux.set(rundir=config.get('CIC_RUN_DIR'))
-    celery_app.worker_main(argv)
+    current_app.worker_main(argv)
     liveness.linux.reset(rundir=config.get('CIC_RUN_DIR'))
 
 
 @celery.signals.eventlet_pool_postshutdown.connect
 def shutdown(sender=None, headers=None, body=None, **kwargs):
-    logg.warning('in shutdown event hook')
+    logg.warning('in shudown event hook')
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -8,6 +8,13 @@ import sys
 import re
 
 # external imports
+import confini
+import celery
+import rlp
+import cic_base.config
+import cic_base.log
+import cic_base.argparse
+import cic_base.rpc
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -19,16 +26,13 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
-from chainsyncer.db.models.base import SessionBase
-from chainlib.eth.address import (
-        is_checksum_address,
-        to_checksum_address,
+from chainsyncer.driver import (
+        HeadSyncer,
+        HistorySyncer,
         )
+from chainsyncer.db.models.base import SessionBase
 
 # local imports
-import cic_eth.cli
 from cic_eth.db import dsn_from_config
 from cic_eth.runnable.daemons.filters import (
         CallbackFilter,
@@ -44,50 +48,53 @@ from cic_eth.registry import (
         connect_token_registry,
         )
 
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_read
-local_arg_flags = cic_eth.cli.argflag_local_sync
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
+script_dir = os.path.realpath(os.path.dirname(__file__))
 
-# process config
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser
 
-# connect to celery
-cic_eth.cli.CeleryApp.from_config(config)
 
-# set up database
+logg = cic_base.log.create()
+argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
+args = cic_base.argparse.parse(argparser, logg)
+
+config = cic_base.config.create(args.c, args, args.env_prefix)
+
+config.add(args.y, '_KEYSTORE_FILE', True)
+config.add(args.q, '_CELERY_QUEUE', True)
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
+
+cic_base.config.log(config)
+
 dsn = dsn_from_config(config)
+
 SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
 
-# set up rpc
-rpc = cic_eth.cli.RPC.from_config(config)
-conn = rpc.get_default()
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+
+cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
-# set up chain provisions
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-registry = None
-try:
-    registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 def main():
+    # connect to celery
+    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
     # Connect to blockchain with chainlib
+    rpc = RPCConnection.connect(chain_spec, 'default')
 
     o = block_latest()
-    r = conn.do(o)
+    r = rpc.do(o)
     block_current = int(r, 16)
     block_offset = block_current + 1
 
     loop_interval = config.get('SYNCER_LOOP_INTERVAL')
     if loop_interval == None:
-        stat = init_chain_stat(conn, block_start=block_current)
+        stat = init_chain_stat(rpc, block_start=block_current)
         loop_interval = stat.block_average()
 
     logg.debug('current block height {}'.format(block_offset))
@@ -99,9 +106,9 @@ def main():
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
+        initial_block_start = config.get('SYNCER_HISTORY_START')
         initial_block_offset = block_offset
-        if config.true('SYNCER_NO_HISTORY'):
+        if config.get('_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -114,45 +121,39 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface))
+            syncers.append(HistorySyncer(syncer_backend))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface))
+            syncers.append(HeadSyncer(syncer_backend))
 
-    connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
         logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
         sys.exit(1)
     trusted_addresses = trusted_addresses_src.split(',')
-    for i, address in enumerate(trusted_addresses):
-        if not config.get('_UNSAFE'):
-            if not is_checksum_address(address):
-                raise ValueError('address {} is not a valid checksum address'.format(address))
-        else:
-            trusted_addresses[i] = to_checksum_address(address)
+    for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
-    connect_declarator(conn, chain_spec, trusted_addresses)
-    connect_token_registry(conn, chain_spec)
+    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_token_registry(rpc, chain_spec)
     CallbackFilter.trusted_addresses = trusted_addresses
 
     callback_filters = []
     for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
         task_split = cb.split(':')
-        task_queue = config.get('CELERY_QUEUE')
+        task_queue = config.get('_CELERY_QUEUE')
         if len(task_split) > 1:
             task_queue = task_split[0]
         callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
         callback_filters.append(callback_filter)
 
-    tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE'))
+    tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    account_registry_address = registry.by_name('AccountRegistry')
-    registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE'))
+    registration_filter = RegistrationFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
-    gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
+    gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
 
     #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
 
@@ -167,7 +168,7 @@ def main():
         for cf in callback_filters:
             syncer.add_filter(cf)
 
-        r = syncer.loop(int(loop_interval), conn)
+        r = syncer.loop(int(loop_interval), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 
         i += 1

apps/cic-eth/cic_eth/runnable/info.py

@@ -12,27 +12,52 @@ import confini
 import celery
 
 # local imports
-import cic_eth.cli
-from cic_eth.api import Api
-from cic_eth.api.admin import AdminApi
+from cic_eth.api import (
+        Api,
+        AdminApi,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_base
-local_arg_flags = cic_eth.cli.argflag_local_taskcallback
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
 args = argparser.parse_args()
 
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
 
-celery_app = cic_eth.cli.CeleryApp.from_config(config)
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.dict_override(args_override, 'cli args')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
-api = Api(config.get('CHAIN_SPEC'), queue=config.get('CELERY_QUEUE'))
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+queue = args.q
+
+api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
 admin_api = AdminApi(None)
 
-
 def main():
     t = admin_api.registry()
     registry_address = t.get()