Add tx sender processing filter

fix(contract-migration): Replace missing environment

See merge request grassrootseconomics/cic-internal-integration!271

.gitignore

@@ -13,3 +13,4 @@ build/
 **/coverage
 **/.venv
 .idea
+**/.vim

.gitlab-ci.yml

@@ -1,14 +1,43 @@
 include:
-  - local: 'ci_templates/.cic-template.yml'
-  - local: 'apps/contract-migration/.gitlab-ci.yml'
+  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  # these includes are app specific unit tests
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
+    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - release
+  - deploy 
+
+image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
+
+variables:
+  DOCKER_BUILDKIT: "1"
+  COMPOSE_DOCKER_CLI_BUILD: "1" 
+  CI_DEBUG_TRACE: "true"
+
+before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+# runs on protected branches and pushes to repo
+build-push:
+  stage: build
+  tags:
+    - integration
+  #script:
+  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  script:
+    - TAG=latest sh ./scripts/build-push.sh
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+
+deploy-dev:
+  stage: deploy
+  trigger: grassrootseconomics/devops
+  when: manual

README.md

@@ -2,25 +2,21 @@
 
 ## Getting started 
 
-## Make some keys
+This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
 ```
-docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
+export COMPOSE_DOCKER_CLI_BUILD=1
+export DOCKER_BUILDKIT=1
 ```
 
-
-### Prepare the repo
-
-This is stuff we need to put in  makefile but for now...
-
-File mounts and permisssions need to be set
+start services, database, redis and local ethereum node
 ```
-chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
-````
-
-start cluster
+docker-compose up -d
 ```
-docker-compose up
+
+Run app/contract-migration to deploy contracts
+```
+RUN_MASK=3 docker-compose up contract-migration
 ```
 
 stop cluster
@@ -28,7 +24,7 @@ stop cluster
 docker-compose down
 ```
 
-delete data
+stop cluster and delete data
 ```
 docker-compose down -v
 ```
@@ -38,5 +34,4 @@ rebuild an images
 docker-compose up --build <service_name>
 ```
 
-Deployment variables are writtend to service-configs/.env after everthing is up.
 

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-cache/.coveragerc

@@ -2,4 +2,6 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/postgres/*
+	cic_cache/db/migrations/*
+	cic_cache/version.py
+	cic_cache/cli

apps/cic-cache/.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,22 +1,17 @@
-.cic_cache_variables:
-    variables:
-        APP_NAME: cic-cache
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_cache_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
-
-build-mr-cic-cache:
-    extends:
-        - .cic_cache_changes_target
-        - .py_build_merge_request
-        - .cic_cache_variables
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-        
-
+build-test-cic-cache:
+  stage: test 
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-cache
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-cache
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  allow_failure: true
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    changes:
+        - apps/$APP_NAME/**/*
+    when: always

apps/cic-cache/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt cic_cache/data/config/*

apps/cic-cache/README.md

@@ -0,0 +1 @@
+# CIC-CACHE

apps/cic-cache/cic_cache/api.py

@@ -55,15 +55,37 @@ class Api:
                     queue=callback_queue,
                     )       
 
-    def list(self, offset, limit, address=None):
+    def list(self, offset=0, limit=100, address=None, oldest=False):
         s = celery.signature(
         'cic_cache.tasks.tx.tx_filter',
         [
-            0,
-            100,
+            offset,
+            limit,
             address,
+            oldest,
             ],
-            queue=None
+            queue=self.queue,
+        )
+        if self.callback_param != None:
+            s.link(self.callback_success).on_error(self.callback_error)
+
+        t = s.apply_async()
+
+        return t
+
+
+    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
+        s = celery.signature(
+        'cic_cache.tasks.tx.tx_filter_content',
+        [
+            offset,
+            limit,
+            address,
+            block_offset,
+            block_limit,
+            oldest,
+            ],
+            queue=self.queue,
         )
         if self.callback_param != None:
             s.link(self.callback_success).on_error(self.callback_error)

apps/cic-cache/cic_cache/cache.py

@@ -10,11 +10,17 @@ from cic_cache.db.list import (
         list_transactions_mined,
         list_transactions_account_mined,
         list_transactions_mined_with_data,
+        list_transactions_mined_with_data_index,
+        list_transactions_account_mined_with_data_index,
+        list_transactions_account_mined_with_data,
         )
 
 logg = logging.getLogger()
 
 
+DEFAULT_FILTER_SIZE = 8192 * 8
+DEFAULT_LIMIT = 100
+
 class Cache:
 
     def __init__(self, session):
@@ -25,12 +31,12 @@ class BloomCache(Cache):
 
     @staticmethod
     def __get_filter_size(n):
-        n = 8192 * 8
+        n = DEFAULT_FILTER_SIZE
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 
 
-    def load_transactions(self, offset, limit):
+    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
         """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
 
         Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -47,7 +53,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_mined(self.session, offset, limit) 
+        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -56,7 +62,12 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-            lowest_block = r[0]
+                lowest_block = r[0]
+            else:
+                if oldest:
+                    highest_block = r[0]
+                else:
+                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -65,7 +76,7 @@ class BloomCache(Cache):
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
 
 
-    def load_transactions_account(self, address, offset, limit):
+    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
         """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
 
         :param address: Address to retrieve transactions for.
@@ -77,7 +88,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_account_mined(self.session, address, offset, limit) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -86,7 +97,12 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-            lowest_block = r[0]
+                lowest_block = r[0]
+            else:
+                if oldest:
+                    highest_block = r[0]
+                else:
+                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -97,8 +113,21 @@ class BloomCache(Cache):
 
 class DataCache(Cache):
 
-    def load_transactions_with_data(self, offset, end):
-        rows = list_transactions_mined_with_data(self.session, offset, end) 
+    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+        if limit == 0:
+            limit = DEFAULT_LIMIT
+        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
+        return self.__process_rows(rows, oldest)
+
+
+    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+        if limit == 0:
+            limit = DEFAULT_LIMIT
+        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
+        return self.__process_rows(rows, oldest)
+
+
+    def __process_rows(self, rows, oldest):
         tx_cache = []
         highest_block = -1;
         lowest_block = -1;
@@ -106,7 +135,12 @@ class DataCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r['block_number']
-            lowest_block = r['block_number']
+                lowest_block = r['block_number']
+            else:
+                if oldest:
+                    highest_block = r['block_number']
+                else:
+                    lowest_block = r['block_number']
             tx_type = 'unknown'
 
             if r['value'] != None:

apps/cic-cache/cic_cache/cli/__init__.py

@@ -0,0 +1,15 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp
+from .registry import (
+        connect_registry,
+        connect_token_registry,
+        connect_declarator,
+        )

apps/cic-cache/cic_cache/cli/arg.py

@@ -0,0 +1,20 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+#    REDIS = 16
+#    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-cache/cic_cache/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-cache/cic_cache/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-cache/cic_cache/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+#        if local_arg_flags & CICFlag.REDIS:
+#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-cache/cic_cache/cli/registry.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# external imports
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
+from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+from chainlib.eth.constant import ZERO_ADDRESS
+
+logg = logging.getLogger()
+
+
+def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
+    logg.debug('using token registry address {}'.format(token_registry_address))
+    lookup = TokenIndexLookup(chain_spec, token_registry_address)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
+    registry = CICRegistry(chain_spec, conn)
+    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
+    logg.debug('using declarator address {}'.format(declarator_address))
+    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
+    CICRegistry.address = registry_address
+    registry = CICRegistry(chain_spec, conn)
+    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
+    return registry

apps/cic-cache/cic_cache/cli/rpc.py

@@ -0,0 +1,43 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return RPCConnection.connect(self.chain_spec, 'default')
+
+
+    @staticmethod
+    def from_config(config):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
+        if config.get('SIGNER_PROVIDER'):
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
+        rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-cache
+debug = 0

apps/cic-cache/cic_cache/data/config/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+trust_address =
+health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas

apps/cic-cache/cic_cache/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name = cic-cache
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-cache/cic_cache/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-cache/cic_cache/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-cache/cic_cache/db/list.py

@@ -13,6 +13,9 @@ def list_transactions_mined(
         session,
         offset,
         limit,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -23,15 +26,62 @@ def list_transactions_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
     r = session.execute(s)
     return r
 
 
 def list_transactions_mined_with_data(
+        session,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param block_offset: First block to include in search
+    :type block_offset: int
+    :param block_limit: Last block to include in search
+    :type block_limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
+
+
+    r = session.execute(s)
+    return r
+
+
+def list_transactions_mined_with_data_index(
         session,
         offset,
         end,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -42,7 +92,87 @@ def list_transactions_mined_with_data(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
+
+    r = session.execute(s)
+    return r
+
+
+def list_transactions_account_mined_with_data_index(
+        session,
+        address,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
+
+    :param offset: Offset in data set to return transactions from
+    :type offset: int
+    :param limit: Max number of transactions to retrieve
+    :type limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+
+    r = session.execute(s)
+    return r
+
+def list_transactions_account_mined_with_data(
+        session,
+        address,
+        offset,
+        limit,
+        block_offset,
+        block_limit,
+        oldest=False,
+        ):
+    """Executes db query to return all confirmed transactions according to the specified offset and limit.
+
+    :param block_offset: First block to include in search
+    :type block_offset: int
+    :param block_limit: Last block to include in search
+    :type block_limit: int
+    :result: Result set
+    :rtype: SQLAlchemy.ResultProxy
+    """
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+    else:
+        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
 
     r = session.execute(s)
     return r
@@ -53,6 +183,9 @@ def list_transactions_account_mined(
         address,
         offset,
         limit,
+        block_offset,
+        block_limit,
+        oldest=False,
         ):
     """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
 
@@ -65,7 +198,20 @@ def list_transactions_account_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
+
+    order_by = 'DESC'
+    if oldest:
+        order_by = 'ASC'
+
+    if block_offset:
+        if block_limit:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
+        else:
+            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
+
+    else:
+        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+
     r = session.execute(s)
     return r
 

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,3 +100,4 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -4,6 +4,9 @@ import json
 import re
 import base64
 
+# external imports
+from hexathon import add_0x
+
 # local imports
 from cic_cache.cache import (
         BloomCache,
@@ -11,10 +14,11 @@ from cic_cache.cache import (
     )
 
 logg = logging.getLogger(__name__)
+#logg = logging.getLogger()
 
 re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
+re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
 
 DEFAULT_LIMIT = 100
 
@@ -26,13 +30,13 @@ def process_transactions_account_bloom(session, env):
 
     address = r[1]
     if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
+        address = add_0x(address)
+    offset = 0
     if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
+        offset = r[4]
+    limit = DEFAULT_LIMIT
+    if r.lastindex > 4:
+        limit = r[6]
 
     c = BloomCache(session)
     (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
@@ -87,13 +91,14 @@ def process_transactions_all_data(session, env):
     if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
         return None
 
-    offset = r[1]
-    end = r[2]
+    logg.debug('got data request {}'.format(env))
+    block_offset = r[1]
+    block_end = r[2]
     if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -8,15 +8,7 @@ import sys
 import re
 
 # external imports
-import confini
-import celery
 import sqlalchemy
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
+import cic_cache.cli
 from cic_cache.db import (
         dsn_from_config,
         add_tag,
@@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import (
         FaucetFilter,
         )
 
-script_dir = os.path.realpath(os.path.dirname(__file__))
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
 
-def add_block_args(argparser):
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
+# process args
+arg_flags = cic_cache.cli.argflag_std_read
+local_arg_flags = cic_cache.cli.argflag_local_sync
+argparser = cic_cache.cli.ArgumentParser(arg_flags)
+argparser.process_local_flags(local_arg_flags)
+args = argparser.parse_args()
 
+# process config
+config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
 
-logg = cic_base.log.create()
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
+# connect to database
 dsn = dsn_from_config(config)
-
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+# set up rpc
+rpc = cic_cache.cli.RPC.from_config(config)
+conn = rpc.get_default()
 
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
+# set up chain provisions
+chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+registry = None
+try:
+    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
 
 def register_filter_tags(filters, session):
@@ -95,14 +92,12 @@ def main():
 
     syncers = []
 
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
     syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
+        initial_block_start = config.get('SYNCER_OFFSET')
         initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
+        if config.get('SYNCER_NO_HISTORY'):
             initial_block_start = block_offset
             initial_block_offset += 1
         syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -112,10 +107,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
+        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
+    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/cic_cache/tasks/tx.py

@@ -2,14 +2,17 @@
 import celery
 
 # local imports
-from cic_cache.cache import BloomCache
+from cic_cache.cache import (
+        BloomCache,
+        DataCache,
+        )
 from cic_cache.db.models.base import SessionBase
 
 celery_app = celery.current_app
 
 
 @celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     queue = self.request.delivery_info.get('routing_key')
 
     session = SessionBase.create_session()
@@ -17,9 +20,9 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
     c = BloomCache(session)
     b = None
     if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
     else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
 
     session.close()
 
@@ -35,4 +38,17 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
     return o
 
 
+@celery_app.task(bind=True)
+def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
+    session = SessionBase.create_session()
 
+    c = DataCache(session)
+    b = None
+    if address == None:
+        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+    else:
+        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
+
+    session.close()
+
+    return (lowest_block, highest_block, tx_cache,)

apps/cic-cache/cic_cache/version.py

@@ -4,7 +4,7 @@ import semver
 version = (
         0,
         2,
-        0,
+        1,
         'alpha.2',
         )
 

apps/cic-cache/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/cic.ini

@@ -1,4 +1,3 @@
 [cic]
 registry_address = 
-chain_spec =
 trust_address = 

apps/cic-cache/config/docker/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 
-dir = /usr/local/share/bancor

apps/cic-cache/config/docker/cic.ini

@@ -1,4 +1,3 @@
 [cic]
-chain_spec =
 registry_address =  
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,3 +1,4 @@
 [syncer]
 loop_interval = 1
-history_start = 0
+offset = 0
+no_history = 0

apps/cic-cache/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 5
-history_start = 0

apps/cic-cache/docker/Dockerfile

@@ -1,52 +1,39 @@
-FROM python:3.8.6-slim-buster
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
 
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
 
-WORKDIR /usr/src/cic-cache
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+ARG EXTRA_PIP_ARGS=""
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    -r requirements.txt
 
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
+COPY . . 
 
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY cic-cache/requirements.txt ./
-COPY cic-cache/setup.cfg \
-	 cic-cache/setup.py \
- 	 ./
-COPY cic-cache/cic_cache/ ./cic_cache/
-COPY cic-cache/scripts/ ./scripts/
-COPY cic-cache/test_requirements.txt ./
-RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-RUN pip install $pip_extra_index_url_flag .
-RUN pip install .[server]
-
-COPY cic-cache/tests/ ./tests/
-#COPY db/ cic-cache/db
-#RUN apk add postgresql-client
+RUN python setup.py install
 
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-cache/config/ /usr/local/etc/cic-cache/
+COPY config/ /usr/local/etc/cic-cache/
 
 # for db migrations
 RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
 
-COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY cic-cache/docker/db.sh ./db.sh
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
 RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
+--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
+-r test_requirements.txt 
+
+export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+

apps/cic-cache/requirements.txt

@@ -1,13 +1,15 @@
-cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
-confini~=0.3.6rc3
+confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
-moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
+moolb~=0.1.1b2
+cic-eth-registry~=0.6.1a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1
+chainsyncer[sql]>=0.0.6a3,<0.1.0
+erc20-faucet>=0.3.2a1, <0.4.0
+chainlib-eth>=0.0.9a7,<0.1.0
+chainlib>=0.0.9a3,<0.1.0
+eth-address-index>=0.2.3a1,<0.3.0

apps/cic-cache/setup.cfg

@@ -23,11 +23,13 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
+include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
+	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
@@ -39,3 +41,4 @@ console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
+	cic-cache-list = cic_cache.runable.list:main

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
-sarafu-faucet~=0.0.4a1
+sarafu-faucet~=0.0.7a1
+erc20-transfer-authorization>=0.3.5a1,<0.4.0

apps/cic-cache/tests/cli/test_cli_args.py

@@ -0,0 +1,40 @@
+# standard imports
+import os
+
+# external imports
+import chainlib.cli 
+
+# local imports
+import cic_cache.cli
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
+
+
+def test_argumentparserto_config():
+
+    argparser = cic_cache.cli.ArgumentParser()
+    
+    local_flags = 0xffff
+    argparser.process_local_flags(local_flags) 
+    argparser.add_argument('--foo', type=str)
+    args = argparser.parse_args([
+        '-q', 'baz',
+        '--offset', '13',
+        '--no-history',
+        '-r','0xdeadbeef',
+        '-vv',
+        '--foo', 'bar',
+        ])
+
+    extra_args = {
+            'foo': '_BARBARBAR',
+            }
+    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
+
+    assert config.get('_BARBARBAR') == 'bar'
+    assert config.get('CELERY_QUEUE') == 'baz'
+    assert config.get('SYNCER_NO_HISTORY') == True
+    assert config.get('SYNCER_OFFSET') == 13
+    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
+

apps/cic-cache/tests/cli/test_cli_celery.py

@@ -0,0 +1,17 @@
+# standard imports
+import tempfile
+
+# local imports
+import cic_cache.cli
+
+
+def test_cli_celery():
+    cf = tempfile.mkdtemp()
+
+    config = {
+            'CELERY_RESULT_URL': 'filesystem://' + cf,
+            }
+    cic_cache.cli.CeleryApp.from_config(config)
+
+    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
+    cic_cache.cli.CeleryApp.from_config(config)

apps/cic-cache/tests/cli/test_cli_chain.py

@@ -0,0 +1,68 @@
+# external imports
+import pytest
+from chainlib.eth.gas import (
+        Gas,
+        RPCGasOracle,
+        )
+from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.block import (
+        block_latest,
+        Block,
+        )
+from chainlib.eth.pytest.fixtures_chain import default_chain_spec
+from chainlib.eth.pytest.fixtures_ethtester import *
+from cic_eth_registry.pytest.fixtures_contracts import *
+from hexathon import add_0x
+
+# local imports
+import cic_cache.cli
+
+
+@pytest.mark.xfail()
+def test_cli_rpc(
+        eth_rpc,
+        eth_signer,
+        default_chain_spec,
+        ):
+    config = {
+        'CHAIN_SPEC': str(default_chain_spec),
+        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
+            }
+    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
+    conn = rpc.get_by_label('foo')
+    #o = block_latest()
+    #conn.do(o)
+
+
+def test_cli_chain(
+        default_chain_spec,
+        eth_rpc,
+        eth_signer,
+        contract_roles,
+        ):
+    ifc = cic_cache.cli.EthChainInterface()
+
+    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
+    gas_oracle = RPCGasOracle(conn=eth_rpc)
+    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
+    recipient = add_0x(os.urandom(20).hex())
+    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
+    r = eth_rpc.do(o)
+
+    o = ifc.tx_receipt(r)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    o = ifc.block_by_number(1)
+    block_src = eth_rpc.do(o)
+    block = ifc.block_from_src(block_src)
+    assert block.number == 1
+
+    with pytest.raises(KeyError):
+        assert block_src['gasUsed'] == 21000
+        assert block_src['gas_used'] == 21000
+
+    block_src = ifc.src_normalize(block_src)
+    assert block_src['gasUsed'] == 21000
+    assert block_src['gas_used'] == 21000
+

apps/cic-cache/tests/conftest.py

@@ -5,9 +5,12 @@ import datetime
 
 # external imports
 import pytest
+import moolb
 
 # local imports
 from cic_cache import db
+from cic_cache import BloomCache
+from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -61,7 +64,6 @@ def txs(
         dt.timestamp(),
             )
 
-
     tx_number = 42
     tx_hash_second = '0x' + os.urandom(32).hex()
     tx_signed_second = '0x' + os.urandom(128).hex()
@@ -90,6 +92,44 @@ def txs(
             ]
 
 
+@pytest.fixture(scope='function')
+def more_txs(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        ):
+
+    session = init_database
+
+    tx_number = 666
+    tx_hash = '0x' + os.urandom(32).hex()
+    tx_signed = '0x' + os.urandom(128).hex()
+    nonce = 3
+
+    dt = datetime.datetime.utcnow()
+    dt += datetime.timedelta(hours=1)
+    db.add_transaction(
+        session,
+        tx_hash,
+        list_defaults['block']+2,
+        tx_number,
+        list_actors['alice'],
+        list_actors['diane'],
+        list_tokens['bar'],
+        list_tokens['bar'],
+        2048,
+        4096,
+        False,
+        dt.timestamp(),
+        )
+
+    session.commit()
+
+    return [tx_hash] + txs
+
+
 @pytest.fixture(scope='function')
 def tag_txs(
         init_database,
@@ -101,3 +141,7 @@ def tag_txs(
 
     db.tag_transaction(init_database, txs[1], 'taag', domain='test')
 
+
+@pytest.fixture(scope='session')
+def zero_filter():
+    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/test_erc20.py

@@ -10,6 +10,7 @@ from sqlalchemy import text
 from chainlib.eth.tx import Tx
 from chainlib.eth.block import Block
 from chainlib.chain import ChainSpec
+from chainlib.eth.error import RequestMismatchException
 from hexathon import (
         strip_0x,
         add_0x,
@@ -18,10 +19,21 @@ from hexathon import (
 # local imports
 from cic_cache.db import add_tag
 from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
+from cic_cache.runnable.daemons.filters.base import TagSyncFilter
 
 logg = logging.getLogger()
 
 
+def test_base_filter_str(
+        init_database,
+        ):
+        f = TagSyncFilter('foo')
+        assert 'foo' == str(f)
+        f = TagSyncFilter('foo', domain='bar')
+        assert 'bar.foo' == str(f)
+
+
+
 def test_erc20_filter(
         eth_rpc,
         foo_token,
@@ -67,3 +79,95 @@ def test_erc20_filter(
     s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
     r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
     assert r[0] == tx.hash
+
+
+def test_erc20_filter_nocontract(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = 'a9059cbb'
+    data += strip_0x(list_actors['alice'])
+    data += '1000'.ljust(64, '0')
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': os.urandom(20).hex(),
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
+
+
+@pytest.mark.parametrize(
+        'contract_method,contract_input,expected_exception',
+        [
+            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
+            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
+            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
+            ],
+        )
+def test_erc20_filter_bogus(
+        eth_rpc,
+        foo_token,
+        init_database,
+        list_defaults,
+        list_actors,
+        tags,
+        contract_method,
+        contract_input,
+        expected_exception,
+        ):
+
+    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
+
+    fltr = ERC20TransferFilter(chain_spec)
+    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
+
+    # incomplete args
+    data = contract_method
+    data += contract_input 
+    block = Block({
+        'hash': os.urandom(32).hex(),
+        'number': 42,
+        'timestamp': datetime.datetime.utcnow().timestamp(),
+        'transactions': [],
+        })
+
+    tx = Tx({
+        'to': foo_token,
+        'from': list_actors['bob'],
+        'data': data,
+        'value': 0,
+        'hash': os.urandom(32).hex(),
+        'nonce': 13,
+        'gasPrice': 10000000,
+        'gas': 123456,
+            })
+    block.txs.append(tx)
+    tx.block = block
+
+    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/test_cache.py

@@ -8,6 +8,7 @@ import json
 import pytest
 
 # local imports
+from cic_cache import db
 from cic_cache import BloomCache
 from cic_cache.cache import DataCache
 
@@ -18,7 +19,6 @@ def test_cache(
         init_database,
         list_defaults,
         list_actors,
-        list_tokens,
         txs,
         ):
 
@@ -37,9 +37,6 @@ def test_cache(
 
 def test_cache_data(
         init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
         txs,
         tag_txs,
         ):
@@ -47,10 +44,209 @@ def test_cache_data(
     session = init_database
 
     c = DataCache(session)
-    b = c.load_transactions_with_data(410000, 420000)
+    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
 
     assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[1]
-    assert b[2][1]['tx_type'] == 'unknown'
-    assert b[2][0]['tx_type'] == 'test.taag'
+    assert b[2][0]['tx_hash'] == txs[0]
+    assert b[2][0]['tx_type'] == 'unknown'
+    assert b[2][1]['tx_type'] == 'test.taag'
+
+
+def test_cache_ranges(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        more_txs,
+        ):
+
+    session = init_database
+       
+    oldest = list_defaults['block'] - 1
+    mid = list_defaults['block']
+    newest = list_defaults['block'] + 2
+
+    c = BloomCache(session)
+    b = c.load_transactions(0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions(1, 2)
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    b = c.load_transactions(0, 2)
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 1)
+    assert b[0] == newest
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, oldest=True)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
+    assert b[0] == oldest
+    assert b[1] == mid
+
+    # now check when supplying account
+    b = c.load_transactions_account(list_actors['alice'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    b = c.load_transactions_account(list_actors['bob'], 0, 100)
+    assert b[0] == mid
+    assert b[1] == mid
+
+    b = c.load_transactions_account(list_actors['diane'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+
+    # add block filter to the mix
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
     
+    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+
+    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == mid
+
+    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == oldest
+
+
+def test_cache_ranges_data( 
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        more_txs,
+        ):
+
+    session = init_database
+       
+    oldest = list_defaults['block'] - 1
+    mid = list_defaults['block']
+    newest = list_defaults['block'] + 2
+
+    c = DataCache(session)
+
+    b = c.load_transactions_with_data(0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][2]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(1, 2)
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[1]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(0, 2)
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_with_data(0, 1)
+    assert b[0] == newest
+    assert b[1] == newest
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[0]
+
+    b = c.load_transactions_with_data(0, 100, oldest=True)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[2]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+    assert b[2][2]['tx_hash'] == more_txs[0]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[1]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
+    assert b[0] == oldest
+    assert b[1] == mid
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[2]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    # now check when supplying account
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 3
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+    assert b[2][2]['tx_hash'] == more_txs[2]
+
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
+    assert b[0] == mid
+    assert b[1] == mid
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
+    assert b[0] == oldest
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[2]
+
+    # add block filter to the mix
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == newest
+    assert len(b[2]) == 2
+    assert b[2][0]['tx_hash'] == more_txs[0]
+    assert b[2][1]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == mid
+    assert b[1] == mid
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[1]
+
+    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
+    assert b[0] == oldest
+    assert b[1] == oldest
+    assert len(b[2]) == 1
+    assert b[2][0]['tx_hash'] == more_txs[2]

apps/cic-cache/tests/test_query.py

@@ -0,0 +1,230 @@
+# standard imports
+import logging
+import json
+import base64
+import copy
+import re
+
+# external imports
+import pytest
+from hexathon import strip_0x
+
+# local imports
+from cic_cache.runnable.daemons.query import *
+
+logg = logging.getLogger()
+
+
+
+@pytest.mark.parametrize(
+        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
+        [
+            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
+            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
+            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
+            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
+            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
+            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
+            ],
+        )
+def test_query_regex(
+        list_actors,
+        query_path_prefix,
+        query_role,
+        query_address_index,
+        query_offset,
+        query_offset_index,
+        query_limit,
+        query_limit_index,
+        match_re,
+        ):
+
+        paths = []
+        path = query_path_prefix
+        query_address = None
+        if query_role != None:
+            query_address = strip_0x(list_actors[query_role])
+            paths.append(path + '0x' + query_address)
+            paths.append(path + query_address)
+        if query_offset != None:
+            if query_limit != None:
+                for i in range(len(paths)-1):
+                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
+            else:
+                for i in range(len(paths)-1):
+                    paths[i] += '/' + str(query_offset)
+
+        for i in range(len(paths)):
+            paths.append(paths[i] + '/')
+
+        for p in paths:
+            logg.debug('testing path {} against {}'.format(p, match_re))
+            m = re.match(match_re, p)
+            l = len(m.groups())
+            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
+            for i in range(l+1):
+                logg.debug('group {} {}'.format(i, m[i]))
+            if m.lastindex >= query_offset_index:
+                assert query_offset == int(m[query_offset_index + 1])
+            if m.lastindex >= query_limit_index:
+                assert query_limit == int(m[query_limit_index + 1])
+            if query_address_index != None:
+                match_address = strip_0x(m[query_address_index + 1])
+                assert query_address == match_address
+
+
+
+@pytest.mark.parametrize(
+        'role_name, query_offset, query_limit, query_match',
+        [ 
+            ('alice', None, None, [(420000, 13), (419999, 42)]),
+            ('alice', None, 1, [(420000, 13)]),
+            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
+            ('alice', 2, None, []), # 420000 == list_defaults['block']
+            ],
+        )
+def test_query_process_txs_account(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        role_name,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    actor = None
+    try:
+        actor = list_actors[role_name]
+    except KeyError:
+        actor = os.urandom(20).hex()
+    path_info = '/tx/user/0x' + strip_0x(actor)
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_account_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_offset, query_limit, query_match',
+        [ 
+            (None, 2, [(420000, 13), (419999, 42)]),
+            (0, 1, [(420000, 13)]),
+            (1, 1, [(419999, 42)]),
+            (2, 0, []),
+            ],
+        )
+def test_query_process_txs_bloom(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_offset,
+        query_limit,
+        query_match,
+        ):
+
+    path_info = '/tx'
+    if query_offset != None:
+        path_info += '/' + str(query_offset)
+    if query_limit != None:
+        if query_offset == None:
+            path_info += '/0'
+        path_info += '/' + str(query_limit)
+    env = {
+            'PATH_INFO': path_info,
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_bloom(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
+    zero_filter_data = zero_filter.to_bytes()
+    if len(query_match) == 0:
+        assert block_filter_data == zero_filter_data
+        return
+
+    assert block_filter_data != zero_filter_data
+    block_filter = copy.copy(zero_filter)
+    block_filter.merge(block_filter_data)
+    block_filter_data = block_filter.to_bytes()
+    assert block_filter_data != zero_filter_data
+
+    for (block, tx) in query_match:
+        block = block.to_bytes(4, byteorder='big')
+        assert block_filter.check(block)
+
+
+@pytest.mark.parametrize(
+        'query_block_start, query_block_end, query_match_count',
+        [ 
+            (None, 42, 0),
+            (420000, 420001, 1),
+            (419999, 419999, 1), # matches are inclusive
+            (419999, 420000, 2),
+            (419999, 420001, 2),
+            ],
+        )
+def test_query_process_txs_data(
+        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
+        txs,
+        zero_filter,
+        query_block_start,
+        query_block_end,
+        query_match_count,
+        ):
+
+    path_info = '/txa'
+    if query_block_start != None:
+        path_info += '/' + str(query_block_start)
+    if query_block_end != None:
+        if query_block_start == None:
+            path_info += '/0'
+        path_info += '/' + str(query_block_end)
+    env = {
+            'PATH_INFO': path_info,
+            'HTTP_X_CIC_CACHE_MODE': 'all',
+            }
+    logg.debug('using path {}'.format(path_info))
+    r = process_transactions_all_data(init_database, env)
+    assert r != None
+
+    o = json.loads(r[1])
+    assert len(o['data']) == query_match_count

apps/cic-cache/tests/testdata/config/test.ini

@@ -0,0 +1,2 @@
+[foo]
+bar_baz = xyzzy

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp):
 
 
 def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CIC_CHAIN_SPEC')
+    chain_spec_str = config.get('CHAIN_SPEC')
     chain_spec = ChainSpec.from_chain_str(chain_spec_str)
     token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.2a3
-cic-eth-registry~=0.5.6a1
-chainlib~=0.0.5a1
-cic_eth~=0.12.0a2
+erc20-demurrage-token~=0.0.3a1
+cic-eth-registry~=0.5.8a1
+chainlib~=0.0.7a1
+cic_eth~=0.12.2a4

apps/cic-eth-aux/erc20-demurrage-token/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a4
+version = 0.0.2a6
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -5,8 +5,7 @@ pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
 SQLAlchemy==1.3.20
-cic-eth~=0.12.0a1
 liveness~=0.0.1a7
-eth-accounts-index==0.0.12a1
-eth-contract-registry==0.5.6a1
-eth-address-index==0.1.2a1
+eth-accounts-index==0.1.1a1
+eth-contract-registry==0.5.8a1
+eth-address-index==0.2.1a1

apps/cic-eth/.coveragerc

@@ -6,4 +6,5 @@ omit =
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
 	cic_eth/queue/state.py
+	cic_eth/cli
 	*redis*.py

apps/cic-eth/.dockerignore

@@ -0,0 +1,6 @@
+.git
+.cache
+.dot
+**/doc
+**/.venv
+**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,31 +1,16 @@
-.cic_eth_variables:
-    variables:
-        APP_NAME: cic-eth
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_eth_mr_changes_target:
-    rules:
+build-test-cic-eth:
+  stage: test
+  tags:
+    - integration
+  variables:
+    APP_NAME: cic-eth
+    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  script:
+    - cd apps/cic-eth
+    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+  rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
-          - $CONTEXT/$APP_NAME/**/*
+          - apps/$APP_NAME/**/*
       when: always
-
-build-mr-cic-eth:
-    extends:
-        - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-        - .py_build_target_test
-
-test-mr-cic-eth:
-    extends:
-        - .cic_eth_variables
-        - .cic_eth_mr_changes_target
-    stage: test
-    image: $IMAGE_TAG_BASE
-    script:
-        - cd apps/$APP_NAME/
-        - pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-
-build-push-cic-eth:
-    extends:
-        - .py_build_push

apps/cic-eth/MANIFEST.in

@@ -1,2 +1,2 @@
-include *requirements.txt config/test/*
+include *requirements.txt config/test/* cic_eth/data/config/*
 

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry~=0.5.6a1
-hexathon~=0.0.1a7
-chainqueue~=0.0.2b5
-eth-erc20==0.0.10a2
+cic-eth-registry>=0.6.1a2,<0.7.0
+hexathon~=0.0.1a8
+chainqueue>=0.0.4a6,<0.1.0
+eth-erc20>=0.1.2a2,<0.2.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,8 +4,12 @@ import logging
 
 # external imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
+from hexathon import (
+        add_0x,
+        strip_0x,
+        uniform as hex_uniform,
+        )
 
 # local imports
 from cic_eth.db.enum import LockEnum
@@ -15,12 +19,17 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -32,6 +41,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -41,7 +51,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -53,6 +63,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -62,7 +73,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -72,6 +83,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -79,7 +91,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -89,6 +101,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -96,7 +109,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -106,6 +119,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -113,7 +127,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -123,6 +137,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
+    address = tx_normalize.wallet_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -131,11 +146,13 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
+    if address != None:
+        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -14,7 +14,11 @@ from chainqueue.sql.query import get_tx
 from chainqueue.sql.state import set_cancel
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.models.tx import TxCache
-from hexathon import strip_0x
+from hexathon import (
+        strip_0x,
+        add_0x,
+        uniform as hex_uniform,
+        )
 from potaahto.symbols import snake_and_camel
 
 # local imports
@@ -29,6 +33,7 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -69,15 +74,17 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
+    query_address = tx_normalize.wallet_address(address)
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==address)
+    q = q.filter(TxCache.sender==query_address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
+    gas_total = 0
     for otx in otxs:
         tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
         tx_new = unpack(tx_raw, chain_spec)
@@ -89,8 +96,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         tx_new['gas_price'] += 1
         tx_new['gasPrice'] = tx_new['gas_price']
         tx_new['nonce'] -= delta
+        gas_total += tx_new['gas_price'] * tx_new['gas']
 
         logg.debug('tx_new {}'.format(tx_new))
+        logg.debug('gas running total {}'.format(gas_total))
 
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
@@ -122,8 +131,10 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     s = create_check_gas_task(
          txs, 
          chain_spec,
-         tx_new['from'],
-         gas=tx_new['gas'],
+         #tx_new['from'],
+         address,
+         #gas=tx_new['gas'],
+         gas=gas_total,
          tx_hashes_hex=tx_hashes, 
          queue=queue,
         )
@@ -132,7 +143,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_send',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )
@@ -140,7 +152,8 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
         'cic_eth.admin.ctrl.unlock_queue',
         [
             chain_spec.asdict(),
-            tx_new['from'],
+            address,
+            #tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/api/admin.py

@@ -21,6 +21,7 @@ from chainlib.hash import keccak256_hex_to_hex
 from hexathon import (
         strip_0x,
         add_0x,
+        uniform as hex_uniform,
         )
 from chainlib.eth.gas import balance
 from chainqueue.db.enum import (
@@ -31,7 +32,6 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -39,6 +39,7 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
+from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -283,7 +284,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
+            tx_dict = get_tx_local(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()
@@ -307,6 +308,8 @@ class AdminApi:
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
         """
+
+        address = add_0x(hex_uniform(strip_0x(address)))
         last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.query.get_account_tx',

apps/cic-eth/cic_eth/api/api_task.py

@@ -520,9 +520,9 @@ class Api(ApiBase):
             s_external_get = celery.signature(
                 external_task,
                 [
-                    address,
                     offset,
                     limit,
+                    address,
                     ],
                 queue=external_queue,
                 )

apps/cic-eth/cic_eth/check/gas.py

@@ -21,7 +21,7 @@ def health(*args, **kwargs):
     session = SessionBase.create_session()
 
     config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
     logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
 
     try:

apps/cic-eth/cic_eth/check/signer.py

@@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__)
 def health(*args, **kwargs):
     blocked = True
     max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
+    conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer')
     for i in range(max_attempts):
         idx = i + 1
         logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))

apps/cic-eth/cic_eth/cli/__init__.py

@@ -0,0 +1,10 @@
+# local imports
+from .base import *
+from .chain import (
+        EthChainInterface,
+        chain_interface,
+        )
+from .rpc import RPC
+from .arg import ArgumentParser
+from .config import Config
+from .celery import CeleryApp

apps/cic-eth/cic_eth/cli/arg.py

@@ -0,0 +1,31 @@
+# external imports
+from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
+
+# local imports
+from .base import (
+        CICFlag,
+        Flag,
+        )
+
+
+class ArgumentParser(BaseArgumentParser):
+
+    def process_local_flags(self, local_arg_flags):
+        if local_arg_flags & CICFlag.REDIS:
+            self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+            self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+            self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+            self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
+        if local_arg_flags & CICFlag.CELERY:
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
+        if local_arg_flags & CICFlag.SYNCER:
+            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
+            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+        if local_arg_flags & CICFlag.CHAIN:
+            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
+
+
+

apps/cic-eth/cic_eth/cli/base.py

@@ -0,0 +1,31 @@
+# standard imports
+import enum
+
+# external imports
+from chainlib.eth.cli import (
+    argflag_std_read,
+    argflag_std_write,
+    argflag_std_base,
+    Flag,
+    )
+
+class CICFlag(enum.IntEnum):
+   
+    # celery - nibble 1
+    CELERY = 1
+
+    # redis - nibble 2
+    REDIS = 16
+    REDIS_CALLBACK = 32
+
+    # chain - nibble 3
+    CHAIN = 256
+
+    # sync - nibble 4
+    SYNCER = 4096
+
+
+argflag_local_task = CICFlag.CELERY 
+argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
+argflag_local_chain = CICFlag.CHAIN
+argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-eth/cic_eth/cli/celery.py

@@ -0,0 +1,24 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+logg = logging.getLogger(__name__)
+
+
+class CeleryApp:
+   
+    @classmethod
+    def from_config(cls, config):
+        backend_url = config.get('CELERY_RESULT_URL')
+        broker_url = config.get('CELERY_BROKER_URL')
+        celery_app = None
+        if backend_url != None:
+            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
+            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
+        else:
+            celery_app = celery.Celery(broker=broker_url)
+            logg.info('creating celery app without results backend on {}'.format(broker_url))
+
+        return celery_app

apps/cic-eth/cic_eth/cli/chain.py

@@ -0,0 +1,21 @@
+# external imports
+from chainlib.eth.block import (
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
+
+
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()

apps/cic-eth/cic_eth/cli/config.py

@@ -0,0 +1,63 @@
+# standard imports
+import os
+import logging
+
+# external imports
+from chainlib.eth.cli import (
+        Config as BaseConfig,
+        Flag,
+        )
+
+# local imports
+from .base import CICFlag
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+logg = logging.getLogger(__name__)
+
+
+class Config(BaseConfig):
+
+    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
+
+    @classmethod
+    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
+        expanded_base_config_dir = [cls.local_base_config_dir]
+        if base_config_dir != None:
+            if isinstance(base_config_dir, str):
+                base_config_dir = [base_config_dir]
+            for d in base_config_dir:
+                expanded_base_config_dir.append(d)
+        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
+
+        local_args_override = {}
+        if local_arg_flags & CICFlag.REDIS:
+            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
+            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
+            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
+            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
+
+        if local_arg_flags & CICFlag.CHAIN:
+            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
+
+        if local_arg_flags & CICFlag.CELERY:
+            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
+
+        if local_arg_flags & CICFlag.SYNCER:
+            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
+            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
+
+        config.dict_override(local_args_override, 'local cli args')
+
+        if local_arg_flags & CICFlag.REDIS_CALLBACK:
+            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
+            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
+
+        if local_arg_flags & CICFlag.CELERY:
+            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
+
+        logg.debug('config loaded:\n{}'.format(config))
+
+        return config
+
+

apps/cic-eth/cic_eth/cli/rpc.py

@@ -0,0 +1,90 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import (
+        RPCConnection,
+        ConnType,
+        )
+from chainlib.eth.connection import (
+        EthUnixSignerConnection,
+        EthHTTPSignerConnection,
+        )
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+
+class RPC:
+
+    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
+        self.chain_spec = chain_spec
+        self.rpc_provider = rpc_provider
+        self.signer_provider = signer_provider
+
+
+    def get_default(self):
+        return self.get_by_label('default')
+
+
+    def get_by_label(self, label):
+        return RPCConnection.connect(self.chain_spec, label)
+
+
+    @staticmethod
+    def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
+        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
+        if use_signer:
+
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
+        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
+        logg.info('set up rpc: {}'.format(rpc))
+        return rpc
+
+
+    def __str__(self):
+        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
+
+
+# TOOD: re-implement file backend option from omittec code:
+#broker = config.get('CELERY_BROKER_URL')
+#if broker[:4] == 'file':
+#    bq = tempfile.mkdtemp()
+#    bp = tempfile.mkdtemp()
+#    conf_update = {
+#            'broker_url': broker,
+#            'broker_transport_options': {
+#                'data_folder_in': bq,
+#                'data_folder_out': bq,
+#                'data_folder_processed': bp,
+#            },
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
+#else:
+#    conf_update = {
+#            'broker_url': broker,
+#            }
+#    if config.true('CELERY_DEBUG'):
+#        conf_update['result_extended'] = True
+#    current_app.conf.update(conf_update)
+#
+#result = config.get('CELERY_RESULT_URL')
+#if result[:4] == 'file':
+#    rq = tempfile.mkdtemp()
+#    current_app.conf.update({
+#        'result_backend': 'file://{}'.format(rq),
+#        })
+#    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
+#else:
+#    current_app.conf.update({
+#        'result_backend': result,
+#        })
+#

apps/cic-eth/cic_eth/data/config/celery.ini

@@ -0,0 +1,5 @@
+[celery]
+broker_url = redis://localhost:6379
+result_url = 
+queue = cic-eth
+debug = 0

apps/cic-eth/cic_eth/data/config/cic.ini

@@ -1,8 +1,6 @@
 [cic]
 registry_address = 
-chain_spec = evm:bloxberg:8996
-tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
+default_token_symbol =
 health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/cic_eth/data/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+engine =
+driver =
+host =
+port =
+name =
+user =
+password =
+debug = 0
+pool_size = 0

apps/cic-eth/cic_eth/data/config/dispatcher.ini

@@ -1,2 +1,2 @@
-[SYNCER]
+[dispatcher]
 loop_interval = 1

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,3 +1,2 @@
 [eth]
-provider = http://localhost:8545
 gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/cic_eth/data/config/redis.ini

@@ -0,0 +1,5 @@
+[redis]
+host = localhost
+port = 6379
+db = 0
+timeout = 20.0

apps/cic-eth/cic_eth/data/config/retry.ini

@@ -0,0 +1,3 @@
+[retry]
+delay =
+batch_size =

apps/cic-eth/cic_eth/data/config/signer.ini

@@ -0,0 +1,2 @@
+[signer]
+provider =

apps/cic-eth/cic_eth/data/config/syncer.ini

@@ -0,0 +1,4 @@
+[syncer]
+loop_interval = 1
+offset = 0
+no_history = 0

apps/cic-eth/cic_eth/db/migrations/default/versions/0ec0d6d1e785_add_chainqueue.py

@@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:30:55.398388
 from alembic import op
 import sqlalchemy as sa
 
-from chainqueue.db.migrations.sqlalchemy import (
+#from chainqueue.db.migrations.sqlalchemy import (
+from chainqueue.db.migrations.default.export import (
         chainqueue_upgrade,
         chainqueue_downgrade,
         )

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -8,8 +8,8 @@ Create Date: 2021-04-02 18:41:20.864265
 import datetime
 from alembic import op
 import sqlalchemy as sa
-from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth.db.enum import LockEnum
+from cic_eth.encode import ZERO_ADDRESS_NORMAL
 
 
 # revision identifiers, used by Alembic.
@@ -30,7 +30,7 @@ def upgrade():
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS_NORMAL, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/db/migrations/default/versions/b125cbf81e32_add_chain_syncer.py

@@ -8,7 +8,8 @@ Create Date: 2021-04-02 18:36:44.459603
 from alembic import op
 import sqlalchemy as sa
 
-from chainsyncer.db.migrations.sqlalchemy import (
+#from chainsyncer.db.migrations.sqlalchemy import (
+from chainsyncer.db.migrations.default.export import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-eth/cic_eth/db/models/base.py

@@ -58,6 +58,7 @@ class SessionBase(Model):
     def connect(dsn, pool_size=16, debug=False):
         """Create new database connection engine and connect to database backend.
 
+        
         :param dsn: DSN string defining connection.
         :type dsn: str
         """
@@ -126,3 +127,4 @@ class SessionBase(Model):
             logg.debug('commit and destroy session {}'.format(session_key))
             session.commit()
             session.close()
+            del SessionBase.localsessions[session_key]

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,12 +4,12 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
+from cic_eth.encode  import ZERO_ADDRESS_NORMAL
 
 logg = logging.getLogger()
 
@@ -37,7 +37,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.

apps/cic-eth/cic_eth/encode.py

@@ -0,0 +1,16 @@
+# external imports
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainqueue.encode import TxHexNormalizer
+from chainlib.eth.tx import unpack
+
+tx_normalize = TxHexNormalizer()
+
+ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
+
+
+def unpack_normal(signed_tx_bytes, chain_spec):
+    tx = unpack(signed_tx_bytes, chain_spec)
+    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
+    tx['from'] = tx_normalize.wallet_address(tx['from'])
+    tx['to'] = tx_normalize.wallet_address(tx['to'])
+    return tx

apps/cic-eth/cic_eth/eth/account.py

@@ -14,16 +14,13 @@ from chainlib.eth.sign import (
         sign_message,
         )
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.tx import (
-        TxFormat,
-        unpack,
-        )
+from chainlib.eth.tx import TxFormat
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
 from eth_accounts_index import AccountsIndex 
 from sarafu_faucet import MinterFaucet
-from chainqueue.db.models.tx import TxCache
+from chainqueue.sql.tx import cache_tx_dict
 
 # local import
 from cic_eth_registry import CICRegistry
@@ -49,6 +46,10 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
         register_tx,
         )
+from cic_eth.encode import (
+        unpack_normal,
+        ZERO_ADDRESS_NORMAL,
+        )
 
 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -295,25 +296,22 @@ def cache_gift_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
 
     session = self.create_session()
 
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        0,
-        0,
-        session=session,
-            )
+    tx_dict = {
+            'hash': tx['hash'],
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
+            'from_value': 0,
+            'to_value': 0,
+            }
 
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -337,23 +335,20 @@ def cache_account_data(
     :rtype: tuple
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        0,
-        0,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx['hash'],
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
+            'from_value': 0,
+            'to_value': 0,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/bancor.py.bak

@@ -1,385 +0,0 @@
-# standard imports
-import os
-import logging
-
-# third-party imports
-import celery
-import web3
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import SessionBase
-from cic_eth.db.models.convert import TxConvertTransfer
-from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.tx import TxCache
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.eth.token import TokenTxFactory
-from cic_eth.eth.factory import TxFactory
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.rpc import RpcClient
-
-celery_app = celery.current_app 
-#logg = celery_app.log.get_default_logger()
-logg = logging.getLogger()
-
-contract_function_signatures = {
-        'convert': 'f3898a97',
-        'convert2': '569706eb',
-        }
-
-
-class BancorTxFactory(TxFactory):
-
-    """Factory for creating Bancor network transactions.
-    """
-    def convert(
-            self,
-            source_token_address,
-            destination_token_address,
-            reserve_address,
-            source_amount,
-            minimum_return,
-            chain_spec,
-            fee_beneficiary='0x0000000000000000000000000000000000000000',
-            fee_ppm=0,
-            ):
-        """Create a BancorNetwork "convert" transaction.
-
-        :param source_token_address: ERC20 contract address for token to convert from
-        :type source_token_address: str, 0x-hex
-        :param destination_token_address: ERC20 contract address for token to convert to
-        :type destination_token_address: str, 0x-hex
-        :param reserve_address: ERC20 contract address of Common reserve token
-        :type reserve_address: str, 0x-hex
-        :param source_amount: Amount of source tokens to convert
-        :type source_amount: int
-        :param minimum_return: Minimum amount of destination tokens to accept as result for conversion
-        :type source_amount: int
-        :return: Unsigned "convert" transaction in standard Ethereum format
-        :rtype: dict
-        """
-        network_contract = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
-        network_gas = network_contract.gas('convert')
-        tx_convert_buildable = network_contract.contract.functions.convert2(
-            [
-                source_token_address,
-                source_token_address,
-                reserve_address,
-                destination_token_address,
-                destination_token_address,
-            ],
-            source_amount,
-            minimum_return,
-            fee_beneficiary,
-            fee_ppm,
-            )
-        tx_convert = tx_convert_buildable.buildTransaction({
-                'from': self.address,
-                'gas': network_gas,
-                'gasPrice': self.gas_price,
-                'chainId': chain_spec.chain_id(),
-                'nonce': self.next_nonce(),
-                })
-        return tx_convert
-
-
-def unpack_convert(data):
-    f = data[2:10]
-    if f != contract_function_signatures['convert2']:
-        raise ValueError('Invalid convert data ({})'.format(f))
-
-    d = data[10:]
-    path = d[384:]
-    source = path[64-40:64]
-    destination = path[-40:]
-
-    amount = int(d[64:128], 16)
-    min_return = int(d[128:192], 16)
-    fee_recipient = d[192:256]
-    fee = int(d[256:320], 16)
-    return {
-        'amount': amount,
-        'min_return': min_return,
-        'source_token': web3.Web3.toChecksumAddress('0x' + source),
-        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
-        'fee_recipient': fee_recipient,
-        'fee': fee,
-        }
-
-
-
-# Kept for historical reference, it unpacks a convert call without fee parameters
-#def _unpack_convert_mint(data):
-#    f = data[2:10]
-#    if f != contract_function_signatures['convert2']:
-#        raise ValueError('Invalid convert data ({})'.format(f))
-#
-#    d = data[10:]
-#    path = d[256:]
-#    source = path[64-40:64]
-#    destination = path[-40:]
-#
-#    amount = int(d[64:128], 16)
-#    min_return = int(d[128:192], 16)
-#    return {
-#        'amount': amount,
-#        'min_return': min_return,
-#        'source_token': web3.Web3.toChecksumAddress('0x' + source),
-#        'destination_token': web3.Web3.toChecksumAddress('0x' + destination),
-#        }
-
-
-@celery_app.task(bind=True)
-def convert_with_default_reserve(self, tokens, from_address, source_amount, minimum_return, to_address, chain_str):
-    """Performs a conversion between two liquid tokens using Bancor network.
-
-    :param tokens: Token pair, source and destination respectively
-    :type tokens: list of str, 0x-hex
-    :param from_address: Ethereum address of sender
-    :type from_address: str, 0x-hex
-    :param source_amount: Amount of source tokens to convert
-    :type source_amount: int
-    :param minimum_return: Minimum about of destination tokens to receive
-    :type minimum_return: int
-    """
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    queue = self.request.delivery_info['routing_key']
-
-    c = RpcClient(chain_spec, holder_address=from_address)
-
-    cr = CICRegistry.get_contract(chain_spec, 'BancorNetwork')
-    source_token = CICRegistry.get_address(chain_spec, tokens[0]['address'])
-    reserve_address = CICRegistry.get_contract(chain_spec, 'BNTToken', 'ERC20').address()
-
-    tx_factory = TokenTxFactory(from_address, c)
-   
-    tx_approve_zero = tx_factory.approve(source_token.address(), cr.address(), 0, chain_spec)
-    (tx_approve_zero_hash_hex, tx_approve_zero_signed_hex) = sign_and_register_tx(tx_approve_zero, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
-
-    tx_approve = tx_factory.approve(source_token.address(), cr.address(), source_amount, chain_spec)
-    (tx_approve_hash_hex, tx_approve_signed_hex) = sign_and_register_tx(tx_approve, chain_str, queue, 'cic_eth.eth.token.otx_cache_approve') 
-
-    tx_factory = BancorTxFactory(from_address, c)
-    tx_convert = tx_factory.convert(
-            tokens[0]['address'],
-            tokens[1]['address'],
-            reserve_address,
-            source_amount,
-            minimum_return,
-            chain_spec,
-            )
-    (tx_convert_hash_hex, tx_convert_signed_hex) = sign_and_register_tx(tx_convert, chain_str, queue, 'cic_eth.eth.bancor.otx_cache_convert')
-
-    # TODO: consider moving save recipient to async task / chain it before the tx send
-    if to_address != None:
-        save_convert_recipient(tx_convert_hash_hex, to_address, chain_str)
-
-    s = create_check_gas_and_send_task(
-            [tx_approve_zero_signed_hex, tx_approve_signed_hex, tx_convert_signed_hex],
-            chain_str,
-            from_address,
-            tx_approve_zero['gasPrice'] * tx_approve_zero['gas'],
-            tx_hashes_hex=[tx_approve_hash_hex],
-            queue=queue,
-            )
-    s.apply_async()
-    return tx_convert_hash_hex
-
-
-#@celery_app.task()
-#def process_approval(tx_hash_hex):
-#    t = session.query(TxConvertTransfer).query(TxConvertTransfer.approve_tx_hash==tx_hash_hex).first()
-#    c = session.query(Otx).query(Otx.tx_hash==t.convert_tx_hash)
-#    gas_limit = 8000000
-#    gas_price = GasOracle.gas_price()
-#
-#    # TODO: use celery group instead
-#    s_queue = celery.signature(
-#            'cic_eth.queue.tx.create',
-#            [
-#                nonce,
-#                c['address'], # TODO: check that this is in fact sender address
-#                c['tx_hash'],
-#                c['signed_tx'],
-#                ]
-#            )
-#    s_queue.apply_async()
-#
-#    s_check_gas = celery.signature(
-#            'cic_eth.eth.gas.check_gas',
-#            [
-#                c['address'],
-#                [c['signed_tx']],
-#                gas_limit * gas_price,
-#                ]
-#            )
-#    s_send = celery.signature(
-#            'cic_eth.eth.tx.send',
-#            [],
-#            )
-#
-#    s_set_sent = celery.signature(
-#            'cic_eth.queue.state.set_sent',
-#            [False],
-#            )
-#    s_send.link(s_set_sent)
-#    s_check_gas.link(s_send)
-#    s_check_gas.apply_async()
-#    return tx_hash_hex
-
-
-
-@celery_app.task()
-def save_convert_recipient(convert_hash, recipient_address, chain_str):
-    """Registers the recipient target for a convert-and-transfer operation.
-
-    :param convert_hash: Transaction hash of convert operation
-    :type convert_hash: str, 0x-hex
-    :param recipient_address: Address of consequtive transfer recipient
-    :type recipient_address: str, 0x-hex
-    """
-    session = SessionBase.create_session()
-    t = TxConvertTransfer(convert_hash, recipient_address, chain_str)
-    session.add(t)
-    session.commit()
-    session.close()
-
-
-@celery_app.task()
-def save_convert_transfer(convert_hash, transfer_hash):
-    """Registers that the transfer part of a convert-and-transfer operation has been executed.
-
-    :param convert_hash: Transaction hash of convert operation
-    :type convert_hash: str, 0x-hex
-    :param convert_hash: Transaction hash of transfer operation
-    :type convert_hash: str, 0x-hex
-    :returns: transfer_hash,
-    :rtype: list, single str, 0x-hex
-    """
-    session = SessionBase.create_session()
-    t = TxConvertTransfer.get(convert_hash)
-    t.transfer(transfer_hash)
-    session.add(t)
-    session.commit()
-    session.close()
-    return [transfer_hash]
-
-
-# TODO: seems unused, consider removing
-@celery_app.task()
-def resolve_converters_by_tokens(tokens, chain_str):
-    """Return converters for a list of tokens.
-
-    :param tokens: Token addresses to look up
-    :type tokens: list of str, 0x-hex
-    :return: Addresses of matching converters
-    :rtype: list of str, 0x-hex
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    for t in tokens:
-        c = CICRegistry.get_contract(chain_spec, 'ConverterRegistry')
-        fn = c.function('getConvertersByAnchors')
-        try:
-            converters = fn([t['address']]).call()
-        except Exception as e:
-            raise e
-        t['converters'] = converters
-
-    return tokens
-
-
-@celery_app.task(bind=True)
-def transfer_converted(self, tokens, holder_address, receiver_address, value, tx_convert_hash_hex, chain_str):
-    """Execute the ERC20 transfer of a convert-and-transfer operation.
-
-    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
-
-    :param tokens: Token addresses 
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Token holder address
-    :type holder_address: str, 0x-hex
-    :param holder_address: Token receiver address
-    :type holder_address: str, 0x-hex
-    :param value: Amount of token, in 'wei'
-    :type value: int
-    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
-    :return: Transaction hash
-    :rtype: str, 0x-hex
-    """
-    # we only allow one token, one transfer
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    c = RpcClient(chain_spec, holder_address=holder_address)
-
-    # get transaction parameters
-    gas_price = c.gas_price()
-    tx_factory = TokenTxFactory(holder_address, c)
-
-    token_address = tokens[0]['address']
-    tx_transfer = tx_factory.transfer(
-        token_address,
-        receiver_address,
-        value,
-        chain_spec,
-            )
-    (tx_transfer_hash_hex, tx_transfer_signed_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.token.otx_cache_transfer')
-
-    # send transaction
-    logg.info('transfer converted token {} from {} to {} value {} {}'.format(token_address, holder_address, receiver_address, value, tx_transfer_signed_hex))
-    s = create_check_gas_and_send_task(
-            [tx_transfer_signed_hex],
-            chain_str,
-            holder_address,
-            tx_transfer['gasPrice'] * tx_transfer['gas'],
-            None,
-            queue,
-            )
-    s_save = celery.signature(
-            'cic_eth.eth.bancor.save_convert_transfer',
-            [
-                tx_convert_hash_hex,
-                tx_transfer_hash_hex,
-                ],
-            queue=queue,
-            )
-    s_save.link(s)
-    s_save.apply_async()
-    return tx_transfer_hash_hex
-
-
-@celery_app.task()
-def otx_cache_convert(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = unpack_convert(tx['data'])
-    logg.debug('tx data {}'.format(tx_data))
-
-    session = TxCache.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['from'],
-        tx_data['source_token'],
-        tx_data['destination_token'],
-        tx_data['amount'],
-        tx_data['amount'],
-            )
-    session.add(tx_cache)
-    session.commit()
-    session.close()
-    return tx_hash_hex
-

apps/cic-eth/cic_eth/eth/erc20.py

@@ -13,9 +13,9 @@ from chainlib.eth.tx import (
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
-from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
+from chainqueue.sql.tx import cache_tx_dict
 
 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -375,19 +375,16 @@ def cache_transfer_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -417,19 +414,16 @@ def cache_transfer_from_data(
     token_value = tx_data[2]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
@@ -458,19 +452,16 @@ def cache_approve_data(
     token_value = tx_data[1]
 
     session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        recipient_address,
-        tx['to'],
-        tx['to'],
-        token_value,
-        token_value,
-        session=session,
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    tx_dict = {
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': recipient_address,
+            'source_token': tx['to'],
+            'destination_token': tx['to'],
+            'from_value': token_value,
+            'to_value': token_value,
+            }
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -3,12 +3,16 @@ import logging
 
 # external imports
 import celery
-from hexathon import strip_0x
-from chainlib.eth.constant import ZERO_ADDRESS
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+#from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
+from chainqueue.sql.tx import cache_tx_dict
 from chainlib.eth.gas import (
         balance,
         price,
@@ -20,7 +24,6 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
         TxFactory,
         TxFormat,
-        unpack,
         )
 from chainlib.eth.contract import (
         abi_decode_single,
@@ -44,6 +47,7 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
         queue_create,
         register_tx,
+        unpack,
         )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -52,6 +56,11 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
         CriticalWeb3AndSignerTask,
         )
+from cic_eth.encode import (
+        tx_normalize,
+        ZERO_ADDRESS_NORMAL,
+        unpack_normal,
+        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -65,6 +74,7 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
+#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -129,30 +139,27 @@ def cache_gas_data(
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack(tx_signed_raw_bytes, chain_spec)
+    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
 
     session = SessionBase.create_session()
 
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx['to'],
-        ZERO_ADDRESS,
-        ZERO_ADDRESS,
-        tx['value'],
-        tx['value'],
-        session=session,
-            )
+    tx_dict = {
+            'hash': tx['hash'],
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS_NORMAL,
+            'from_value': tx['value'],
+            'to_value': tx['value'],
+            }
 
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
+    (tx_dict, cache_id) = cache_tx_dict(tx_dict, session=session)
     session.close()
     return (tx_hash_hex, cache_id)
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -172,6 +179,21 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
+    if address != None:
+        if not is_checksum_address(address):
+            raise ValueError('invalid address {}'.format(address))
+        address = tx_normalize.wallet_address(address)
+        address = add_0x(address)
+
+    tx_hashes = []
+    txs = []
+    for tx_hash in tx_hashes_hex:
+        tx_hash = tx_normalize.tx_hash(tx_hash)
+        tx_hashes.append(tx_hash)
+    for tx in txs_hex:
+        tx = tx_normalize.tx_wire(tx)
+        txs.append(tx)
+
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
@@ -189,9 +211,6 @@ def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_requir
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
-    if not is_checksum_address(address):
-        raise ValueError('invalid address {}'.format(address))
-
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
@@ -306,6 +325,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # Determine value of gas tokens to send
     # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
     # however, we will perform a 0-value transaction instead
+    recipient_address = tx_normalize.wallet_address(recipient_address)
     zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -380,6 +400,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     :returns: Transaction hash
     :rtype: str, 0x-hex
     """
+    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
     session = SessionBase.create_session()
 
     otx = Otx.load(txold_hash_hex, session)

apps/cic-eth/cic_eth/eth/tx.py

@@ -18,7 +18,6 @@ from hexathon import (
         strip_0x,
         )
 from chainqueue.db.models.tx import Otx
-from chainqueue.db.models.tx import TxCache
 from chainqueue.db.enum import StatusBits
 from chainqueue.error import NotLocalTxError
 from potaahto.symbols import snake_and_camel

apps/cic-eth/cic_eth/ext/tx.py

@@ -12,15 +12,21 @@ from chainlib.eth.tx import (
         transaction_by_block,
         receipt,
         )
+from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.tx import Tx
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
+from cic_eth_registry.error import UnknownContractError
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
 from eth_erc20 import ERC20
+from erc20_faucet import Faucet
+from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.queue.time import tx_times
@@ -33,6 +39,32 @@ logg = logging.getLogger()
 MAX_BLOCK_TX = 250
 
 
+def parse_transaction(chain_spec, rpc, tx, sender_address=None):
+    try:
+        transfer_data = ERC20.parse_transfer_request(tx['input'])
+        tx_address = transfer_data[0]
+        tx_token_value = transfer_data[1]
+        logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
+        return (tx_address, tx_token_value)
+    except RequestMismatchException:
+        pass
+
+    try:
+        transfer_data = Faucet.parse_give_to_request(tx['input'])
+        tx_address = transfer_data[0]
+        c = Faucet(chain_spec)
+        o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number'])
+        r = rpc.do(o)
+        tx_token_value = Faucet.parse_token_amount(r)
+        logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
+        return (tx_address, tx_token_value)
+
+    except RequestMismatchException:
+        pass
+
+    return None
+
+
 # TODO: Make this method easier to read
 @celery_app.task(bind=True, base=BaseTask)
 def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
@@ -69,36 +101,39 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data)
 
     txs = {}
+    logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high']))
     for block_height in range(bloomspec['low'], bloomspec['high']):
         block_height_bytes = block_height.to_bytes(4, 'big')
         if block_filter.check(block_height_bytes):
             logg.debug('filter matched block {}'.format(block_height))
             o = block_by_number(block_height)
             block = rpc.do(o)
-            logg.debug('block {}'.format(block))
 
             for tx_index in range(0, len(block['transactions'])):
-                composite = tx_index + block_height
-                tx_index_bytes = composite.to_bytes(4, 'big')
-                if tx_filter.check(tx_index_bytes):
+                tx_index_bytes = tx_index.to_bytes(4, 'big')
+                composite = block_height_bytes + tx_index_bytes
+                if tx_filter.check(composite):
                     logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
 
+                    o = transaction_by_block(block['hash'], tx_index)
                     try:
-                        #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
-                        o = transaction_by_block(block['hash'], tx_index)
                         tx = rpc.do(o)
                     except Exception as e:
                         logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e))
                         continue
+
+                    tx = Tx(tx).src()
+
+                    logg.debug('got tx {}'.format(tx))
                     tx_address = None
                     tx_token_value = 0
-                    try:
-                        transfer_data = ERC20.parse_transfer_request(tx['data'])
-                        tx_address = transfer_data[0]
-                        tx_token_value = transfer_data[1]
-                    except ValueError:
-                        logg.debug('not a transfer transaction, skipping {}'.format(tx))
+
+                    transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address)
+                    if transfer_data == None:
                         continue
+                    tx_address = transfer_data[0]
+                    tx_token_value = transfer_data[1]
+                    
                     if address == tx_address:
                         status = StatusEnum.SENT
                         try:
@@ -114,9 +149,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
-                        token_symbol = token.symbol
-                        token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)
                         tx_r = {
                             'hash': tx['hash'],
@@ -126,12 +158,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
                             'destination_value': tx_token_value,
                             'source_token': tx['to'],
                             'destination_token': tx['to'],
-                            'source_token_symbol': token_symbol,
-                            'destination_token_symbol': token_symbol,
-                            'source_token_decimals': token_decimals,
-                            'destination_token_decimals': token_decimals,
-                            'source_token_chain': chain_str,
-                            'destination_token_chain': chain_str,
                             'nonce': tx['nonce'],
                                 }
                         if times['queue'] != None:
@@ -143,11 +169,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     return txs
 
 
+
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task()
-def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
+@celery_app.task(bind=True, base=BaseTask)
+def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=True, verify_contracts=True):
     """Merges transaction data from multiple sources and sorts them in chronological order.
 
     :param tx_batches: Transaction data inputs
@@ -196,6 +223,32 @@ def tx_collate(tx_batches, chain_spec_dict, offset, limit, newest_first=True):
     if newest_first:
         ks.reverse()
     for k in ks:
-        txs.append(txs_by_block[k])
+        tx = txs_by_block[k]
+        if verify_contracts:
+            try:
+                tx = verify_and_expand(tx, chain_spec, sender_address=BaseTask.call_address)
+            except UnknownContractError:
+                logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
+                continue
+        txs.append(tx)
 
     return txs
+
+
+def verify_and_expand(tx, chain_spec, sender_address=ZERO_ADDRESS):
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    registry = CICRegistry(chain_spec, rpc)
+
+    if tx.get('source_token_symbol') == None and tx['source_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['source_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['source_token'])
+        tx['source_token_symbol'] = token.symbol
+        tx['source_token_decimals'] = token.decimals
+
+    if tx.get('destination_token_symbol') == None and tx['destination_token'] != ZERO_ADDRESS:
+        r = registry.by_address(tx['destination_token'], sender_address=sender_address)
+        token = ERC20Token(chain_spec, rpc, tx['destination_token'])
+        tx['destination_token_symbol'] = token.symbol
+        tx['destination_token_decimals'] = token.decimals
+
+    return tx

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -27,7 +27,7 @@ def database_engine(
         SessionBase.poolable = False
     dsn = dsn_from_config(load_config)
     #SessionBase.connect(dsn, True)
-    SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') != None)
+    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
     return dsn
 
 

apps/cic-eth/cic_eth/queue/balance.py

@@ -15,6 +15,7 @@ from chainqueue.db.enum import (
 # local imports
 from cic_eth.db import SessionBase
 from cic_eth.task import CriticalSQLAlchemyTask
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -22,6 +23,9 @@ logg = logging.getLogger()
 
 
 def __balance_outgoing_compatible(token_address, holder_address):
+    token_address = tx_normalize.executable_address(token_address)
+    holder_address = tx_normalize.wallet_address(holder_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.from_value)
     q = q.join(Otx)
@@ -58,6 +62,9 @@ def balance_outgoing(tokens, holder_address, chain_spec_dict):
 
 
 def __balance_incoming_compatible(token_address, receiver_address):
+    token_address = tx_normalize.executable_address(token_address)
+    receiver_address = tx_normalize.wallet_address(receiver_address)
+
     session = SessionBase.create_session()
     q = session.query(TxCache.to_value)
     q = q.join(Otx)
@@ -110,7 +117,7 @@ def assemble_balances(balances_collection):
     logg.debug('received collection {}'.format(balances_collection))
     for c in balances_collection:
         for b in c:
-            address = b['address']
+            address = tx_normalize.executable_address(b['address'])
             if tokens.get(address) == None:
                 tokens[address] = {
                     'address': address,

apps/cic-eth/cic_eth/queue/lock.py

@@ -6,6 +6,7 @@ import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db import SessionBase
 from cic_eth.db.models.lock import Lock
+from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -21,6 +22,9 @@ def get_lock(address=None):
     :returns: List of locks
     :rtype: list of dicts
     """
+    if address != None:
+        address = tx_normalize.wallet_address(address)
+
     session = SessionBase.create_session()
     q = session.query(
             Lock.date_created,