Reinstate skipped admin noncritical tests

Add cov to notify tests
Remove cic-base from cic-notify and cic-ussd
2021-07-21 14:13:56 +02:00 · 2021-07-21 14:11:21 +02:00 · 2021-07-21 13:34:08 +02:00 · 2021-07-21 12:18:45 +02:00 · 2021-07-21 08:49:05 +02:00 · 2021-07-21 08:23:12 +02:00
522 changed files with 8733 additions and 17624 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,3 @@ build/
 **/.venv
 .idea
 **/.vim
-**/*secret.yaml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,43 +1,14 @@
 include:
-  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
-  # these includes are app specific unit tests
+  - local: 'ci_templates/.cic-template.yml'
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
  - local: 'apps/cic-eth/.gitlab-ci.yml'
  - local: 'apps/cic-ussd/.gitlab-ci.yml'
  - local: 'apps/cic-notify/.gitlab-ci.yml'
  - local: 'apps/cic-meta/.gitlab-ci.yml'
  - local: 'apps/cic-cache/.gitlab-ci.yml'
-    #- local: 'apps/contract-migration/.gitlab-ci.yml'
-    #- local: 'apps/data-seeding/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'

 stages:
  - build
  - test
-  - deploy
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
-
-variables:
-  DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1"
-  CI_DEBUG_TRACE: "true"
-
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-
-# runs on protected branches and pushes to repo
-build-push:
-  stage: build
-  tags:
-    - integration
-  #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
-  script:
-    - TAG=latest sh ./scripts/build-push.sh
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-
-deploy-dev:
-  stage: deploy
-  trigger: grassrootseconomics/devops
-  when: manual
+  - release
--- a/README.md
+++ b/README.md
@@ -2,21 +2,25 @@

 ## Getting started 

-This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
+## Make some keys

 ```
-export COMPOSE_DOCKER_CLI_BUILD=1
-export DOCKER_BUILDKIT=1
+docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
 ```

-start services, database, redis and local ethereum node
-```
-docker-compose up -d
-```

-Run app/contract-migration to deploy contracts
+### Prepare the repo
+
+This is stuff we need to put in  makefile but for now...
+
+File mounts and permisssions need to be set
 ```
-RUN_MASK=3 docker-compose up contract-migration
+chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
+````
+
+start cluster
+```
+docker-compose up
 ```

 stop cluster
@@ -24,9 +28,9 @@ stop cluster
 docker-compose down
 ```

-stop cluster and delete data
+delete data
 ```
-docker-compose down -v --remove-orphans
+docker-compose down -v
 ```

 rebuild an images
@@ -34,7 +38,5 @@ rebuild an images
 docker-compose up --build <service_name>
 ```

-to delete the buildkit cache
-```
-docker builder prune --filter type=exec.cachemount
-```
+Deployment variables are writtend to service-configs/.env after everthing is up.
+
--- a/apps/cic-base-os/Dockerfile
+++ b/apps/cic-base-os/Dockerfile
@@ -0,0 +1,34 @@
+# The solc image messes up the alpine environment, so we have to go all over again
+FROM python:3.8.6-slim-buster
+
+LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
+LABEL spdx-license-identifier="GPL-3.0-or-later"
+LABEL description="Base layer for buiding development images for the cic component suite"
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
+
+
+RUN echo installing nodejs tooling
+
+COPY ./dev/nvm.sh /root/
+
+# Install nvm with node and npm
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
+ENV NVM_DIR /root/.nvm
+ENV NODE_VERSION 15.3.0
+ENV BANCOR_NODE_VERSION 10.16.0
+
+RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
+	&& . $NVM_DIR/nvm.sh \
+	&& nvm install $NODE_VERSION \
+	&& nvm alias default $NODE_VERSION \
+	&& nvm use $NODE_VERSION \
+# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
+# owner of these files is "1001" by default - wtf
+	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
+
--- a/apps/cic-base-os/README.md
+++ b/apps/cic-base-os/README.md
@@ -0,0 +1 @@
+## this is an example base image if we wanted one for all the other apps. Its just OS level things
--- a/apps/cic-base-os/aux/wait-for-it/.gitignore
+++ b/apps/cic-base-os/aux/wait-for-it/.gitignore
@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/
--- a/apps/cic-base-os/aux/wait-for-it/.travis.yml
+++ b/apps/cic-base-os/aux/wait-for-it/.travis.yml
@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-
--- a/apps/cic-base-os/aux/wait-for-it/LICENSE
+++ b/apps/cic-base-os/aux/wait-for-it/LICENSE
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
--- a/apps/cic-base-os/aux/wait-for-it/README.md
+++ b/apps/cic-base-os/aux/wait-for-it/README.md
@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST       Host or IP under test
-p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
-s | --strict               Only execute subcommand if the test succeeds
-q | --quiet                Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
-- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
--- a/apps/cic-base-os/aux/wait-for-it/wait-for-it.sh
+++ b/apps/cic-base-os/aux/wait-for-it/wait-for-it.sh
@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi
--- a/apps/cic-cache/.coveragerc
+++ b/apps/cic-cache/.coveragerc
@@ -4,4 +4,3 @@ omit =
 	scripts/*
 	cic_cache/db/migrations/*
 	cic_cache/version.py
-	cic_cache/cli
--- a/apps/cic-cache/.gitlab-ci.yml
+++ b/apps/cic-cache/.gitlab-ci.yml
@@ -1,17 +1,52 @@
-build-test-cic-cache:
-  stage: test 
-  tags:
-    - integration
-  variables:
-    APP_NAME: cic-cache
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-  script:
-    - cd apps/cic-cache
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    changes:
-        - apps/$APP_NAME/**/*
-    when: always
+.cic_cache_variables:
+    variables:
+        APP_NAME: cic-cache
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
+
+build-mr-cic-cache:
+    extends:
+        - .py_build_merge_request
+        - .cic_cache_variables
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+
+test-mr-cic-cache:
+    stage: test
+    extends:
+        - .cic_cache_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-cache"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-cache:
+    extends:
+        - .py_build_push
+        - .cic_cache_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+        
+
--- a/apps/cic-cache/MANIFEST.in
+++ b/apps/cic-cache/MANIFEST.in
@@ -1 +0,0 @@
-include *requirements.txt cic_cache/data/config/*
--- a/apps/cic-cache/README.md
+++ b/apps/cic-cache/README.md
@@ -1 +0,0 @@
-# CIC-CACHE
--- a/apps/cic-cache/aux/wait-for-it/.gitignore
+++ b/apps/cic-cache/aux/wait-for-it/.gitignore
@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/
--- a/apps/cic-cache/aux/wait-for-it/.travis.yml
+++ b/apps/cic-cache/aux/wait-for-it/.travis.yml
@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-
--- a/apps/cic-cache/aux/wait-for-it/LICENSE
+++ b/apps/cic-cache/aux/wait-for-it/LICENSE
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
--- a/apps/cic-cache/aux/wait-for-it/README.md
+++ b/apps/cic-cache/aux/wait-for-it/README.md
@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST       Host or IP under test
-p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
-s | --strict               Only execute subcommand if the test succeeds
-q | --quiet                Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
-- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
--- a/apps/cic-cache/aux/wait-for-it/wait-for-it.sh
+++ b/apps/cic-cache/aux/wait-for-it/wait-for-it.sh
@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi
--- a/apps/cic-cache/cic_cache/api.py
+++ b/apps/cic-cache/cic_cache/api.py
@@ -55,37 +55,15 @@ class Api:
                    queue=callback_queue,
                    )       

-    def list(self, offset=0, limit=100, address=None, oldest=False):
+    def list(self, offset, limit, address=None):
        s = celery.signature(
        'cic_cache.tasks.tx.tx_filter',
        [
-            offset,
-            limit,
+            0,
+            100,
            address,
-            oldest,
            ],
-            queue=self.queue,
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t
-
-
-    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter_content',
-        [
-            offset,
-            limit,
-            address,
-            block_offset,
-            block_limit,
-            oldest,
-            ],
-            queue=self.queue,
+            queue=None
        )
        if self.callback_param != None:
            s.link(self.callback_success).on_error(self.callback_error)
--- a/apps/cic-cache/cic_cache/cache.py
+++ b/apps/cic-cache/cic_cache/cache.py
@@ -10,16 +10,12 @@ from cic_cache.db.list import (
        list_transactions_mined,
        list_transactions_account_mined,
        list_transactions_mined_with_data,
-        list_transactions_mined_with_data_index,
-        list_transactions_account_mined_with_data_index,
-        list_transactions_account_mined_with_data,
        )

 logg = logging.getLogger()


 DEFAULT_FILTER_SIZE = 8192 * 8
-DEFAULT_LIMIT = 100

 class Cache:

@@ -36,7 +32,7 @@ class BloomCache(Cache):
        return n


-    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions(self, offset, limit):
        """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.

        Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -53,7 +49,7 @@ class BloomCache(Cache):
        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
        :rtype: tuple
        """
-        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+        rows = list_transactions_mined(self.session, offset, limit) 

        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -62,12 +58,7 @@ class BloomCache(Cache):
        for r in rows:
            if highest_block == -1:
                highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
            block = r[0].to_bytes(4, byteorder='big')
            tx = r[1].to_bytes(4, byteorder='big')
            f_block.add(block)
@@ -76,7 +67,7 @@ class BloomCache(Cache):
        return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)


-    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions_account(self, address, offset, limit):
        """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.

        :param address: Address to retrieve transactions for.
@@ -88,7 +79,7 @@ class BloomCache(Cache):
        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
        :rtype: tuple
        """
-        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit) 

        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -97,12 +88,7 @@ class BloomCache(Cache):
        for r in rows:
            if highest_block == -1:
                highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
            block = r[0].to_bytes(4, byteorder='big')
            tx = r[1].to_bytes(4, byteorder='big')
            f_block.add(block)
@@ -113,21 +99,8 @@ class BloomCache(Cache):

 class DataCache(Cache):

-    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def __process_rows(self, rows, oldest):
+    def load_transactions_with_data(self, offset, end):
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
        tx_cache = []
        highest_block = -1;
        lowest_block = -1;
@@ -135,12 +108,7 @@ class DataCache(Cache):
        for r in rows:
            if highest_block == -1:
                highest_block = r['block_number']
-                lowest_block = r['block_number']
-            else:
-                if oldest:
-                    highest_block = r['block_number']
-                else:
-                    lowest_block = r['block_number']
+            lowest_block = r['block_number']
            tx_type = 'unknown'

            if r['value'] != None:
--- a/apps/cic-cache/cic_cache/cli/init.py
+++ b/apps/cic-cache/cic_cache/cli/init.py
@@ -1,15 +0,0 @@
-# local imports
-from .base import *
-from .chain import (
-        EthChainInterface,
-        chain_interface,
-        )
-from .rpc import RPC
-from .arg import ArgumentParser
-from .config import Config
-from .celery import CeleryApp
-from .registry import (
-        connect_registry,
-        connect_token_registry,
-        connect_declarator,
-        )
--- a/apps/cic-cache/cic_cache/cli/arg.py
+++ b/apps/cic-cache/cic_cache/cli/arg.py
@@ -1,20 +0,0 @@
-# external imports
-from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
-
-# local imports
-from .base import (
-        CICFlag,
-        Flag,
-        )
-
-
-class ArgumentParser(BaseArgumentParser):
-
-    def process_local_flags(self, local_arg_flags):
-        if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
-        if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
-            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-        if local_arg_flags & CICFlag.CHAIN:
-            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
--- a/apps/cic-cache/cic_cache/cli/base.py
+++ b/apps/cic-cache/cic_cache/cli/base.py
@@ -1,31 +0,0 @@
-# standard imports
-import enum
-
-# external imports
-from chainlib.eth.cli import (
-    argflag_std_read,
-    argflag_std_write,
-    argflag_std_base,
-    Flag,
-    )
-
-class CICFlag(enum.IntEnum):
-   
-    # celery - nibble 1
-    CELERY = 1
-
-    # redis - nibble 2
-#    REDIS = 16
-#    REDIS_CALLBACK = 32
-
-    # chain - nibble 3
-    CHAIN = 256
-
-    # sync - nibble 4
-    SYNCER = 4096
-
-
-argflag_local_task = CICFlag.CELERY 
-#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
-argflag_local_chain = CICFlag.CHAIN
-argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
--- a/apps/cic-cache/cic_cache/cli/celery.py
+++ b/apps/cic-cache/cic_cache/cli/celery.py
@@ -1,24 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-logg = logging.getLogger(__name__)
-
-
-class CeleryApp:
-   
-    @classmethod
-    def from_config(cls, config):
-        backend_url = config.get('CELERY_RESULT_URL')
-        broker_url = config.get('CELERY_BROKER_URL')
-        celery_app = None
-        if backend_url != None:
-            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
-            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
-        else:
-            celery_app = celery.Celery(broker=broker_url)
-            logg.info('creating celery app without results backend on {}'.format(broker_url))
-
-        return celery_app
--- a/apps/cic-cache/cic_cache/cli/chain.py
+++ b/apps/cic-cache/cic_cache/cli/chain.py
@@ -1,21 +0,0 @@
-# external imports
-from chainlib.eth.block import (
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        Tx,
-        )
-from chainlib.interface import ChainInterface
-
-
-class EthChainInterface(ChainInterface):
-    
-    def __init__(self):
-        self._tx_receipt = receipt
-        self._block_by_number = block_by_number
-        self._block_from_src = Block.from_src
-        self._src_normalize = Tx.src_normalize
-
-chain_interface = EthChainInterface()
--- a/apps/cic-cache/cic_cache/cli/config.py
+++ b/apps/cic-cache/cic_cache/cli/config.py
@@ -1,63 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.cli import (
-        Config as BaseConfig,
-        Flag,
-        )
-
-# local imports
-from .base import CICFlag
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-
-logg = logging.getLogger(__name__)
-
-
-class Config(BaseConfig):
-
-    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
-
-    @classmethod
-    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
-        expanded_base_config_dir = [cls.local_base_config_dir]
-        if base_config_dir != None:
-            if isinstance(base_config_dir, str):
-                base_config_dir = [base_config_dir]
-            for d in base_config_dir:
-                expanded_base_config_dir.append(d)
-        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
-
-        local_args_override = {}
-#        if local_arg_flags & CICFlag.REDIS:
-#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
-#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
-#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
-#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
-
-        if local_arg_flags & CICFlag.CHAIN:
-            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
-
-        if local_arg_flags & CICFlag.CELERY:
-            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
-
-        if local_arg_flags & CICFlag.SYNCER:
-            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
-            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
-
-        config.dict_override(local_args_override, 'local cli args')
-
-#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
-        if local_arg_flags & CICFlag.CELERY:
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        logg.debug('config loaded:\n{}'.format(config))
-
-        return config
-
-
--- a/apps/cic-cache/cic_cache/cli/registry.py
+++ b/apps/cic-cache/cic_cache/cli/registry.py
@@ -1,33 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
-from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
-from chainlib.eth.constant import ZERO_ADDRESS
-
-logg = logging.getLogger()
-
-
-def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
-    logg.debug('using token registry address {}'.format(token_registry_address))
-    lookup = TokenIndexLookup(chain_spec, token_registry_address)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    logg.debug('using declarator address {}'.format(declarator_address))
-    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
-    CICRegistry.address = registry_address
-    registry = CICRegistry(chain_spec, conn)
-    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
-    return registry
--- a/apps/cic-cache/cic_cache/cli/rpc.py
+++ b/apps/cic-cache/cic_cache/cli/rpc.py
@@ -1,43 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import EthUnixSignerConnection
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-
-class RPC:
-
-    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
-        self.chain_spec = chain_spec
-        self.rpc_provider = rpc_provider
-        self.signer_provider = signer_provider
-
-
-    def get_default(self):
-        return RPCConnection.connect(self.chain_spec, 'default')
-
-
-    @staticmethod
-    def from_config(config):
-        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
-        if config.get('SIGNER_PROVIDER'):
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
-        logg.info('set up rpc: {}'.format(rpc))
-        return rpc
-
-
-    def __str__(self):
-        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
-
-
-
--- a/apps/cic-cache/cic_cache/data/config/celery.ini
+++ b/apps/cic-cache/cic_cache/data/config/celery.ini
@@ -1,5 +0,0 @@
-[celery]
-broker_url = redis://localhost:6379
-result_url = 
-queue = cic-cache
-debug = 0
--- a/apps/cic-cache/cic_cache/data/config/cic.ini
+++ b/apps/cic-cache/cic_cache/data/config/cic.ini
@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-trust_address =
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
--- a/apps/cic-cache/cic_cache/data/config/database.ini
+++ b/apps/cic-cache/cic_cache/data/config/database.ini
@@ -1,10 +0,0 @@
-[database]
-engine =
-driver =
-host =
-port =
-name = cic-cache
-user =
-password =
-debug = 0
-pool_size = 0
--- a/apps/cic-cache/cic_cache/data/config/signer.ini
+++ b/apps/cic-cache/cic_cache/data/config/signer.ini
@@ -1,2 +0,0 @@
-[signer]
-provider =
--- a/apps/cic-cache/cic_cache/data/config/syncer.ini
+++ b/apps/cic-cache/cic_cache/data/config/syncer.ini
@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0
--- a/apps/cic-cache/cic_cache/db/list.py
+++ b/apps/cic-cache/cic_cache/db/list.py
@@ -13,9 +13,6 @@ def list_transactions_mined(
        session,
        offset,
        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
        ):
    """Executes db query to return all confirmed transactions according to the specified offset and limit.

@@ -26,62 +23,15 @@ def list_transactions_mined(
    :result: Result set
    :rtype: SQLAlchemy.ResultProxy
    """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
+    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
    r = session.execute(s)
    return r


 def list_transactions_mined_with_data(
-        session,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
-
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data_index(
        session,
        offset,
        end,
-        block_offset,
-        block_limit,
-        oldest=False,
        ):
    """Executes db query to return all confirmed transactions according to the specified offset and limit.

@@ -92,87 +42,7 @@ def list_transactions_mined_with_data_index(
    :result: Result set
    :rtype: SQLAlchemy.ResultProxy
    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined_with_data_index(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
-    r = session.execute(s)
-    return r
-
-def list_transactions_account_mined_with_data(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)

    r = session.execute(s)
    return r
@@ -183,9 +53,6 @@ def list_transactions_account_mined(
        address,
        offset,
        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
        ):
    """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.

@@ -198,20 +65,7 @@ def list_transactions_account_mined(
    :result: Result set
    :rtype: SQLAlchemy.ResultProxy
    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-
-    else:
-        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
+    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
    r = session.execute(s)
    return r

--- a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py
+++ b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py
@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.default.export import (
+from chainsyncer.db.migrations.sqlalchemy import (
        chainsyncer_upgrade,
        chainsyncer_downgrade,
        )
--- a/apps/cic-cache/cic_cache/runnable/daemons/query.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py
@@ -91,14 +91,13 @@ def process_transactions_all_data(session, env):
    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
        return None

-    logg.debug('got data request {}'.format(env))
-    block_offset = r[1]
-    block_end = r[2]
+    offset = r[1]
+    end = r[2]
    if int(r[2]) < int(r[1]):
        raise ValueError('cart before the horse, dude')

    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)

    for r in tx_cache:
        r['date_block'] = r['date_block'].timestamp()
--- a/apps/cic-cache/cic_cache/runnable/daemons/server.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/server.py
@@ -8,7 +8,6 @@ import base64
 import confini

 # local imports
-import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.runnable.daemons.query import (
@@ -24,17 +23,26 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_cache', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')

-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_task
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-cache')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()

-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))

-# connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))

--- a/apps/cic-cache/cic_cache/runnable/daemons/tasker.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/tasker.py
@@ -9,7 +9,6 @@ import celery
 import confini

 # local imports
-import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -17,20 +16,35 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()

-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_task
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-cache')
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+
 args = argparser.parse_args()

-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()

 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn)

+# verify database connection with minimal sanity query
+#session = SessionBase.create_session()
+#session.execute('select version_num from alembic_version')
+#session.close()
+
 # set up celery
 current_app = celery.Celery(__name__)

@@ -73,9 +87,9 @@ def main():
    elif args.v:
        argv.append('--loglevel=INFO')
    argv.append('-Q')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
    argv.append('-n')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)

    current_app.worker_main(argv)

--- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py
+++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py
@@ -8,7 +8,15 @@ import sys
 import re

 # external imports
+import confini
+import celery
 import sqlalchemy
+import rlp
+import cic_base.config
+import cic_base.log
+import cic_base.argparse
+import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -26,7 +34,6 @@ from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase

 # local imports
-import cic_cache.cli
 from cic_cache.db import (
        dsn_from_config,
        add_tag,
@@ -36,36 +43,32 @@ from cic_cache.runnable.daemons.filters import (
        FaucetFilter,
        )

-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
+script_dir = os.path.realpath(os.path.dirname(__file__))

-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_sync
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
+def add_block_args(argparser):
+    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
+    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
+    return argparser

-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)

-# connect to database
+logg = cic_base.log.create()
+argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
+argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
+args = cic_base.argparse.parse(argparser, logg)
+config = cic_base.config.create(args.c, args, args.env_prefix)
+
+config.add(args.history_start, 'SYNCER_HISTORY_START', True)
+config.add(args.no_history, '_NO_HISTORY', True)
+
+cic_base.config.log(config)
+
 dsn = dsn_from_config(config)
+
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))

-# set up rpc
-rpc = cic_cache.cli.RPC.from_config(config)
-conn = rpc.get_default()
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))

-# set up chain provisions
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-registry = None
-try:
-    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
+cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))


 def register_filter_tags(filters, session):
@@ -92,12 +95,14 @@ def main():

    syncers = []

+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
    syncer_backends = SQLBackend.resume(chain_spec, block_offset)

    if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_OFFSET')
+        initial_block_start = config.get('SYNCER_HISTORY_START')
        initial_block_offset = block_offset
-        if config.get('SYNCER_NO_HISTORY'):
+        if config.get('_NO_HISTORY'):
            initial_block_start = block_offset
            initial_block_offset += 1
        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
@@ -107,10 +112,10 @@ def main():
            logg.info('resuming sync session {}'.format(syncer_backend))

    for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))

    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))

    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
    if trusted_addresses_src == None:
--- a/apps/cic-cache/cic_cache/tasks/tx.py
+++ b/apps/cic-cache/cic_cache/tasks/tx.py
@@ -2,17 +2,14 @@
 import celery

 # local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-        )
+from cic_cache.cache import BloomCache
 from cic_cache.db.models.base import SessionBase

 celery_app = celery.current_app


@celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, encoding='hex'):
    queue = self.request.delivery_info.get('routing_key')

    session = SessionBase.create_session()
@@ -20,9 +17,9 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
    c = BloomCache(session)
    b = None
    if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
    else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)

    session.close()

@@ -38,17 +35,4 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
    return o


-@celery_app.task(bind=True)
-def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
-    session = SessionBase.create_session()

-    c = DataCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
-    else:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
-
-    session.close()
-
-    return (lowest_block, highest_block, tx_cache,)
--- a/apps/cic-cache/cic_cache/version.py
+++ b/apps/cic-cache/cic_cache/version.py
@@ -4,7 +4,7 @@ import semver
 version = (
        0,
        2,
-        1,
+        0,
        'alpha.2',
        )

--- a/apps/cic-cache/config/bancor.ini
+++ b/apps/cic-cache/config/bancor.ini
@@ -0,0 +1,2 @@
+[bancor]
+dir =
--- a/apps/cic-cache/config/cic.ini
+++ b/apps/cic-cache/config/cic.ini
@@ -1,3 +1,4 @@
 [cic]
 registry_address = 
+chain_spec =
 trust_address = 
--- a/apps/cic-cache/config/docker/bancor.ini
+++ b/apps/cic-cache/config/docker/bancor.ini
@@ -0,0 +1,3 @@
+[bancor]
+registry_address = 
+dir = /usr/local/share/bancor
--- a/apps/cic-cache/config/docker/cic.ini
+++ b/apps/cic-cache/config/docker/cic.ini
@@ -1,3 +1,4 @@
 [cic]
+chain_spec =
 registry_address =  
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
--- a/apps/cic-cache/config/docker/eth.ini
+++ b/apps/cic-cache/config/docker/eth.ini
@@ -0,0 +1,2 @@
+[eth]
+provider = http://localhost:63545
--- a/apps/cic-cache/config/docker/syncer.ini
+++ b/apps/cic-cache/config/docker/syncer.ini
@@ -1,4 +1,3 @@
 [syncer]
 loop_interval = 1
-offset = 0
-no_history = 0
+history_start = 0
--- a/apps/cic-cache/config/eth.ini
+++ b/apps/cic-cache/config/eth.ini
@@ -0,0 +1,2 @@
+[eth]
+provider = ws://localhost:8545
--- a/apps/cic-cache/config/syncer.ini
+++ b/apps/cic-cache/config/syncer.ini
@@ -0,0 +1,3 @@
+[syncer]
+loop_interval = 5
+history_start = 0
--- a/apps/cic-cache/docker/Dockerfile
+++ b/apps/cic-cache/docker/Dockerfile
@@ -1,20 +1,18 @@
-ARG DOCKER_REGISTRY=registry.gitlab.com/grassrootseconomics
-
-FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9

 COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]

-RUN apt-get install libffi-dev -y
-
-ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG EXTRA_PIP_ARGS=""
-ARG PIP_INDEX_URL="https://pypi.org/simple"
-
-RUN    pip install --index-url $PIP_INDEX_URL \
-    --pre \
-    --force-reinstall \
-    --no-cache \
-    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
    -r requirements.txt

 COPY . . 
@@ -24,10 +22,10 @@ RUN python setup.py install
 # ini files in config directory defines the configurable parameters for the application
 # they can all be overridden by environment variables
 # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-#COPY config/ /usr/local/etc/cic-cache/
+COPY config/ /usr/local/etc/cic-cache/

 # for db migrations
-COPY ./aux/wait-for-it/wait-for-it.sh ./
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
 COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/

 COPY /docker/start_tracker.sh ./start_tracker.sh
--- a/apps/cic-cache/docker/Dockerfile_ci
+++ b/apps/cic-cache/docker/Dockerfile_ci
@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
+
+# for db migrations
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
+# Tracker
+# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
+# Server
+# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []
--- a/apps/cic-cache/docker/run_tests.sh
+++ b/apps/cic-cache/docker/run_tests.sh
@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-
--- a/apps/cic-cache/requirements.txt
+++ b/apps/cic-cache/requirements.txt
@@ -1,14 +1,13 @@
+cic-base~=0.2.0a4
 alembic==1.4.2
-confini>=0.3.6rc4,<0.5.0
+confini>=0.3.6rc3,<0.5.0
 uwsgi==2.0.19.1
-moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a1
+moolb~=0.1.0
+cic-eth-registry~=0.5.6a2
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a3,<0.1.0
-erc20-faucet>=0.3.2a2, <0.4.0
-chainlib-eth>=0.0.9a14,<0.1.0
-eth-address-index>=0.2.3a4,<0.3.0
+chainsyncer[sql]~=0.0.3a5
+erc20-faucet~=0.2.2a2
--- a/apps/cic-cache/setup.cfg
+++ b/apps/cic-cache/setup.cfg
@@ -23,13 +23,11 @@ licence_files =

 [options]
 python_requires = >= 3.6
-include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
-	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
@@ -41,4 +39,3 @@ console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
-	cic-cache-list = cic_cache.runable.list:main
--- a/apps/cic-cache/test_requirements.txt
+++ b/apps/cic-cache/test_requirements.txt
@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.7a1
-erc20-transfer-authorization>=0.3.5a1,<0.4.0
+cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.4a1
--- a/apps/cic-cache/tests/cli/test_cli_args.py
+++ b/apps/cic-cache/tests/cli/test_cli_args.py
@@ -1,40 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_cache.cli
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
-
-
-def test_argumentparserto_config():
-
-    argparser = cic_cache.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-
--- a/apps/cic-cache/tests/cli/test_cli_celery.py
+++ b/apps/cic-cache/tests/cli/test_cli_celery.py
@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_cache.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_cache.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_cache.cli.CeleryApp.from_config(config)
--- a/apps/cic-cache/tests/cli/test_cli_chain.py
+++ b/apps/cic-cache/tests/cli/test_cli_chain.py
@@ -1,68 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-from chainlib.eth.pytest.fixtures_chain import default_chain_spec
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from hexathon import add_0x
-
-# local imports
-import cic_cache.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        ):
-    ifc = cic_cache.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    recipient = add_0x(os.urandom(20).hex())
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-
--- a/apps/cic-cache/tests/conftest.py
+++ b/apps/cic-cache/tests/conftest.py
@@ -64,6 +64,7 @@ def txs(
        dt.timestamp(),
            )

+
    tx_number = 42
    tx_hash_second = '0x' + os.urandom(32).hex()
    tx_signed_second = '0x' + os.urandom(128).hex()
@@ -92,44 +93,6 @@ def txs(
            ]


-@pytest.fixture(scope='function')
-def more_txs(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        ):
-
-    session = init_database
-
-    tx_number = 666
-    tx_hash = '0x' + os.urandom(32).hex()
-    tx_signed = '0x' + os.urandom(128).hex()
-    nonce = 3
-
-    dt = datetime.datetime.utcnow()
-    dt += datetime.timedelta(hours=1)
-    db.add_transaction(
-        session,
-        tx_hash,
-        list_defaults['block']+2,
-        tx_number,
-        list_actors['alice'],
-        list_actors['diane'],
-        list_tokens['bar'],
-        list_tokens['bar'],
-        2048,
-        4096,
-        False,
-        dt.timestamp(),
-        )
-
-    session.commit()
-
-    return [tx_hash] + txs
-
-
@pytest.fixture(scope='function')
 def tag_txs(
        init_database,
--- a/apps/cic-cache/tests/test_cache.py
+++ b/apps/cic-cache/tests/test_cache.py
@@ -8,7 +8,6 @@ import json
 import pytest

 # local imports
-from cic_cache import db
 from cic_cache import BloomCache
 from cic_cache.cache import DataCache

@@ -19,6 +18,7 @@ def test_cache(
        init_database,
        list_defaults,
        list_actors,
+        list_tokens,
        txs,
        ):

@@ -37,6 +37,9 @@ def test_cache(

 def test_cache_data(
        init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
        txs,
        tag_txs,
        ):
@@ -44,209 +47,10 @@ def test_cache_data(
    session = init_database

    c = DataCache(session)
-    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
+    b = c.load_transactions_with_data(410000, 420000)

    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[0]
-    assert b[2][0]['tx_type'] == 'unknown'
-    assert b[2][1]['tx_type'] == 'test.taag'
-
-
-def test_cache_ranges(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = BloomCache(session)
-    b = c.load_transactions(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    # now check when supplying account
-    b = c.load_transactions_account(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    # add block filter to the mix
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
+    assert b[2][0]['tx_hash'] == txs[1]
+    assert b[2][1]['tx_type'] == 'unknown'
+    assert b[2][0]['tx_type'] == 'test.taag'
    
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-
-
-def test_cache_ranges_data( 
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = DataCache(session)
-
-    b = c.load_transactions_with_data(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    # now check when supplying account
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    # add block filter to the mix
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[2]
--- a/apps/cic-cache/tests/testdata/config/test.ini
+++ b/apps/cic-cache/tests/testdata/config/test.ini
@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy
--- a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/init.py
+++ b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/init.py
@@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp):


 def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CHAIN_SPEC')
+    chain_spec_str = config.get('CIC_CHAIN_SPEC')
    chain_spec = ChainSpec.from_chain_str(chain_spec_str)
    token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
    
--- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt
+++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt
@@ -1,5 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.5a3
-cic-eth-registry~=0.6.1a6
-chainlib~=0.0.9rc1
-cic_eth~=0.12.4a11
+erc20-demurrage-token~=0.0.2a3
+cic-eth-registry~=0.5.6a1
+chainlib~=0.0.5a1
+cic_eth~=0.12.0a2
--- a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg
+++ b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = cic-eth-aux-erc20-demurrage-token
-version = 0.0.2a7
+version = 0.0.2a4
 description = cic-eth tasks supporting erc20 demurrage token
 author = Louis Holbrook
 author_email = dev@holbrook.no
--- a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt
+++ b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt
@@ -5,7 +5,8 @@ pytest-cov==2.10.1
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
 SQLAlchemy==1.3.20
+cic-eth~=0.12.0a1
 liveness~=0.0.1a7
-eth-accounts-index==0.1.1a1
-eth-contract-registry==0.5.8a1
-eth-address-index==0.2.1a1
+eth-accounts-index==0.0.12a1
+eth-contract-registry==0.5.6a1
+eth-address-index==0.1.2a1
--- a/apps/cic-eth/.coveragerc
+++ b/apps/cic-eth/.coveragerc
@@ -6,5 +6,4 @@ omit =
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
 	cic_eth/queue/state.py
-	cic_eth/cli
 	*redis*.py
--- a/apps/cic-eth/.gitlab-ci.yml
+++ b/apps/cic-eth/.gitlab-ci.yml
@@ -1,16 +1,52 @@
-build-test-cic-eth:
-  stage: test
-  tags:
-    - integration
-  variables:
-    APP_NAME: cic-eth
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-  script:
-    - cd apps/cic-eth
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-  rules:
+.cic_eth_variables:
+    variables:
+        APP_NAME: cic-eth
+        DOCKERFILE_PATH: docker/Dockerfile_ci
+        CONTEXT: apps/$APP_NAME
+
+build-mr-cic-eth:
+    extends:
+        - .cic_eth_variables
+        - .py_build_target_dev
+    rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
-          - apps/$APP_NAME/**/*
+          - apps/cic-eth/**/*
+      when: always
+
+test-mr-cic-eth:
+    stage: test
+    extends:
+        - .cic_eth_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r admin_requirements.txt
+          -r services_requirements.txt
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
+
+build-push-cic-eth:
+    extends:
+        - .py_build_push
+        - .cic_eth_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-eth/**/*
      when: always
--- a/apps/cic-eth/MANIFEST.in
+++ b/apps/cic-eth/MANIFEST.in
@@ -1,2 +1,2 @@
-include *requirements.txt config/test/* cic_eth/data/config/*
+include *requirements.txt config/test/*

--- a/apps/cic-eth/admin_requirements.txt
+++ b/apps/cic-eth/admin_requirements.txt
@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a6,<0.7.0
-hexathon~=0.0.1a8
-chainqueue>=0.0.4a6,<0.1.0
-eth-erc20>=0.1.2a2,<0.2.0
+cic-eth-registry>=0.5.6a2,<0.6.0
+hexathon~=0.0.1a7
+chainqueue>=0.0.3a1,<0.1.0
+eth-erc20>=0.0.10a3,<0.1.0
--- a/apps/cic-eth/cic_eth/admin/ctrl.py
+++ b/apps/cic-eth/cic_eth/admin/ctrl.py
@@ -4,6 +4,7 @@ import logging

 # external imports
 import celery
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from hexathon import (
        add_0x,
@@ -19,17 +20,18 @@ from cic_eth.task import (
        CriticalSQLAlchemyTask,
        )
 from cic_eth.error import LockedError
-from cic_eth.encode import (
-        tx_normalize,
-        ZERO_ADDRESS_NORMAL,
-        )

 celery_app = celery.current_app
 logg = logging.getLogger()


+def normalize_address(a):
+    if a == None:
+        return None
+    return add_0x(hex_uniform(strip_0x(a)))
+
@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
    """Task wrapper to set arbitrary locks

    :param chain_str: Chain spec string representation
@@ -41,7 +43,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = '::'
    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -51,7 +53,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock


@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
    """Task wrapper to reset arbitrary locks

    :param chain_str: Chain spec string representation
@@ -63,7 +65,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = '::'
    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -73,7 +75,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo


@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
    """Task wrapper to set send lock

    :param chain_str: Chain spec string representation
@@ -83,7 +85,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
    logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -91,7 +93,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha


@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
    """Task wrapper to reset send lock

    :param chain_str: Chain spec string representation
@@ -101,7 +103,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.reset(chain_str, LockEnum.SEND, address=address)
    logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -109,7 +111,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):


@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
    """Task wrapper to set queue direct lock

    :param chain_str: Chain spec string representation
@@ -119,7 +121,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
    logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -127,7 +129,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h


@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
    """Task wrapper to reset queue direct lock

    :param chain_str: Chain spec string representation
@@ -137,7 +139,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
    :returns: New lock state for address
    :rtype: number
    """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
    logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -146,13 +148,12 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):

@celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    if address != None:
-        address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
    chain_str = '::'
    if chain_spec_dict != None:
        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
    session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
    if address != None:
        r |= Lock.check(chain_str, lock_flags, address=address, session=session)
    if r > 0:
--- a/apps/cic-eth/cic_eth/admin/nonce.py
+++ b/apps/cic-eth/cic_eth/admin/nonce.py
@@ -33,7 +33,6 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
-from cic_eth.encode import tx_normalize

 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -74,7 +73,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):

    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)

-    query_address = tx_normalize.wallet_address(address)
+    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
    q = session.query(Otx)
    q = q.join(TxCache)
    q = q.filter(TxCache.sender==query_address)
--- a/apps/cic-eth/cic_eth/admin/token.py
+++ b/apps/cic-eth/cic_eth/admin/token.py
@@ -1,2 +1,21 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
 # local imports
-from cic_eth.eth.erc20 import default_token
+from cic_eth.task import BaseTask
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+            'symbol': self.default_token_symbol,
+            'address': self.default_token_address,
+            'name': self.default_token_name,
+            'decimals': self.default_token_decimals,
+            }
--- a/apps/cic-eth/cic_eth/api/admin.py
+++ b/apps/cic-eth/cic_eth/api/admin.py
@@ -32,6 +32,7 @@ from chainqueue.db.enum import (
        status_str,
    )
 from chainqueue.error import TxStateChangeError
+from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20

 # local imports
@@ -39,7 +40,6 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
-from cic_eth.queue.query import get_tx_local

 app = celery.current_app

@@ -284,7 +284,7 @@ class AdminApi:
        tx_hash_hex = None
        session = SessionBase.create_session()
        for k in txs.keys():
-            tx_dict = get_tx_local(chain_spec, k, session=session)
+            tx_dict = get_tx(chain_spec, k, session=session)
            if tx_dict['nonce'] == nonce:
                tx_hash_hex = k
        session.close()
--- a/apps/cic-eth/cic_eth/api/api_task.py
+++ b/apps/cic-eth/cic_eth/api/api_task.py
@@ -9,7 +9,6 @@ import logging
 # external imports 
 import celery
 from chainlib.chain import ChainSpec
-from hexathon import strip_0x

 # local imports
 from cic_eth.api.base import ApiBase
@@ -17,50 +16,15 @@ from cic_eth.enum import LockEnum

 app = celery.current_app

-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)


 class Api(ApiBase):
-
-    @staticmethod
-    def to_v_list(v, n):
-        """Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
-
-        :param v: Arguments
-        :type v: str or list
-        :param n: Number of elements to generate arguments for
-        :type n: int
-        :rtype: list
-        :returns: list of assembled arguments
-        """
-        if isinstance(v, str):
-            vv = v
-            v = []
-            for i in range(n):
-                v.append([vv])
-        elif not isinstance(v, list):
-            raise ValueError('argument must be single string, or list or strings or lists')
-        else:
-            if len(v) != n:
-                raise ValueError('v argument count must match integer n')
-            for i in range(n):
-                if isinstance(v[i], str):
-                    v[i] = [v[i]]
-                elif not isinstance(v, list):
-                    raise ValueError('proof argument must be single string, or list or strings or lists')
-
-        return v
-   
+    

    def default_token(self):
-        """Retrieves the default fallback token of the custodial network.
-
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
        s_token = celery.signature(
-                'cic_eth.eth.erc20.default_token',
+                'cic_eth.admin.token.default_token',
                [],
                queue=self.queue,
                )
@@ -70,97 +34,6 @@ class Api(ApiBase):
        return s_token.apply_async()


-    def token(self, token_symbol, proof=None):
-        """Single-token alias for tokens method.
-
-        See tokens method for details.
-
-        :param token_symbol: Token symbol to look up
-        :type token_symbol: str
-        :param proof: Proofs to add to signature verification for the token
-        :type proof: str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbol, str):
-            raise ValueError('token symbol must be string')
-
-        return self.tokens([token_symbol], proof=proof)
-
-
-    def tokens(self, token_symbols, proof=None):
-        """Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
-
-        Token symbols are always strings, and should be specified using uppercase letters.
-
-        If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
-
-        The proof argument may be specified in a number of ways:
-
-        - as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
-        - as a single string, where the same proof is used for each token lookup
-        - as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
-        - as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
-
-        The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
-
-        This method is not intended to be used synchronously. Do so at your peril.
-
-        :param token_symbols: Token symbol strings to look up
-        :type token_symbol: list
-        :param proof: Proof(s) to verify tokens against
-        :type proof: None, str or list
-        :returns: uuid of root task
-        :rtype: celery.Task
-        """
-        if not isinstance(token_symbols, list):
-            raise ValueError('token symbols argument must be list')
-
-        if proof == None:
-            logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
-            proof = ''
-
-        logg.debug('proof is {}'.format(proof))
-        l = len(token_symbols)
-        if len(proof) == 0:
-            l = 0 
-        proof = Api.to_v_list(proof, l)
-
-        chain_spec_dict = self.chain_spec.asdict()
-
-        s_token_resolve = celery.signature(
-                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
-                [
-                    token_symbols,
-                    chain_spec_dict,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_info = celery.signature(
-                'cic_eth.eth.erc20.token_info',
-                [
-                    chain_spec_dict,
-                    proof,
-                    ],
-                queue=self.queue,
-                )
-
-        s_token_verify = celery.signature(
-                    'cic_eth.eth.erc20.verify_token_info',
-                    [
-                        chain_spec_dict,
-                        self.callback_success,
-                        self.callback_error,
-                        ],
-                    queue=self.queue,
-                    )
-
-        s_token_info.link(s_token_verify)
-        s_token_resolve.link(s_token_info)
-        return s_token_resolve.apply_async()
-
-
 #    def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
 #        """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 #
@@ -381,8 +254,6 @@ class Api(ApiBase):
        :returns: uuid of root task
        :rtype: celery.Task
        """
-        #from_address = strip_0x(from_address)
-        #to_address = strip_0x(to_address)
        s_check = celery.signature(
                'cic_eth.admin.ctrl.check_lock',
                [
@@ -649,9 +520,9 @@ class Api(ApiBase):
            s_external_get = celery.signature(
                external_task,
                [
+                    address,
                    offset,
                    limit,
-                    address,
                    ],
                queue=external_queue,
                )
--- a/apps/cic-eth/cic_eth/callbacks/noop.py
+++ b/apps/cic-eth/cic_eth/callbacks/noop.py
@@ -1,10 +1,7 @@
-import logging
-
 import celery

 celery_app = celery.current_app
-#logg = celery_app.log.get_default_logger()
-logg = logging.getLogger()
+logg = celery_app.log.get_default_logger()


@celery_app.task(bind=True)
--- a/apps/cic-eth/cic_eth/check/gas.py
+++ b/apps/cic-eth/cic_eth/check/gas.py
@@ -21,7 +21,7 @@ def health(*args, **kwargs):
    session = SessionBase.create_session()

    config = kwargs['config']
-    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
    logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))

    try:
--- a/apps/cic-eth/cic_eth/check/signer.py
+++ b/apps/cic-eth/cic_eth/check/signer.py
@@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__)
 def health(*args, **kwargs):
    blocked = True
    max_attempts = 5
-    conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer')
+    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
    for i in range(max_attempts):
        idx = i + 1
        logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
--- a/apps/cic-eth/cic_eth/cli/init.py
+++ b/apps/cic-eth/cic_eth/cli/init.py
@@ -1,10 +0,0 @@
-# local imports
-from .base import *
-from .chain import (
-        EthChainInterface,
-        chain_interface,
-        )
-from .rpc import RPC
-from .arg import ArgumentParser
-from .config import Config
-from .celery import CeleryApp
--- a/apps/cic-eth/cic_eth/cli/arg.py
+++ b/apps/cic-eth/cic_eth/cli/arg.py
@@ -1,31 +0,0 @@
-# external imports
-from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
-
-# local imports
-from .base import (
-        CICFlag,
-        Flag,
-        )
-
-
-class ArgumentParser(BaseArgumentParser):
-
-    def process_local_flags(self, local_arg_flags):
-        if local_arg_flags & CICFlag.REDIS:
-            self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
-            self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
-            self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-            self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
-            self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
-        if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
-        if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
-            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-        if local_arg_flags & CICFlag.CHAIN:
-            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
-
-
-
--- a/apps/cic-eth/cic_eth/cli/base.py
+++ b/apps/cic-eth/cic_eth/cli/base.py
@@ -1,31 +0,0 @@
-# standard imports
-import enum
-
-# external imports
-from chainlib.eth.cli import (
-    argflag_std_read,
-    argflag_std_write,
-    argflag_std_base,
-    Flag,
-    )
-
-class CICFlag(enum.IntEnum):
-   
-    # celery - nibble 1
-    CELERY = 1
-
-    # redis - nibble 2
-    REDIS = 16
-    REDIS_CALLBACK = 32
-
-    # chain - nibble 3
-    CHAIN = 256
-
-    # sync - nibble 4
-    SYNCER = 4096
-
-
-argflag_local_task = CICFlag.CELERY 
-argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
-argflag_local_chain = CICFlag.CHAIN
-argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
--- a/apps/cic-eth/cic_eth/cli/celery.py
+++ b/apps/cic-eth/cic_eth/cli/celery.py
@@ -1,24 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-logg = logging.getLogger(__name__)
-
-
-class CeleryApp:
-   
-    @classmethod
-    def from_config(cls, config):
-        backend_url = config.get('CELERY_RESULT_URL')
-        broker_url = config.get('CELERY_BROKER_URL')
-        celery_app = None
-        if backend_url != None:
-            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
-            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
-        else:
-            celery_app = celery.Celery(broker=broker_url)
-            logg.info('creating celery app without results backend on {}'.format(broker_url))
-
-        return celery_app
--- a/apps/cic-eth/cic_eth/cli/chain.py
+++ b/apps/cic-eth/cic_eth/cli/chain.py
@@ -1,21 +0,0 @@
-# external imports
-from chainlib.eth.block import (
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        Tx,
-        )
-from chainlib.interface import ChainInterface
-
-
-class EthChainInterface(ChainInterface):
-    
-    def __init__(self):
-        self._tx_receipt = receipt
-        self._block_by_number = block_by_number
-        self._block_from_src = Block.from_src
-        self._src_normalize = Tx.src_normalize
-
-chain_interface = EthChainInterface()
--- a/apps/cic-eth/cic_eth/cli/config.py
+++ b/apps/cic-eth/cic_eth/cli/config.py
@@ -1,63 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.cli import (
-        Config as BaseConfig,
-        Flag,
-        )
-
-# local imports
-from .base import CICFlag
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-
-logg = logging.getLogger(__name__)
-
-
-class Config(BaseConfig):
-
-    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
-
-    @classmethod
-    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
-        expanded_base_config_dir = [cls.local_base_config_dir]
-        if base_config_dir != None:
-            if isinstance(base_config_dir, str):
-                base_config_dir = [base_config_dir]
-            for d in base_config_dir:
-                expanded_base_config_dir.append(d)
-        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
-
-        local_args_override = {}
-        if local_arg_flags & CICFlag.REDIS:
-            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
-            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
-            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
-            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
-
-        if local_arg_flags & CICFlag.CHAIN:
-            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
-
-        if local_arg_flags & CICFlag.CELERY:
-            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
-
-        if local_arg_flags & CICFlag.SYNCER:
-            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
-            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
-
-        config.dict_override(local_args_override, 'local cli args')
-
-        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
-        if local_arg_flags & CICFlag.CELERY:
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        logg.debug('config loaded:\n{}'.format(config))
-
-        return config
-
-
--- a/apps/cic-eth/cic_eth/cli/rpc.py
+++ b/apps/cic-eth/cic_eth/cli/rpc.py
@@ -1,90 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import (
-        EthUnixSignerConnection,
-        EthHTTPSignerConnection,
-        )
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-
-class RPC:
-
-    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
-        self.chain_spec = chain_spec
-        self.rpc_provider = rpc_provider
-        self.signer_provider = signer_provider
-
-
-    def get_default(self):
-        return self.get_by_label('default')
-
-
-    def get_by_label(self, label):
-        return RPCConnection.connect(self.chain_spec, label)
-
-
-    @staticmethod
-    def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
-        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
-        if use_signer:
-
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
-            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
-            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
-        logg.info('set up rpc: {}'.format(rpc))
-        return rpc
-
-
-    def __str__(self):
-        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
-
-
-# TOOD: re-implement file backend option from omittec code:
-#broker = config.get('CELERY_BROKER_URL')
-#if broker[:4] == 'file':
-#    bq = tempfile.mkdtemp()
-#    bp = tempfile.mkdtemp()
-#    conf_update = {
-#            'broker_url': broker,
-#            'broker_transport_options': {
-#                'data_folder_in': bq,
-#                'data_folder_out': bq,
-#                'data_folder_processed': bp,
-#            },
-#            }
-#    if config.true('CELERY_DEBUG'):
-#        conf_update['result_extended'] = True
-#    current_app.conf.update(conf_update)
-#    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
-#else:
-#    conf_update = {
-#            'broker_url': broker,
-#            }
-#    if config.true('CELERY_DEBUG'):
-#        conf_update['result_extended'] = True
-#    current_app.conf.update(conf_update)
-#
-#result = config.get('CELERY_RESULT_URL')
-#if result[:4] == 'file':
-#    rq = tempfile.mkdtemp()
-#    current_app.conf.update({
-#        'result_backend': 'file://{}'.format(rq),
-#        })
-#    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
-#else:
-#    current_app.conf.update({
-#        'result_backend': result,
-#        })
-#
--- a/apps/cic-eth/cic_eth/data/config/celery.ini
+++ b/apps/cic-eth/cic_eth/data/config/celery.ini
@@ -1,5 +0,0 @@
-[celery]
-broker_url = redis://localhost:6379
-result_url = 
-queue = cic-eth
-debug = 0
--- a/apps/cic-eth/cic_eth/data/config/database.ini
+++ b/apps/cic-eth/cic_eth/data/config/database.ini
@@ -1,10 +0,0 @@
-[database]
-engine =
-driver =
-host =
-port =
-name =
-user =
-password =
-debug = 0
-pool_size = 0
--- a/apps/cic-eth/cic_eth/data/config/retry.ini
+++ b/apps/cic-eth/cic_eth/data/config/retry.ini
@@ -1,3 +0,0 @@
-[retry]
-delay =
-batch_size =
--- a/apps/cic-eth/cic_eth/data/config/signer.ini
+++ b/apps/cic-eth/cic_eth/data/config/signer.ini
@@ -1,2 +0,0 @@
-[signer]
-provider =
--- a/apps/cic-eth/cic_eth/data/config/syncer.ini
+++ b/apps/cic-eth/cic_eth/data/config/syncer.ini
@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0
--- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
+++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
@@ -8,8 +8,8 @@ Create Date: 2021-04-02 18:41:20.864265
 import datetime
 from alembic import op
 import sqlalchemy as sa
+from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth.db.enum import LockEnum
-from cic_eth.encode import ZERO_ADDRESS_NORMAL


 # revision identifiers, used by Alembic.
@@ -30,7 +30,7 @@ def upgrade():
            sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
            )
    op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
-    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS_NORMAL, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))


 def downgrade():
--- a/apps/cic-eth/cic_eth/db/models/lock.py
+++ b/apps/cic-eth/cic_eth/db/models/lock.py
@@ -4,12 +4,12 @@ import logging

 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx

 # local imports
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode  import ZERO_ADDRESS_NORMAL

 logg = logging.getLogger()

@@ -37,7 +37,7 @@ class Lock(SessionBase):


    @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
        """Sets flags associated with the given address and chain.

        If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):


    @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
        """Resets flags associated with the given address and chain.

        If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):


    @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
        """Checks whether all given flags are set for given address and chain. 

        Does not validate the address against any other tables or components.
--- a/apps/cic-eth/cic_eth/encode.py
+++ b/apps/cic-eth/cic_eth/encode.py
@@ -1,16 +0,0 @@
-# external imports
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainqueue.encode import TxHexNormalizer
-from chainlib.eth.tx import unpack
-
-tx_normalize = TxHexNormalizer()
-
-ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
-
-
-def unpack_normal(signed_tx_bytes, chain_spec):
-    tx = unpack(signed_tx_bytes, chain_spec)
-    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
-    tx['from'] = tx_normalize.wallet_address(tx['from'])
-    tx['to'] = tx_normalize.wallet_address(tx['to'])
-    return tx
--- a/apps/cic-eth/cic_eth/error.py
+++ b/apps/cic-eth/cic_eth/error.py
@@ -48,6 +48,8 @@ class RoleMissingError(Exception):
    pass


+
+
 class IntegrityError(Exception):
    """Exception raised to signal irregularities with deduplication and ordering of tasks

@@ -83,8 +85,3 @@ class RoleAgencyError(SeppukuError):
 class YouAreBrokeError(Exception):
    """Exception raised when a value transfer is attempted without access to sufficient funds
    """
-
-
-class TrustError(Exception):
-    """Exception raised when required trust proofs are missing for a request
-    """
--- a/apps/cic-eth/cic_eth/eth/account.py
+++ b/apps/cic-eth/cic_eth/eth/account.py
@@ -13,8 +13,11 @@ from chainlib.eth.sign import (
        new_account,
        sign_message,
        )
-from chainlib.eth.address import to_checksum_address, is_address
-from chainlib.eth.tx import TxFormat
+from chainlib.eth.address import to_checksum_address
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
@@ -31,7 +34,6 @@ from cic_eth.eth.gas import (
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
-from cic_eth.encode import tx_normalize
 from cic_eth.error import (
        RoleMissingError,
        SignerError,
@@ -47,11 +49,6 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
        register_tx,
        )
-from cic_eth.encode import (
-        unpack_normal,
-        ZERO_ADDRESS_NORMAL,
-        tx_normalize,
-        )

 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -86,7 +83,7 @@ def create(self, password, chain_spec_dict):
    # TODO: It seems infeasible that a can be None in any case, verify
    if a == None:
        raise SignerError('create account')
-    a = tx_normalize.wallet_address(a)
+        
    logg.debug('created account {}'.format(a))

    # Initialize nonce provider record for account
@@ -177,9 +174,6 @@ def gift(self, account_address, chain_spec_dict):
    """
    chain_spec = ChainSpec.from_dict(chain_spec_dict)

-    if is_address(account_address):
-        account_address = tx_normalize.wallet_address(account_address)
-
    logg.debug('gift account address {} to index'.format(account_address))
    queue = self.request.delivery_info.get('routing_key')

@@ -253,9 +247,8 @@ def have(self, account, chain_spec_dict):

@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):
-    if not is_address(address):
-        raise ValueError('invalid address {}'.format(address))
-    address = tx_normalize.wallet_address(address)
+    if not to_checksum_address(address):
+        raise ValueError('invalid checksum address {}'.format(address))
    session = SessionBase.create_session()
    role = AccountRole.set(tag, address, session=session) 
    session.add(role)
@@ -302,19 +295,17 @@ def cache_gift_data(
    chain_spec = ChainSpec.from_dict(chain_spec_dict)

    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
    tx_data = Faucet.parse_give_to_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx['to'])

    session = self.create_session()

    tx_dict = {
-            'hash': tx['hash'],
-            'from': sender_address,
-            'to': recipient_address,
-            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
            'from_value': 0,
            'to_value': 0,
            }
@@ -343,19 +334,17 @@ def cache_account_data(
    :rtype: tuple
    """
    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
    tx_data = AccountsIndex.parse_add_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx['to'])

    session = SessionBase.create_session()
    tx_dict = {
-            'hash': tx['hash'],
-            'from': sender_address,
-            'to': recipient_address,
-            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'hash': tx_hash_hex,
+            'from': tx['from'],
+            'to': tx['to'],
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
            'from_value': 0,
            'to_value': 0,
            }
--- a/apps/cic-eth/cic_eth/eth/erc20.py
+++ b/apps/cic-eth/cic_eth/eth/erc20.py
@@ -12,14 +12,10 @@ from chainlib.eth.tx import (
        )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
+from hexathon import strip_0x
 from chainqueue.error import NotLocalTxError
 from eth_erc20 import ERC20
 from chainqueue.sql.tx import cache_tx_dict
-from okota.token_index import to_identifier

 # local imports
 from cic_eth.db.models.base import SessionBase
@@ -40,11 +36,8 @@ from cic_eth.task import (
        CriticalSQLAlchemyTask,
        CriticalWeb3Task,
        CriticalSQLAlchemyAndSignerTask,
-        BaseTask,
    )
 from cic_eth.eth.nonce import CustodialTaskNonceOracle
-from cic_eth.encode import tx_normalize
-from cic_eth.eth.trust import verify_proofs

 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -69,8 +62,7 @@ def balance(tokens, holder_address, chain_spec_dict):

    for t in tokens:
        address = t['address']
-        logg.debug('address {} {}'.format(address, holder_address))
-        token = ERC20Token(chain_spec, rpc, add_0x(address))
+        token = ERC20Token(chain_spec, rpc, address)
        c = ERC20(chain_spec)
        o = c.balance_of(address, holder_address, sender_address=caller_address)
        r = rpc.do(o)
@@ -379,15 +371,13 @@ def cache_transfer_data(
    tx = unpack(tx_signed_raw_bytes, chain_spec)

    tx_data = ERC20.parse_transfer_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx_data[0])
+    recipient_address = tx_data[0]
    token_value = tx_data[1]

-
    session = SessionBase.create_session()
    tx_dict = {
            'hash': tx_hash_hex,
-            'from': sender_address,
+            'from': tx['from'],
            'to': recipient_address,
            'source_token': tx['to'],
            'destination_token': tx['to'],
@@ -458,14 +448,13 @@ def cache_approve_data(
    tx = unpack(tx_signed_raw_bytes, chain_spec)

    tx_data = ERC20.parse_approve_request(tx['data'])
-    sender_address = tx_normalize.wallet_address(tx['from'])
-    recipient_address = tx_normalize.wallet_address(tx_data[0])
+    recipient_address = tx_data[0]
    token_value = tx_data[1]

    session = SessionBase.create_session()
    tx_dict = {
            'hash': tx_hash_hex,
-            'from': sender_address,
+            'from': tx['from'],
            'to': recipient_address,
            'source_token': tx['to'],
            'destination_token': tx['to'],
@@ -476,69 +465,3 @@ def cache_approve_data(
    session.close()
    return (tx_hash_hex, cache_id)

-
-@celery_app.task(bind=True, base=BaseTask)
-def token_info(self, tokens, chain_spec_dict, proofs=[]):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    i = 0
-
-    for token in tokens:
-        result_data = []
-        token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
-        token_chain_object.load(rpc)
-
-        token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
-        token_proofs = [token_symbol_proof_hex]
-        if len(proofs) > 0:
-            token_proofs += proofs[i]
-
-        tokens[i] = {
-            'decimals': token_chain_object.decimals,
-            'name': token_chain_object.name,
-            'symbol': token_chain_object.symbol,
-            'address': tx_normalize.executable_address(token_chain_object.address),
-            'proofs': token_proofs,
-            'converters': tokens[i]['converters'],
-                }   
-        i += 1
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    for token in tokens:
-        s = celery.signature(
-                'cic_eth.eth.trust.verify_proofs',
-                [
-                    token,
-                    token['address'],
-                    token['proofs'],
-                    chain_spec_dict,
-                    success_callback,
-                    error_callback,
-                    ],
-                queue=queue,
-                )
-
-        if success_callback != None:
-            s.link(success_callback)
-        if error_callback != None:
-            s.on_error(error_callback)
-        s.apply_async()
-
-    return tokens
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-        'symbol': self.default_token_symbol,
-        'address': self.default_token_address,
-        'name': self.default_token_name,
-        'decimals': self.default_token_decimals,
-        }
--- a/apps/cic-eth/cic_eth/eth/gas.py
+++ b/apps/cic-eth/cic_eth/eth/gas.py
@@ -3,17 +3,10 @@ import logging

 # external imports
 import celery
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-#from chainlib.eth.constant import ZERO_ADDRESS
+from hexathon import strip_0x
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import (
-        is_checksum_address,
-        to_checksum_address,
-        is_address
-        )
+from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.tx import cache_tx_dict
@@ -28,6 +21,7 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
        TxFactory,
        TxFormat,
+        unpack,
        )
 from chainlib.eth.contract import (
        abi_decode_single,
@@ -51,7 +45,6 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
        queue_create,
        register_tx,
-        unpack,
        )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -60,11 +53,6 @@ from cic_eth.task import (
        CriticalSQLAlchemyAndSignerTask,
        CriticalWeb3AndSignerTask,
        )
-from cic_eth.encode import (
-        tx_normalize,
-        ZERO_ADDRESS_NORMAL,
-        unpack_normal,
-        )

 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -142,16 +130,16 @@ def cache_gas_data(
    """
    chain_spec = ChainSpec.from_dict(chain_spec_dict)
    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx = unpack(tx_signed_raw_bytes, chain_spec)

    session = SessionBase.create_session()

    tx_dict = {
-            'hash': tx['hash'],
+            'hash': tx_hash_hex,
            'from': tx['from'],
            'to': tx['to'],
-            'source_token': ZERO_ADDRESS_NORMAL,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'source_token': ZERO_ADDRESS,
+            'destination_token': ZERO_ADDRESS,
            'from_value': tx['value'],
            'to_value': tx['value'],
            }
@@ -162,7 +150,7 @@ def cache_gas_data(


@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
    """Check the gas level of the sender address of a transaction.

    If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -182,23 +170,8 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
    :return: Signed raw transaction data list
    :rtype: param txs, unchanged
    """
-    rpc_format_address = None
-    if address != None:
-        if not is_address(address):
-            raise ValueError('invalid address {}'.format(address))
-        address = tx_normalize.wallet_address(address)
-        address = add_0x(address)
-
-    tx_hashes = []
-    txs = []
-    for tx_hash in tx_hashes_hex:
-        tx_hash = tx_normalize.tx_hash(tx_hash)
-        tx_hashes.append(tx_hash)
-    for tx in txs_hex:
-        tx = tx_normalize.tx_wire(tx)
-        txs.append(tx)
-
    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))

    addresspass = None
    if len(txs) == 0:
@@ -214,7 +187,8 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
                raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
            addresspass.append(address)

-    rpc_format_address = add_0x(to_checksum_address(address))
+    if not is_checksum_address(address):
+        raise ValueError('invalid address {}'.format(address))

    queue = self.request.delivery_info.get('routing_key')

@@ -222,7 +196,7 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga

    gas_balance = 0
    try:
-        o = balance(rpc_format_address)
+        o = balance(address)
        r = conn.do(o)
        conn.disconnect()
        gas_balance = abi_decode_single(ABIContractType.UINT256, r)
@@ -330,7 +304,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
    # Determine value of gas tokens to send
    # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
    # however, we will perform a 0-value transaction instead
-    recipient_address = tx_normalize.wallet_address(recipient_address)
    zero_amount = False
    session = SessionBase.create_session()
    status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -405,7 +378,6 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
    :returns: Transaction hash
    :rtype: str, 0x-hex
    """
-    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
    session = SessionBase.create_session()

    otx = Otx.load(txold_hash_hex, session)
--- a/apps/cic-eth/cic_eth/eth/meta.py
+++ b/apps/cic-eth/cic_eth/eth/meta.py
@@ -2,9 +2,8 @@
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.status import Status as TxStatus
 from cic_eth_registry.erc20 import ERC20Token
-from hexathon import add_0x

-# local impor:ts
+# local imports
 from cic_eth.ext.address import translate_address


@@ -45,8 +44,8 @@ class ExtendedTx:
            destination = source
        if destination_value == None:
            destination_value = source_value
-        st = ERC20Token(self.chain_spec, self.rpc, add_0x(source))
-        dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination))
+        st = ERC20Token(self.chain_spec, self.rpc, source)
+        dt = ERC20Token(self.chain_spec, self.rpc, destination)
        self.source_token = source
        self.source_token_symbol = st.symbol
        self.source_token_name = st.name
--- a/apps/cic-eth/cic_eth/eth/nonce.py
+++ b/apps/cic-eth/cic_eth/eth/nonce.py
@@ -3,12 +3,11 @@ import logging

 # external imports
 import celery
-from chainlib.eth.address import is_checksum_address, is_address, strip_0x
+from chainlib.eth.address import is_checksum_address

 # local imports
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import tx_normalize
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.nonce import (
        Nonce,
@@ -43,8 +42,7 @@ class CustodialTaskNonceOracle():
        :returns: Nonce
        :rtype: number
        """
-        address = tx_normalize.wallet_address(self.address)
-        r = NonceReservation.release(address, self.uuid, session=self.session)
+        r = NonceReservation.release(self.address, self.uuid, session=self.session)
        return r[1]


@@ -60,18 +58,17 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
        address = chained_input
        logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
    else:
-        if is_address(signer_address):
+        if is_checksum_address(signer_address):
            address = signer_address
            logg.debug('explicit address for reserve nonce {}'.format(signer_address))
        else:
            address = AccountRole.get_address(signer_address, session=session)
            logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))

-    if not is_address(address):
+    if not is_checksum_address(address):
        raise ValueError('invalid result when resolving address for nonce {}'.format(address))

    root_id = self.request.root_id
-    address = tx_normalize.wallet_address(address)
    r = NonceReservation.next(address, root_id, session=session)
    logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))

--- a/apps/cic-eth/cic_eth/eth/trust.py
+++ b/apps/cic-eth/cic_eth/eth/trust.py
@@ -1,77 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from eth_address_declarator import Declarator
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from cic_eth.db.models.role import AccountRole
-from cic_eth_registry import CICRegistry
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.task import BaseTask
-from cic_eth.error import TrustError
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
-    proof = strip_0x(proof)
-
-    proofs = []
- 
-    logg.debug('proof count {}'.format(len(proofs)))
-    if len(proofs) == 0:
-        logg.debug('error {}'.format(len(proofs)))
-        raise TrustError('foo')
-
-    return (chained_input, (proof, proofs))
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
-    queue = self.request.delivery_info.get('routing_key')
-
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    session = self.create_session()
-    sender_address = AccountRole.get_address('DEFAULT', session)
-
-    registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-
-    declarator = Declarator(chain_spec)
-
-    have_proofs = {}
-
-    for proof in proofs:
-
-        proof = strip_0x(proof)
-
-        have_proofs[proof] = []
-
-        for trusted_address in self.trusted_addresses:
-            o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
-            r = rpc.do(o)
-            declarations = declarator.parse_declaration(r)
-            logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
-
-            for declaration in declarations:
-                declaration = strip_0x(declaration)
-                if declaration == proof:
-                    logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
-                    have_proofs[proof].append(trusted_address)
-
-    out_proofs = {}
-    for proof in have_proofs.keys():
-        if len(have_proofs[proof]) == 0:
-            logg.error('missing signer for proof {} subject {}'.format(proof, subject))
-            raise TrustError((subject, proof,))
-        out_proofs[proof] = have_proofs[proof]
-       
-    return (chained_input, out_proofs)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
nolash	ea0e2f89d6	Reinstate skipped admin noncritical tests	2021-07-21 14:13:56 +02:00
nolash	135bdfe45f	Add cov to notify tests	2021-07-21 14:11:21 +02:00
nolash	c3e2738860	Remove cic-base from cic-notify and cic-ussd	2021-07-21 13:34:08 +02:00
nolash	7bf21ae6d8	Upgrade cic-base in notify	2021-07-21 12:18:45 +02:00
nolash	882c48b806	Rehabilitate contract-migration after chainlib nextgen and chainqueue upgrade	2021-07-21 08:49:05 +02:00
nolash	04e05c9d45	Make tests pass after chainqueue upgrade	2021-07-21 08:23:12 +02:00
nolash	839c24f748	WIP dep hell with chainlib nextgen contd	2021-07-21 06:20:35 +02:00
nolash	fbb3609202	WIP get nextgen dep hell solved	2021-07-20 22:40:33 +02:00
nolash	21b531aa12	Upgrade cic-teh tools	2021-07-20 20:38:43 +02:00
nolash	756124bada	Upgrade all deps to chainlib nextgen	2021-07-20 20:29:21 +02:00
				`@@ -0,0 +1 @@`
				`## this is an example base image if we wanted one for all the other apps. Its just OS level things`
				`@@ -1 +0,0 @@`
				`include requirements.txt cic_cache/data/config/`